Star873
1771
代码
代码
Issues223
Pull Requests304
Actions
项目讨论
Wiki
分析
Star873
1771
已合并
refactor: arch35 算子 MicroAPI/Reg 旧接口替换为新接口 #8450
yuanbin_22创建于 18 天前
refactor: arch35 算子 MicroAPI/Reg 旧接口替换为新接口 #8450
已合并
yuanbin_22创建于 18 天前
从yuanbin_22/ops-nnrefactor-microapi-ops合入到cann/ops-nnmaster
共 108 个文件变更+2790-2805
Mactivation/elu/op_kernel/arch35/elu_dag.h+17-13
@@ -23,10 +23,12 @@ namespace EluOp {
23using namespace Ops::Base;23using namespace Ops::Base;
24using namespace AscendC;24using namespace AscendC;
25#ifdef __CCE_AICORE__25#ifdef __CCE_AICORE__
26- constexpr static AscendC::MicroAPI::CastTrait castTrait0 = { AscendC::MicroAPI::RegLayout::ZERO,26+constexpr static AscendC::MicroAPI::CastTrait castTrait0 = {
27- AscendC::MicroAPI::SatMode::UNKNOWN, AscendC::MicroAPI::MaskMergeMode::ZEROING, AscendC::RoundMode::UNKNOWN };27+ AscendC::MicroAPI::RegLayout::ZERO, AscendC::MicroAPI::SatMode::UNKNOWN, AscendC::MicroAPI::MaskMergeMode::ZEROING,
28- constexpr static AscendC::MicroAPI::CastTrait castTrait1 = { AscendC::MicroAPI::RegLayout::ZERO,28+ AscendC::RoundMode::UNKNOWN};
29- AscendC::MicroAPI::SatMode::NO_SAT, AscendC::MicroAPI::MaskMergeMode::ZEROING, AscendC::RoundMode::CAST_RINT };29+constexpr static AscendC::MicroAPI::CastTrait castTrait1 = {
30+ AscendC::MicroAPI::RegLayout::ZERO, AscendC::MicroAPI::SatMode::NO_SAT, AscendC::MicroAPI::MaskMergeMode::ZEROING,
31+ AscendC::RoundMode::CAST_RINT};
30#endif32#endif
31constexpr int ELU_ATTR_ALPHA_INDEX = 0;33constexpr int ELU_ATTR_ALPHA_INDEX = 0;
32constexpr int ELU_ATTR_SCALE_INDEX = 1;34constexpr int ELU_ATTR_SCALE_INDEX = 1;
@@ -71,7 +73,7 @@ struct EluCustom : public Vec::ElemwiseQuaternaryOP<T, T, float, float, float> {
71 for (uint16_t loopIdx = 0; loopIdx < static_cast<uint16_t>(loopNum); loopIdx++) {73 for (uint16_t loopIdx = 0; loopIdx < static_cast<uint16_t>(loopNum); loopIdx++) {
72 mask = MicroAPI::UpdateMask<T, MicroAPI::RegTraitNumOne>(count);74 mask = MicroAPI::UpdateMask<T, MicroAPI::RegTraitNumOne>(count);
73 // OpCopyIn75 // OpCopyIn
74- MicroAPI::DataCopy(vregInput, (__ubuf__ T*)(srcAddr + loopIdx * vlSize));76+ MicroAPI::LoadAlign(vregInput, (__ubuf__ T*)(srcAddr + loopIdx * vlSize));
75 MicroAPI::Muls(vregNeg, vregInput, inputScale, mask);77 MicroAPI::Muls(vregNeg, vregInput, inputScale, mask);
76 78 
77 // Polynomial expm1(z) = z + z^2/2 + z^3/6 + z^4/24 + z^5/120 for |z| < threshold79 // Polynomial expm1(z) = z + z^2/2 + z^3/6 + z^4/24 + z^5/120 for |z| < threshold
@@ -92,17 +94,17 @@ struct EluCustom : public Vec::ElemwiseQuaternaryOP<T, T, float, float, float> {
92 94 
93 // Select expm1 result based on |z|95 // Select expm1 result based on |z|
94 MicroAPI::Abs(vregAbsZ, vregNeg, mask);96 MicroAPI::Abs(vregAbsZ, vregNeg, mask);
95- MicroAPI::CompareScalar<T, CMPMODE::LT>(cmpExpm1, vregAbsZ, expm1Threshold, mask);97+ MicroAPI::Compares<T, CMPMODE::LT>(cmpExpm1, vregAbsZ, expm1Threshold, mask);
96 MicroAPI::Select<T>(vregNeg, vregPoly, vregExp, cmpExpm1);98 MicroAPI::Select<T>(vregNeg, vregPoly, vregExp, cmpExpm1);
97 99 
98 MicroAPI::Muls(vregNeg, vregNeg, alpha, mask);100 MicroAPI::Muls(vregNeg, vregNeg, alpha, mask);
99 101 
100- MicroAPI::CompareScalar<T, CMPMODE::GT>(cmpMask, vregInput, (float)0.0, mask);102+ MicroAPI::Compares<T, CMPMODE::GT>(cmpMask, vregInput, (float)0.0, mask);
101 MicroAPI::Select<T>(vregOutput, vregInput, vregNeg, cmpMask);103 MicroAPI::Select<T>(vregOutput, vregInput, vregNeg, cmpMask);
102 MicroAPI::Muls(vregOutput, vregOutput, scale, mask);104 MicroAPI::Muls(vregOutput, vregOutput, scale, mask);
103 105 
104 // OpCopyOut106 // OpCopyOut
105- MicroAPI::DataCopy((__ubuf__ T*)(dstAddr + loopIdx * vlSize), vregOutput, mask);107+ MicroAPI::StoreAlign((__ubuf__ T*)(dstAddr + loopIdx * vlSize), vregOutput, mask);
106 }108 }
107 }109 }
108 } else {110 } else {
@@ -111,7 +113,8 @@ struct EluCustom : public Vec::ElemwiseQuaternaryOP<T, T, float, float, float> {
111 for (uint16_t loopIdx = 0; loopIdx < static_cast<uint16_t>(loopNum); loopIdx++) {113 for (uint16_t loopIdx = 0; loopIdx < static_cast<uint16_t>(loopNum); loopIdx++) {
112 mask = MicroAPI::UpdateMask<float, MicroAPI::RegTraitNumOne>(count);114 mask = MicroAPI::UpdateMask<float, MicroAPI::RegTraitNumOne>(count);
113 // OpCopyIn115 // OpCopyIn
114- MicroAPI::DataCopy<T, MicroAPI::LoadDist::DIST_UNPACK_B16>(vregInput, (__ubuf__ T*)(srcAddr + loopIdx * vlSize));116+ MicroAPI::LoadAlign<T, MicroAPI::LoadDist::DIST_UNPACK_B16>(
117+ vregInput, (__ubuf__ T*)(srcAddr + loopIdx * vlSize));
115 MicroAPI::Cast<float, T, castTrait0>(vregInputFloat, vregInput, mask);118 MicroAPI::Cast<float, T, castTrait0>(vregInputFloat, vregInput, mask);
116 MicroAPI::Muls(vregNeg, vregInputFloat, inputScale, mask);119 MicroAPI::Muls(vregNeg, vregInputFloat, inputScale, mask);
117 120 
@@ -131,19 +134,20 @@ struct EluCustom : public Vec::ElemwiseQuaternaryOP<T, T, float, float, float> {
131 134 
132 // Select expm1 result based on |z|135 // Select expm1 result based on |z|
133 MicroAPI::Abs(vregAbsZ, vregNeg, mask);136 MicroAPI::Abs(vregAbsZ, vregNeg, mask);
134- MicroAPI::CompareScalar<float, CMPMODE::LT>(cmpExpm1, vregAbsZ, expm1Threshold, mask);137+ MicroAPI::Compares<float, CMPMODE::LT>(cmpExpm1, vregAbsZ, expm1Threshold, mask);
135 MicroAPI::Select<float>(vregNeg, vregPoly, vregExp, cmpExpm1);138 MicroAPI::Select<float>(vregNeg, vregPoly, vregExp, cmpExpm1);
136 139 
137 MicroAPI::Muls(vregNeg, vregNeg, alpha, mask);140 MicroAPI::Muls(vregNeg, vregNeg, alpha, mask);
138 141 
139- MicroAPI::CompareScalar<float, CMPMODE::GT>(cmpMask, vregInputFloat, (float)0.0, mask);142+ MicroAPI::Compares<float, CMPMODE::GT>(cmpMask, vregInputFloat, (float)0.0, mask);
140 MicroAPI::Select<float>(vregOutputFloat, vregInputFloat, vregNeg, cmpMask);143 MicroAPI::Select<float>(vregOutputFloat, vregInputFloat, vregNeg, cmpMask);
141 MicroAPI::Muls(vregOutputFloat, vregOutputFloat, scale, mask);144 MicroAPI::Muls(vregOutputFloat, vregOutputFloat, scale, mask);
142 MicroAPI::Cast<T, float, castTrait1>(vregOutput, vregOutputFloat, mask);145 MicroAPI::Cast<T, float, castTrait1>(vregOutput, vregOutputFloat, mask);
143 // OpCopyOut146 // OpCopyOut
144- MicroAPI::DataCopy<T, MicroAPI::StoreDist::DIST_PACK_B32>((__ubuf__ T*)(dstAddr + loopIdx * vlSize), vregOutput, mask);147+ MicroAPI::StoreAlign<T, MicroAPI::StoreDist::DIST_PACK_B32>(
148+ (__ubuf__ T*)(dstAddr + loopIdx * vlSize), vregOutput, mask);
145 }149 }
146- } 150+ }
147 }151 }
148#endif152#endif
149 }153 }
Mactivation/fast_gelu/op_kernel/arch35/fast_gelu_dag.h+3-3
@@ -43,14 +43,14 @@ struct FastGeluCustom : public Vec::ElemwiseUnaryOP<T, T> {
43 43 
44 for (uint16_t loopIdx = 0; loopIdx < loopNum; loopIdx++) {44 for (uint16_t loopIdx = 0; loopIdx < loopNum; loopIdx++) {
45 mask = AscendC::MicroAPI::UpdateMask<T, AscendC::MicroAPI::RegTraitNumOne>(count);45 mask = AscendC::MicroAPI::UpdateMask<T, AscendC::MicroAPI::RegTraitNumOne>(count);
46- AscendC::MicroAPI::DataCopy(x, (__ubuf__ T*)(srcAddr + loopIdx * vlSize));46+ AscendC::MicroAPI::LoadAlign(x, (__ubuf__ T*)(srcAddr + loopIdx * vlSize));
47 AscendC::MicroAPI::Muls(denominator, x, value1, mask);47 AscendC::MicroAPI::Muls(denominator, x, value1, mask);
48 AscendC::MicroAPI::Exp(denominator, denominator, mask);48 AscendC::MicroAPI::Exp(denominator, denominator, mask);
49 AscendC::MicroAPI::Adds(denominator, denominator, value2, mask);49 AscendC::MicroAPI::Adds(denominator, denominator, value2, mask);
50 // result = x / (Exp(-1.702 * x) + 1)50 // result = x / (Exp(-1.702 * x) + 1)
51 AscendC::MicroAPI::Div<T, &mode>(result, x, denominator, mask);51 AscendC::MicroAPI::Div<T, &mode>(result, x, denominator, mask);
52 // OpCopyOut52 // OpCopyOut
53- AscendC::MicroAPI::DataCopy((__ubuf__ T*)(dstAddr + loopIdx * vlSize), result, mask);53+ AscendC::MicroAPI::StoreAlign((__ubuf__ T*)(dstAddr + loopIdx * vlSize), result, mask);
54 }54 }
55 }55 }
56#endif56#endif
@@ -83,5 +83,5 @@ struct FastGeluNeedCast {
83 using OpDag = DAGSch<Outputs>;83 using OpDag = DAGSch<Outputs>;
84};84};
85 85 
86-}; // namespace FastGeluDag86+}; // namespace FastGeluDag
87#endif // CANN_CUSTOM_OPS_FAST_GELU_DAG_H87#endif // CANN_CUSTOM_OPS_FAST_GELU_DAG_H
Mactivation/fast_gelu/tests/ut/op_kernel/test_fast_gelu_apt.cpp+2-2
@@ -55,7 +55,7 @@ TEST_F(fast_gelu_test, test_case_fp32_1)
55 55 
56 tilingDatafromBin->dim0 = 256;56 tilingDatafromBin->dim0 = 256;
57 tilingDatafromBin->coreNum = 1;57 tilingDatafromBin->coreNum = 1;
58- tilingDatafromBin->ubFormer = 256;58+ tilingDatafromBin->ubFormer = 1024;
59 tilingDatafromBin->blockFormer = 256;59 tilingDatafromBin->blockFormer = 256;
60 tilingDatafromBin->blockNum = 1;60 tilingDatafromBin->blockNum = 1;
61 tilingDatafromBin->ubLoopOfFormerBlock = 1;61 tilingDatafromBin->ubLoopOfFormerBlock = 1;
@@ -77,4 +77,4 @@ TEST_F(fast_gelu_test, test_case_fp32_1)
77 AscendC::GmFree(y);77 AscendC::GmFree(y);
78 AscendC::GmFree(workspace);78 AscendC::GmFree(workspace);
79 AscendC::GmFree(tiling);79 AscendC::GmFree(tiling);
80-}80+}
Mactivation/fast_gelu_grad/op_kernel/arch35/fast_gelu_grad_dag.h+4-4
@@ -51,7 +51,7 @@ struct FastGeluGradCustom : public Vec::ElemwiseBinaryOP<T, T, T> {
51 for (uint16_t loopIdx = 0; loopIdx < loopNum; loopIdx++) {51 for (uint16_t loopIdx = 0; loopIdx < loopNum; loopIdx++) {
52 mask = AscendC::MicroAPI::UpdateMask<T, AscendC::MicroAPI::RegTraitNumOne>(count);52 mask = AscendC::MicroAPI::UpdateMask<T, AscendC::MicroAPI::RegTraitNumOne>(count);
53 // OpCopyIn053 // OpCopyIn0
54- AscendC::MicroAPI::DataCopy(x, (__ubuf__ T*)(src2Addr + loopIdx * vlSize));54+ AscendC::MicroAPI::LoadAlign(x, (__ubuf__ T*)(src2Addr + loopIdx * vlSize));
55 // temp1Reg = e^(-1.702x) + 155 // temp1Reg = e^(-1.702x) + 1
56 AscendC::MicroAPI::Muls(value1MulsX, x, value2, mask);56 AscendC::MicroAPI::Muls(value1MulsX, x, value2, mask);
57 AscendC::MicroAPI::Exp(temp1Reg, value1MulsX, mask);57 AscendC::MicroAPI::Exp(temp1Reg, value1MulsX, mask);
@@ -63,11 +63,11 @@ struct FastGeluGradCustom : public Vec::ElemwiseBinaryOP<T, T, T> {
63 AscendC::MicroAPI::Mul(temp2Reg, temp2Reg, value1MulsX, mask);63 AscendC::MicroAPI::Mul(temp2Reg, temp2Reg, value1MulsX, mask);
64 AscendC::MicroAPI::Adds(temp2Reg, temp2Reg, value3, mask);64 AscendC::MicroAPI::Adds(temp2Reg, temp2Reg, value3, mask);
65 AscendC::MicroAPI::Mul(divRes, temp2Reg, divRes, mask);65 AscendC::MicroAPI::Mul(divRes, temp2Reg, divRes, mask);
66- AscendC::MicroAPI::DataCopy(dy, (__ubuf__ T*)(src1Addr + loopIdx * vlSize));66+ AscendC::MicroAPI::LoadAlign(dy, (__ubuf__ T*)(src1Addr + loopIdx * vlSize));
67 AscendC::MicroAPI::Mul(divRes, dy, divRes, mask);67 AscendC::MicroAPI::Mul(divRes, dy, divRes, mask);
68 68 
69 // OpCopyOut69 // OpCopyOut
70- AscendC::MicroAPI::DataCopy((__ubuf__ T*)(dstAddr + loopIdx * vlSize), divRes, mask);70+ AscendC::MicroAPI::StoreAlign((__ubuf__ T*)(dstAddr + loopIdx * vlSize), divRes, mask);
71 }71 }
72 }72 }
73#endif73#endif
@@ -99,5 +99,5 @@ struct FastGeluGradNoCast {
99 using OpDag = DAGSch<Outputs>;99 using OpDag = DAGSch<Outputs>;
100};100};
101 101 
102-}; // namespace FastGeluGradDag102+}; // namespace FastGeluGradDag
103#endif // CANN_CUSTOM_OPS_FAST_GELU_GRAD_DAG_H103#endif // CANN_CUSTOM_OPS_FAST_GELU_GRAD_DAG_H
Mactivation/fast_gelu_grad/tests/ut/op_kernel/test_fast_gelu_grad_apt.cpp+2-2
@@ -59,7 +59,7 @@ TEST_F(fast_gelu_grad_test, test_case_fp32_1)
59 59 
60 tilingDatafromBin->dim0 = 256;60 tilingDatafromBin->dim0 = 256;
61 tilingDatafromBin->coreNum = 1;61 tilingDatafromBin->coreNum = 1;
62- tilingDatafromBin->ubFormer = 256;62+ tilingDatafromBin->ubFormer = 1024;
63 tilingDatafromBin->blockFormer = 256;63 tilingDatafromBin->blockFormer = 256;
64 tilingDatafromBin->blockNum = 1;64 tilingDatafromBin->blockNum = 1;
65 tilingDatafromBin->ubLoopOfFormerBlock = 1;65 tilingDatafromBin->ubLoopOfFormerBlock = 1;
@@ -83,4 +83,4 @@ TEST_F(fast_gelu_grad_test, test_case_fp32_1)
83 AscendC::GmFree(z);83 AscendC::GmFree(z);
84 AscendC::GmFree(workspace);84 AscendC::GmFree(workspace);
85 AscendC::GmFree(tiling);85 AscendC::GmFree(tiling);
86-}86+}
Mactivation/gelu/op_kernel/arch35/gelu_dag.h+3-3
@@ -52,7 +52,7 @@ struct GeluCustom : public Vec::ElemwiseUnaryOP<T, T> {
52 for (uint16_t loopIdx = 0; loopIdx < loopNum; loopIdx++) {52 for (uint16_t loopIdx = 0; loopIdx < loopNum; loopIdx++) {
53 mask = MicroAPI::UpdateMask<T, MicroAPI::RegTraitNumOne>(count);53 mask = MicroAPI::UpdateMask<T, MicroAPI::RegTraitNumOne>(count);
54 // OpCopyIn54 // OpCopyIn
55- MicroAPI::DataCopy(vregInput, (__ubuf__ T*)(srcAddr + loopIdx * vlSize));55+ MicroAPI::LoadAlign(vregInput, (__ubuf__ T*)(srcAddr + loopIdx * vlSize));
56 MicroAPI::Mul(vregInputSqr, vregInput, vregInput, mask);56 MicroAPI::Mul(vregInputSqr, vregInput, vregInput, mask);
57 MicroAPI::Mul(vregInputCub, vregInputSqr, vregInput, mask);57 MicroAPI::Mul(vregInputCub, vregInputSqr, vregInput, mask);
58 MicroAPI::Axpy(vregInputCub, vregInput, TANH_APPROX_FACTOR, mask);58 MicroAPI::Axpy(vregInputCub, vregInput, TANH_APPROX_FACTOR, mask);
@@ -62,7 +62,7 @@ struct GeluCustom : public Vec::ElemwiseUnaryOP<T, T> {
62 MicroAPI::Div(vregOutput, vregInput, vregInputCub, mask);62 MicroAPI::Div(vregOutput, vregInput, vregInputCub, mask);
63 63 
64 // OpCopyOut64 // OpCopyOut
65- MicroAPI::DataCopy((__ubuf__ T*)(dstAddr + loopIdx * vlSize), vregOutput, mask);65+ MicroAPI::StoreAlign((__ubuf__ T*)(dstAddr + loopIdx * vlSize), vregOutput, mask);
66 }66 }
67 }67 }
68 }68 }
@@ -85,4 +85,4 @@ struct GeluDAG {
85 using OpDag = DAGSch<Outputs, void, MemCfg>;85 using OpDag = DAGSch<Outputs, void, MemCfg>;
86};86};
87} // namespace GeluOp87} // namespace GeluOp
88-#endif // CANN_CUSTOM_OPS_GELU_DAG_H88+#endif // CANN_CUSTOM_OPS_GELU_DAG_H
Mactivation/gelu_grad/op_kernel/arch35/gelu_grad_dag.h+4-4
@@ -72,8 +72,8 @@ struct GeluGradCustom : public Vec::ElemwiseBinaryOP<T, T, T> {
72 mask = MicroAPI::UpdateMask<T, MicroAPI::RegTraitNumOne>(count);72 mask = MicroAPI::UpdateMask<T, MicroAPI::RegTraitNumOne>(count);
73 MicroAPI::Duplicate(vregInputPX, BETAN);73 MicroAPI::Duplicate(vregInputPX, BETAN);
74 // OpCopyIn74 // OpCopyIn
75- MicroAPI::DataCopy(vregInputDy, (__ubuf__ T*)(src0Addr + loopIdx * vlSize));75+ MicroAPI::LoadAlign(vregInputDy, (__ubuf__ T*)(src0Addr + loopIdx * vlSize));
76- MicroAPI::DataCopy(vregInputX, (__ubuf__ T*)(src1Addr + loopIdx * vlSize));76+ MicroAPI::LoadAlign(vregInputX, (__ubuf__ T*)(src1Addr + loopIdx * vlSize));
77 // compute77 // compute
78 MicroAPI::Mul(vregInputXSqr, vregInputX, vregInputX, mask);78 MicroAPI::Mul(vregInputXSqr, vregInputX, vregInputX, mask);
79 MicroAPI::Axpy(vregInputPX, vregInputXSqr, AN, mask);79 MicroAPI::Axpy(vregInputPX, vregInputXSqr, AN, mask);
@@ -96,7 +96,7 @@ struct GeluGradCustom : public Vec::ElemwiseBinaryOP<T, T, T> {
96 MicroAPI::Mul(vregOutput, vregInputDy, vregInputResp, mask);96 MicroAPI::Mul(vregOutput, vregInputDy, vregInputResp, mask);
97 97 
98 // OpCopyOut98 // OpCopyOut
99- MicroAPI::DataCopy((__ubuf__ T*)(dstAddr + loopIdx * vlSize), vregOutput, mask);99+ MicroAPI::StoreAlign((__ubuf__ T*)(dstAddr + loopIdx * vlSize), vregOutput, mask);
100 }100 }
101 }101 }
102 }102 }
@@ -122,4 +122,4 @@ struct GeluGradDAG {
122};122};
123} // namespace GeluGradOp123} // namespace GeluGradOp
124 124 
125-#endif // CANN_CUSTOM_OPS_GELU_GRAD_DAG_H125+#endif // CANN_CUSTOM_OPS_GELU_GRAD_DAG_H
Mactivation/gelu_grad_v2/op_kernel/arch35/gelu_grad_v2_dag.h+8-8
@@ -75,9 +75,9 @@ struct GeluGradV2ErfPost : public Vec::ElemwiseTernaryOP<T, T, T, T> {
75 for (uint16_t loopIdx = 0; loopIdx < loopNum; loopIdx++) {75 for (uint16_t loopIdx = 0; loopIdx < loopNum; loopIdx++) {
76 mask = MicroAPI::UpdateMask<T, MicroAPI::RegTraitNumOne>(count);76 mask = MicroAPI::UpdateMask<T, MicroAPI::RegTraitNumOne>(count);
77 // OpCopyIn77 // OpCopyIn
78- MicroAPI::DataCopy(vregInput0, (__ubuf__ T*)(src0Addr + loopIdx * vlSize));78+ MicroAPI::LoadAlign(vregInput0, (__ubuf__ T*)(src0Addr + loopIdx * vlSize));
79- MicroAPI::DataCopy(vregInput1, (__ubuf__ T*)(src1Addr + loopIdx * vlSize));79+ MicroAPI::LoadAlign(vregInput1, (__ubuf__ T*)(src1Addr + loopIdx * vlSize));
80- MicroAPI::DataCopy(vregInput2, (__ubuf__ T*)(src2Addr + loopIdx * vlSize));80+ MicroAPI::LoadAlign(vregInput2, (__ubuf__ T*)(src2Addr + loopIdx * vlSize));
81 MicroAPI::Muls(vregCdfMuls, vregInput2, (float)0.5, mask);81 MicroAPI::Muls(vregCdfMuls, vregInput2, (float)0.5, mask);
82 MicroAPI::Adds(vregCdfRes, vregCdfMuls, (float)0.5, mask);82 MicroAPI::Adds(vregCdfRes, vregCdfMuls, (float)0.5, mask);
83 83 
@@ -90,7 +90,7 @@ struct GeluGradV2ErfPost : public Vec::ElemwiseTernaryOP<T, T, T, T> {
90 MicroAPI::Add(vregAddRes, vregCdfRes, vregMulRes, mask);90 MicroAPI::Add(vregAddRes, vregCdfRes, vregMulRes, mask);
91 MicroAPI::Mul(vregOutput, vregAddRes, vregInput0, mask);91 MicroAPI::Mul(vregOutput, vregAddRes, vregInput0, mask);
92 // OpCopyOut92 // OpCopyOut
93- MicroAPI::DataCopy((__ubuf__ T*)(dstAddr + loopIdx * vlSize), vregOutput, mask);93+ MicroAPI::StoreAlign((__ubuf__ T*)(dstAddr + loopIdx * vlSize), vregOutput, mask);
94 }94 }
95 }95 }
96 }96 }
@@ -135,8 +135,8 @@ struct GeluGradV2TanhCustom : public Vec::ElemwiseBinaryOP<T, T, T> {
135 mask = MicroAPI::UpdateMask<T, MicroAPI::RegTraitNumOne>(count);135 mask = MicroAPI::UpdateMask<T, MicroAPI::RegTraitNumOne>(count);
136 MicroAPI::Duplicate(vregInputPX, BETAN);136 MicroAPI::Duplicate(vregInputPX, BETAN);
137 // OpCopyIn137 // OpCopyIn
138- MicroAPI::DataCopy(vregInputDy, (__ubuf__ T*)(src0Addr + loopIdx * vlSize));138+ MicroAPI::LoadAlign(vregInputDy, (__ubuf__ T*)(src0Addr + loopIdx * vlSize));
139- MicroAPI::DataCopy(vregInputX, (__ubuf__ T*)(src1Addr + loopIdx * vlSize));139+ MicroAPI::LoadAlign(vregInputX, (__ubuf__ T*)(src1Addr + loopIdx * vlSize));
140 // compute140 // compute
141 MicroAPI::Mul(vregInputXSqr, vregInputX, vregInputX, mask);141 MicroAPI::Mul(vregInputXSqr, vregInputX, vregInputX, mask);
142 MicroAPI::Axpy(vregInputPX, vregInputXSqr, AN, mask);142 MicroAPI::Axpy(vregInputPX, vregInputXSqr, AN, mask);
@@ -159,7 +159,7 @@ struct GeluGradV2TanhCustom : public Vec::ElemwiseBinaryOP<T, T, T> {
159 MicroAPI::Mul(vregOutput, vregInputDy, vregInputResp, mask);159 MicroAPI::Mul(vregOutput, vregInputDy, vregInputResp, mask);
160 160 
161 // OpCopyOut161 // OpCopyOut
162- MicroAPI::DataCopy((__ubuf__ T*)(dstAddr + loopIdx * vlSize), vregOutput, mask);162+ MicroAPI::StoreAlign((__ubuf__ T*)(dstAddr + loopIdx * vlSize), vregOutput, mask);
163 }163 }
164 }164 }
165 }165 }
@@ -219,4 +219,4 @@ struct GeluGradV2TanhDAG {
219 using OpDag = DAGSch<Outputs, void, MemCfg>;219 using OpDag = DAGSch<Outputs, void, MemCfg>;
220};220};
221} // namespace GeluGradV2Op221} // namespace GeluGradV2Op
222-#endif // CANN_CUSTOM_OPS_GELU_GRAD_V2_DAG_H222+#endif // CANN_CUSTOM_OPS_GELU_GRAD_V2_DAG_H
Mactivation/gelu_quant/op_kernel/arch35/gelu_dynamic_quant_regbase.h+9-9
@@ -304,18 +304,18 @@ __aicore__ inline void GeluDynamicQuant<T1, T2>::ComputeDynamicQuantRegbase(Loca
304 uint32_t sreg0 = endAxisActualAlignLen_;304 uint32_t sreg0 = endAxisActualAlignLen_;
305 for (uint16_t j = 0; j < loopNum; j++) {305 for (uint16_t j = 0; j < loopNum; j++) {
306 preg0 = AscendC::MicroAPI::UpdateMask<float>(sreg0);306 preg0 = AscendC::MicroAPI::UpdateMask<float>(sreg0);
307- AscendC::MicroAPI::DataCopy(vregInput, xAddr + i * endAxisActualAlignLen_ + j * vl);307+ AscendC::MicroAPI::LoadAlign(vregInput, xAddr + i * endAxisActualAlignLen_ + j * vl);
308 // compute smoothscale308 // compute smoothscale
309- AscendC::MicroAPI::DataCopy(vregSmoothScale, smoothScaleAddr + j * vl);309+ AscendC::MicroAPI::LoadAlign(vregSmoothScale, smoothScaleAddr + j * vl);
310 AscendC::MicroAPI::Mul(vregInput, vregInput, vregSmoothScale, preg0);310 AscendC::MicroAPI::Mul(vregInput, vregInput, vregSmoothScale, preg0);
311 AscendC::MicroAPI::Abs(vregAbs, vregInput, preg0);311 AscendC::MicroAPI::Abs(vregAbs, vregInput, preg0);
312 AscendC::MicroAPI::Max(vregMax, vregAbs, vregMax, preg1);312 AscendC::MicroAPI::Max(vregMax, vregAbs, vregMax, preg1);
313 }313 }
314 {314 {
315- AscendC::MicroAPI::ReduceMax(vregReduceMax, vregMax, preg1);315+ AscendC::MicroAPI::Reduce<AscendC::MicroAPI::ReduceType::MAX>(vregReduceMax, vregMax, preg1);
316 AscendC::MicroAPI::Muls(vregReduceMax, vregReduceMax, maxValue_, preg1);316 AscendC::MicroAPI::Muls(vregReduceMax, vregReduceMax, maxValue_, preg1);
317 AscendC::MicroAPI::Duplicate(vregOutScale, vregReduceMax, preg1);317 AscendC::MicroAPI::Duplicate(vregOutScale, vregReduceMax, preg1);
318- AscendC::MicroAPI::DataCopy(scaleOutAddr + i * FP32_BLOCK_NUM, vregOutScale, preg1);318+ AscendC::MicroAPI::StoreAlign(scaleOutAddr + i * FP32_BLOCK_NUM, vregOutScale, preg1);
319 }319 }
320 uint32_t sreg1 = endAxisLen_;320 uint32_t sreg1 = endAxisLen_;
321 AscendC::MicroAPI::LocalMemBar<AscendC::MicroAPI::MemType::VEC_STORE,321 AscendC::MicroAPI::LocalMemBar<AscendC::MicroAPI::MemType::VEC_STORE,
@@ -323,8 +323,8 @@ __aicore__ inline void GeluDynamicQuant<T1, T2>::ComputeDynamicQuantRegbase(Loca
323 for (uint16_t j = 0; j < loopNum; j++) {323 for (uint16_t j = 0; j < loopNum; j++) {
324 auto yOutAddr = yAddr + i * endAxisLenAlignTo8_ + j * vl;324 auto yOutAddr = yAddr + i * endAxisLenAlignTo8_ + j * vl;
325 preg2 = AscendC::MicroAPI::UpdateMask<float>(sreg1);325 preg2 = AscendC::MicroAPI::UpdateMask<float>(sreg1);
326- AscendC::MicroAPI::DataCopy(vregInput, xAddr + i * endAxisActualAlignLen_ + j * vl);326+ AscendC::MicroAPI::LoadAlign(vregInput, xAddr + i * endAxisActualAlignLen_ + j * vl);
327- AscendC::MicroAPI::DataCopy(vregSmoothScale, smoothScaleAddr + j * vl);327+ AscendC::MicroAPI::LoadAlign(vregSmoothScale, smoothScaleAddr + j * vl);
328 AscendC::MicroAPI::Mul(vregInput, vregInput, vregSmoothScale, preg2);328 AscendC::MicroAPI::Mul(vregInput, vregInput, vregSmoothScale, preg2);
329 AscendC::MicroAPI::Div(vregQuantRes, vregInput, vregOutScale, preg2);329 AscendC::MicroAPI::Div(vregQuantRes, vregInput, vregOutScale, preg2);
330 330 
@@ -341,8 +341,8 @@ __aicore__ inline void GeluDynamicQuant<T1, T2>::ComputeDynamicQuantRegbase(Loca
341 roundMode == AscendC::RoundMode::CAST_ROUND) {341 roundMode == AscendC::RoundMode::CAST_ROUND) {
342 AscendC::MicroAPI::Cast<dstType, float, castTraitF32ToH8Round>(vregY, vregQuantRes, preg2);342 AscendC::MicroAPI::Cast<dstType, float, castTraitF32ToH8Round>(vregY, vregQuantRes, preg2);
343 }343 }
344- AscendC::MicroAPI::DataCopy<dstType, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(yOutAddr, vregY,344+ AscendC::MicroAPI::StoreAlign<dstType, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(yOutAddr, vregY,
345- preg2);345+ preg2);
346 }346 }
347 }347 }
348 }348 }
@@ -350,4 +350,4 @@ __aicore__ inline void GeluDynamicQuant<T1, T2>::ComputeDynamicQuantRegbase(Loca
350 scaleOutQueue_.EnQue<float>(scaleOutLocalFp32);350 scaleOutQueue_.EnQue<float>(scaleOutLocalFp32);
351}351}
352} // namespace GeluQuantALL352} // namespace GeluQuantALL
353-#endif353+#endif
Mactivation/gelu_quant/op_kernel/arch35/gelu_dynamic_quant_workspace_regbase.h+8-8
@@ -371,26 +371,26 @@ __aicore__ inline void GeluDynamicQuantWorkspace<T1, T2>::ComputeDynamicQuantReg
371 AscendC::MicroAPI::RegTensor<float> maxTemReg;371 AscendC::MicroAPI::RegTensor<float> maxTemReg;
372 AscendC::MicroAPI::MaskReg preg0;372 AscendC::MicroAPI::MaskReg preg0;
373 AscendC::MicroAPI::MaskReg preg1 = AscendC::MicroAPI::CreateMask<float, AscendC::MicroAPI::MaskPattern::ALL>();373 AscendC::MicroAPI::MaskReg preg1 = AscendC::MicroAPI::CreateMask<float, AscendC::MicroAPI::MaskPattern::ALL>();
374- AscendC::MicroAPI::UnalignReg ureg0;374+ AscendC::MicroAPI::UnalignRegForLoad ureg0;
375 375 
376 AscendC::MicroAPI::Duplicate(vregMax, 0.0);376 AscendC::MicroAPI::Duplicate(vregMax, 0.0);
377 uint32_t sreg0 = calCount;377 uint32_t sreg0 = calCount;
378 for (uint16_t i = 0; i < loopNum; i++) {378 for (uint16_t i = 0; i < loopNum; i++) {
379 preg0 = AscendC::MicroAPI::UpdateMask<float>(sreg0);379 preg0 = AscendC::MicroAPI::UpdateMask<float>(sreg0);
380- AscendC::MicroAPI::DataCopy(geluResReg, xAddr + i * vl);380+ AscendC::MicroAPI::LoadAlign(geluResReg, xAddr + i * vl);
381 // compute smoothscale381 // compute smoothscale
382- AscendC::MicroAPI::DataCopy(scaleReg, smoothScaleAddr + i * vl);382+ AscendC::MicroAPI::LoadAlign(scaleReg, smoothScaleAddr + i * vl);
383 AscendC::MicroAPI::Mul(geluResReg, geluResReg, scaleReg, preg0);383 AscendC::MicroAPI::Mul(geluResReg, geluResReg, scaleReg, preg0);
384- AscendC::MicroAPI::DataCopy(workspaceLocalAddr + i * vl, geluResReg, preg0);384+ AscendC::MicroAPI::StoreAlign(workspaceLocalAddr + i * vl, geluResReg, preg0);
385 AscendC::MicroAPI::Abs(absReg, geluResReg, preg0);385 AscendC::MicroAPI::Abs(absReg, geluResReg, preg0);
386 AscendC::MicroAPI::Max(vregMax, absReg, vregMax, preg1);386 AscendC::MicroAPI::Max(vregMax, absReg, vregMax, preg1);
387 }387 }
388- AscendC::MicroAPI::ReduceMax(reduceMaxReg, vregMax, preg1);388+ AscendC::MicroAPI::Reduce<AscendC::MicroAPI::ReduceType::MAX>(reduceMaxReg, vregMax, preg1);
389- AscendC::MicroAPI::DataCopy(maxTemReg, maxValueAddr);389+ AscendC::MicroAPI::LoadAlign(maxTemReg, maxValueAddr);
390 AscendC::MicroAPI::Max(maxTemReg, reduceMaxReg, maxTemReg, preg1);390 AscendC::MicroAPI::Max(maxTemReg, reduceMaxReg, maxTemReg, preg1);
391- AscendC::MicroAPI::DataCopy(maxValueAddr, maxTemReg, preg1);391+ AscendC::MicroAPI::StoreAlign(maxValueAddr, maxTemReg, preg1);
392 }392 }
393}393}
394 394 
395} // namespace GeluQuantALL395} // namespace GeluQuantALL
396-#endif396+#endif
Mactivation/gelu_quant/op_kernel/arch35/gelu_quant_base_regbase.h+11-10
@@ -168,13 +168,13 @@ __aicore__ inline void GeluQuantBase::GeluV2ErfPost(LocalTensor<float>& dst, Loc
168 for (uint16_t loopIdx = 0; loopIdx < loopNum; loopIdx++) {168 for (uint16_t loopIdx = 0; loopIdx < loopNum; loopIdx++) {
169 mask = MicroAPI::UpdateMask<float, MicroAPI::RegTraitNumOne>(count);169 mask = MicroAPI::UpdateMask<float, MicroAPI::RegTraitNumOne>(count);
170 // OpCopyIn170 // OpCopyIn
171- MicroAPI::DataCopy(vregInput1, (__ubuf__ float*)(src1Addr + loopIdx * vlSize));171+ MicroAPI::LoadAlign(vregInput1, (__ubuf__ float*)(src1Addr + loopIdx * vlSize));
172- MicroAPI::DataCopy(vregInput2, (__ubuf__ float*)(src2Addr + loopIdx * vlSize));172+ MicroAPI::LoadAlign(vregInput2, (__ubuf__ float*)(src2Addr + loopIdx * vlSize));
173 MicroAPI::Adds(vregInputAdds, vregInput2, (float)1.0, mask);173 MicroAPI::Adds(vregInputAdds, vregInput2, (float)1.0, mask);
174 MicroAPI::Muls(vregInputMuls, vregInput1, (float)0.5, mask);174 MicroAPI::Muls(vregInputMuls, vregInput1, (float)0.5, mask);
175 MicroAPI::Mul(vregOutput, vregInputAdds, vregInputMuls, mask);175 MicroAPI::Mul(vregOutput, vregInputAdds, vregInputMuls, mask);
176 // OpCopyOut176 // OpCopyOut
177- MicroAPI::DataCopy((__ubuf__ float*)(dstAddr + loopIdx * vlSize), vregOutput, mask);177+ MicroAPI::StoreAlign((__ubuf__ float*)(dstAddr + loopIdx * vlSize), vregOutput, mask);
178 }178 }
179 }179 }
180#endif180#endif
@@ -203,7 +203,7 @@ __aicore__ inline void GeluQuantBase::ComputeGeluTanh(const LocalTensor<T>& src,
203 for (uint16_t loopIdx = 0; loopIdx < loopNum; loopIdx++) {203 for (uint16_t loopIdx = 0; loopIdx < loopNum; loopIdx++) {
204 mask = MicroAPI::UpdateMask<float, MicroAPI::RegTraitNumOne>(count);204 mask = MicroAPI::UpdateMask<float, MicroAPI::RegTraitNumOne>(count);
205 // OpCopyIn205 // OpCopyIn
206- MicroAPI::DataCopy(vregInput, (__ubuf__ float*)(srcAddr + loopIdx * vlSize));206+ MicroAPI::LoadAlign(vregInput, (__ubuf__ float*)(srcAddr + loopIdx * vlSize));
207 MicroAPI::Mul(vregInputSqr, vregInput, vregInput, mask);207 MicroAPI::Mul(vregInputSqr, vregInput, vregInput, mask);
208 MicroAPI::Mul(vregInputCub, vregInputSqr, vregInput, mask);208 MicroAPI::Mul(vregInputCub, vregInputSqr, vregInput, mask);
209 MicroAPI::Axpy(vregInputCub, vregInput, TANH_APPROX_FACTOR, mask);209 MicroAPI::Axpy(vregInputCub, vregInput, TANH_APPROX_FACTOR, mask);
@@ -213,7 +213,7 @@ __aicore__ inline void GeluQuantBase::ComputeGeluTanh(const LocalTensor<T>& src,
213 MicroAPI::Div(vregOutput, vregInput, vregInputCub, mask);213 MicroAPI::Div(vregOutput, vregInput, vregInputCub, mask);
214 214 
215 // OpCopyOut215 // OpCopyOut
216- MicroAPI::DataCopy((__ubuf__ float*)(dstAddr + loopIdx * vlSize), vregOutput, mask);216+ MicroAPI::StoreAlign((__ubuf__ float*)(dstAddr + loopIdx * vlSize), vregOutput, mask);
217 }217 }
218 }218 }
219 } else {219 } else {
@@ -223,8 +223,8 @@ __aicore__ inline void GeluQuantBase::ComputeGeluTanh(const LocalTensor<T>& src,
223 for (uint16_t loopIdx = 0; loopIdx < loopNum; loopIdx++) {223 for (uint16_t loopIdx = 0; loopIdx < loopNum; loopIdx++) {
224 mask = MicroAPI::UpdateMask<float, MicroAPI::RegTraitNumOne>(count);224 mask = MicroAPI::UpdateMask<float, MicroAPI::RegTraitNumOne>(count);
225 // OpCopyIn225 // OpCopyIn
226- MicroAPI::DataCopy<T, MicroAPI::LoadDist::DIST_UNPACK_B16>(vregInput16,226+ MicroAPI::LoadAlign<T, MicroAPI::LoadDist::DIST_UNPACK_B16>(vregInput16,
227- (__ubuf__ T*)(srcAddr + loopIdx * vlSize));227+ (__ubuf__ T*)(srcAddr + loopIdx * vlSize));
228 MicroAPI::Cast<float, T, castTrait0>(vregInput, vregInput16, mask);228 MicroAPI::Cast<float, T, castTrait0>(vregInput, vregInput16, mask);
229 MicroAPI::Mul(vregInputSqr, vregInput, vregInput, mask);229 MicroAPI::Mul(vregInputSqr, vregInput, vregInput, mask);
230 MicroAPI::Mul(vregInputCub, vregInputSqr, vregInput, mask);230 MicroAPI::Mul(vregInputCub, vregInputSqr, vregInput, mask);
@@ -235,7 +235,7 @@ __aicore__ inline void GeluQuantBase::ComputeGeluTanh(const LocalTensor<T>& src,
235 MicroAPI::Div(vregOutput, vregInput, vregInputCub, mask);235 MicroAPI::Div(vregOutput, vregInput, vregInputCub, mask);
236 236 
237 // OpCopyOut237 // OpCopyOut
238- MicroAPI::DataCopy((__ubuf__ float*)(dstAddr + loopIdx * vlSize), vregOutput, mask);238+ MicroAPI::StoreAlign((__ubuf__ float*)(dstAddr + loopIdx * vlSize), vregOutput, mask);
239 }239 }
240 }240 }
241 }241 }
@@ -275,7 +275,7 @@ __aicore__ inline void GeluQuantBase::CastOutLocal(LocalTensor<float>& src, Loca
275 for (uint16_t i = 0; i < loopNum; i++) {275 for (uint16_t i = 0; i < loopNum; i++) {
276 auto yOutAddr = yAddr + i * vl;276 auto yOutAddr = yAddr + i * vl;
277 preg0 = AscendC::MicroAPI::UpdateMask<float>(sreg1);277 preg0 = AscendC::MicroAPI::UpdateMask<float>(sreg1);
278- AscendC::MicroAPI::DataCopy(vregInput, xAddr + i * vl);278+ AscendC::MicroAPI::LoadAlign(vregInput, xAddr + i * vl);
279 279 
280 if constexpr (IsSameType<dstType, int8_t>::value) {280 if constexpr (IsSameType<dstType, int8_t>::value) {
281 AscendC::MicroAPI::Cast<half, float, castTraitF32ToF16>(vregHalf, vregInput, preg0);281 AscendC::MicroAPI::Cast<half, float, castTraitF32ToF16>(vregHalf, vregInput, preg0);
@@ -293,7 +293,8 @@ __aicore__ inline void GeluQuantBase::CastOutLocal(LocalTensor<float>& src, Loca
293 roundMode == AscendC::RoundMode::CAST_ROUND) {293 roundMode == AscendC::RoundMode::CAST_ROUND) {
294 AscendC::MicroAPI::Cast<dstType, float, castTraitF32ToH8Round>(vregY, vregInput, preg0);294 AscendC::MicroAPI::Cast<dstType, float, castTraitF32ToH8Round>(vregY, vregInput, preg0);
295 }295 }
296- AscendC::MicroAPI::DataCopy<dstType, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(yOutAddr, vregY, preg0);296+ AscendC::MicroAPI::StoreAlign<dstType, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(yOutAddr, vregY,
297+ preg0);
297 }298 }
298 }299 }
299}300}
Mactivation/gelu_v2/op_kernel/arch35/gelu_v2_dag.h+5-5
@@ -62,13 +62,13 @@ struct GeluV2ErfPost : public Vec::ElemwiseBinaryOP<T, T, T> {
62 for (uint16_t loopIdx = 0; loopIdx < loopNum; loopIdx++) {62 for (uint16_t loopIdx = 0; loopIdx < loopNum; loopIdx++) {
63 mask = MicroAPI::UpdateMask<T, MicroAPI::RegTraitNumOne>(count);63 mask = MicroAPI::UpdateMask<T, MicroAPI::RegTraitNumOne>(count);
64 // OpCopyIn64 // OpCopyIn
65- MicroAPI::DataCopy(vregInput1, (__ubuf__ T*)(src1Addr + loopIdx * vlSize));65+ MicroAPI::LoadAlign(vregInput1, (__ubuf__ T*)(src1Addr + loopIdx * vlSize));
66- MicroAPI::DataCopy(vregInput2, (__ubuf__ T*)(src2Addr + loopIdx * vlSize));66+ MicroAPI::LoadAlign(vregInput2, (__ubuf__ T*)(src2Addr + loopIdx * vlSize));
67 MicroAPI::Adds(vregInputAdds, vregInput2, (float)1.0, mask);67 MicroAPI::Adds(vregInputAdds, vregInput2, (float)1.0, mask);
68 MicroAPI::Muls(vregInputMuls, vregInput1, (float)0.5, mask);68 MicroAPI::Muls(vregInputMuls, vregInput1, (float)0.5, mask);
69 MicroAPI::Mul(vregOutput, vregInputAdds, vregInputMuls, mask);69 MicroAPI::Mul(vregOutput, vregInputAdds, vregInputMuls, mask);
70 // OpCopyOut70 // OpCopyOut
71- MicroAPI::DataCopy((__ubuf__ T*)(dstAddr + loopIdx * vlSize), vregOutput, mask);71+ MicroAPI::StoreAlign((__ubuf__ T*)(dstAddr + loopIdx * vlSize), vregOutput, mask);
72 }72 }
73 }73 }
74 }74 }
@@ -99,7 +99,7 @@ struct GeluV2Tanh : public Vec::ElemwiseUnaryOP<T, T> {
99 for (uint16_t loopIdx = 0; loopIdx < loopNum; loopIdx++) {99 for (uint16_t loopIdx = 0; loopIdx < loopNum; loopIdx++) {
100 mask = MicroAPI::UpdateMask<T, MicroAPI::RegTraitNumOne>(count);100 mask = MicroAPI::UpdateMask<T, MicroAPI::RegTraitNumOne>(count);
101 // OpCopyIn101 // OpCopyIn
102- MicroAPI::DataCopy(vregInput, (__ubuf__ T*)(srcAddr + loopIdx * vlSize));102+ MicroAPI::LoadAlign(vregInput, (__ubuf__ T*)(srcAddr + loopIdx * vlSize));
103 MicroAPI::Mul(vregInputSqr, vregInput, vregInput, mask);103 MicroAPI::Mul(vregInputSqr, vregInput, vregInput, mask);
104 MicroAPI::Mul(vregInputCub, vregInputSqr, vregInput, mask);104 MicroAPI::Mul(vregInputCub, vregInputSqr, vregInput, mask);
105 MicroAPI::Axpy(vregInputCub, vregInput, TANH_APPROX_FACTOR, mask);105 MicroAPI::Axpy(vregInputCub, vregInput, TANH_APPROX_FACTOR, mask);
@@ -109,7 +109,7 @@ struct GeluV2Tanh : public Vec::ElemwiseUnaryOP<T, T> {
109 MicroAPI::Div(vregOutput, vregInput, vregInputCub, mask);109 MicroAPI::Div(vregOutput, vregInput, vregInputCub, mask);
110 110 
111 // OpCopyOut111 // OpCopyOut
112- MicroAPI::DataCopy((__ubuf__ T*)(dstAddr + loopIdx * vlSize), vregOutput, mask);112+ MicroAPI::StoreAlign((__ubuf__ T*)(dstAddr + loopIdx * vlSize), vregOutput, mask);
113 }113 }
114 }114 }
115 }115 }
Mactivation/leaky_relu/op_kernel/arch35/leaky_relu_dag.h+2-2
@@ -43,13 +43,13 @@ struct LeakyReluCustom : public Vec::ElemwiseBinaryOP<T, T, float> {
43 MicroAPI::Duplicate(vregZero, (T)0.0);43 MicroAPI::Duplicate(vregZero, (T)0.0);
44 for (uint16_t loopIdx = 0; loopIdx < loopNum; loopIdx++) {44 for (uint16_t loopIdx = 0; loopIdx < loopNum; loopIdx++) {
45 mask = MicroAPI::UpdateMask<T, MicroAPI::RegTraitNumOne>(count);45 mask = MicroAPI::UpdateMask<T, MicroAPI::RegTraitNumOne>(count);
46- MicroAPI::DataCopy(vregInput, (__ubuf__ T*)(srcAddr + loopIdx * vlSize));46+ MicroAPI::LoadAlign(vregInput, (__ubuf__ T*)(srcAddr + loopIdx * vlSize));
47 47 
48 MicroAPI::Muls(vregNegPart, vregInput, negativeSlope, mask);48 MicroAPI::Muls(vregNegPart, vregInput, negativeSlope, mask);
49 MicroAPI::Compare<T, CMPMODE::GT>(cmpMask, vregInput, vregZero, mask);49 MicroAPI::Compare<T, CMPMODE::GT>(cmpMask, vregInput, vregZero, mask);
50 MicroAPI::Select<T>(vregOutput, vregInput, vregNegPart, cmpMask);50 MicroAPI::Select<T>(vregOutput, vregInput, vregNegPart, cmpMask);
51 51 
52- MicroAPI::DataCopy((__ubuf__ T*)(dstAddr + loopIdx * vlSize), vregOutput, mask);52+ MicroAPI::StoreAlign((__ubuf__ T*)(dstAddr + loopIdx * vlSize), vregOutput, mask);
53 }53 }
54 }54 }
55#endif55#endif
Mactivation/log_sigmoid/op_kernel/arch35/log_sigmoid_dag.h+4-4
@@ -66,7 +66,7 @@ struct LogSigmoidCustom : public Vec::ElemwiseUnaryOP<T, T> {
66 for (uint16_t loopIdx = 0; loopIdx < loopNum; loopIdx++) {66 for (uint16_t loopIdx = 0; loopIdx < loopNum; loopIdx++) {
67 // regCopyIn67 // regCopyIn
68 mask = MicroAPI::UpdateMask<T, MicroAPI::RegTraitNumOne>(count);68 mask = MicroAPI::UpdateMask<T, MicroAPI::RegTraitNumOne>(count);
69- MicroAPI::DataCopy(x, (__ubuf__ T*)(srcAddr + loopIdx * vlSize));69+ MicroAPI::LoadAlign(x, (__ubuf__ T*)(srcAddr + loopIdx * vlSize));
70 MicroAPI::Min(minRes, x, zeroReg, mask); // x1 = min(x, 0)70 MicroAPI::Min(minRes, x, zeroReg, mask); // x1 = min(x, 0)
71 MicroAPI::Abs(xAbs, x, mask); // x2 = abs(x)71 MicroAPI::Abs(xAbs, x, mask); // x2 = abs(x)
72 MicroAPI::Muls(xAbsNeg, xAbs, valNegOne, mask); // x3 = -x272 MicroAPI::Muls(xAbsNeg, xAbs, valNegOne, mask); // x3 = -x2
@@ -77,12 +77,12 @@ struct LogSigmoidCustom : public Vec::ElemwiseUnaryOP<T, T> {
77 MicroAPI::Div(divRes, expRes, expResPlusOneSubOne, mask); // y3 = x4 / y277 MicroAPI::Div(divRes, expRes, expResPlusOneSubOne, mask); // y3 = x4 / y2
78 MicroAPI::Log(logExpXPlus1, expResPlusOne, mask); // y4 = log(y1)78 MicroAPI::Log(logExpXPlus1, expResPlusOne, mask); // y4 = log(y1)
79 MicroAPI::Mul(mulRes, logExpXPlus1, divRes, mask); // y5 = y4 * y379 MicroAPI::Mul(mulRes, logExpXPlus1, divRes, mask); // y5 = y4 * y3
80- MicroAPI::CompareScalar<T, CMPMODE::NE>(cmpLog1pPosMaskReg, expResPlusOne, valPosOne, mask);80+ MicroAPI::Compares<T, CMPMODE::NE>(cmpLog1pPosMaskReg, expResPlusOne, valPosOne, mask);
81 MicroAPI::Select(selectRes, mulRes, expRes, cmpLog1pPosMaskReg); // z1 = select(x4, y5)81 MicroAPI::Select(selectRes, mulRes, expRes, cmpLog1pPosMaskReg); // z1 = select(x4, y5)
82 MicroAPI::Sub(ans, minRes, selectRes, mask); // z2 = x1 - z182 MicroAPI::Sub(ans, minRes, selectRes, mask); // z2 = x1 - z1
83 83 
84 // regCopyOut84 // regCopyOut
85- MicroAPI::DataCopy((__ubuf__ T*)(dstAddr + loopIdx * vlSize), ans, mask);85+ MicroAPI::StoreAlign((__ubuf__ T*)(dstAddr + loopIdx * vlSize), ans, mask);
86 }86 }
87 }87 }
88#endif88#endif
@@ -115,5 +115,5 @@ struct LogSigmoidNeedCast {
115 using OpDag = DAGSch<Outputs>;115 using OpDag = DAGSch<Outputs>;
116};116};
117 117 
118-}; // namespace LogSigmoidDag118+}; // namespace LogSigmoidDag
119#endif // CANN_CUSTOM_OPS_LOG_SIGMOID_DAG_H119#endif // CANN_CUSTOM_OPS_LOG_SIGMOID_DAG_H
Mactivation/log_sigmoid/tests/ut/op_kernel/test_log_sigmoid_apt.cpp+2-2
@@ -55,7 +55,7 @@ TEST_F(log_sigmoid_test, test_case_fp32_1)
55 55 
56 tilingDatafromBin->dim0 = 256;56 tilingDatafromBin->dim0 = 256;
57 tilingDatafromBin->coreNum = 1;57 tilingDatafromBin->coreNum = 1;
58- tilingDatafromBin->ubFormer = 256;58+ tilingDatafromBin->ubFormer = 1024;
59 tilingDatafromBin->blockFormer = 256;59 tilingDatafromBin->blockFormer = 256;
60 tilingDatafromBin->blockNum = 1;60 tilingDatafromBin->blockNum = 1;
61 tilingDatafromBin->ubLoopOfFormerBlock = 1;61 tilingDatafromBin->ubLoopOfFormerBlock = 1;
@@ -77,4 +77,4 @@ TEST_F(log_sigmoid_test, test_case_fp32_1)
77 AscendC::GmFree(y);77 AscendC::GmFree(y);
78 AscendC::GmFree(workspace);78 AscendC::GmFree(workspace);
79 AscendC::GmFree(tiling);79 AscendC::GmFree(tiling);
80-}80+}
Mactivation/mish/op_kernel/arch35/mish_dag.h+9-9
@@ -71,10 +71,10 @@ struct MishCustom : public Vec::ElemwiseUnaryOP<T, T> {
71 mask = MicroAPI::UpdateMask<float, MicroAPI::RegTraitNumOne>(count);71 mask = MicroAPI::UpdateMask<float, MicroAPI::RegTraitNumOne>(count);
72 // OpCopyIn72 // OpCopyIn
73 if constexpr (std::is_same_v<T, float>) {73 if constexpr (std::is_same_v<T, float>) {
74- MicroAPI::DataCopy<T, MicroAPI::LoadDist::DIST_NORM>(vregInput,74+ MicroAPI::LoadAlign<T, MicroAPI::LoadDist::DIST_NORM>(vregInput,
75- (__ubuf__ T*)(srcAddr + loopIdx * vlSize));75+ (__ubuf__ T*)(srcAddr + loopIdx * vlSize));
76 } else {76 } else {
77- MicroAPI::DataCopy<T, MicroAPI::LoadDist::DIST_UNPACK_B16>(77+ MicroAPI::LoadAlign<T, MicroAPI::LoadDist::DIST_UNPACK_B16>(
78 vregInput16, (__ubuf__ T*)(srcAddr + loopIdx * vlSize));78 vregInput16, (__ubuf__ T*)(srcAddr + loopIdx * vlSize));
79 MicroAPI::Cast<float, T, castTrait0>(vregInput, vregInput16, mask);79 MicroAPI::Cast<float, T, castTrait0>(vregInput, vregInput16, mask);
80 }80 }
@@ -95,18 +95,18 @@ struct MishCustom : public Vec::ElemwiseUnaryOP<T, T> {
95 MicroAPI::Adds(vregInputDenominator, vregInputNumerator, FP32_TWO, mask); // e^2x + 2e^x + 295 MicroAPI::Adds(vregInputDenominator, vregInputNumerator, FP32_TWO, mask); // e^2x + 2e^x + 2
96 MicroAPI::Div(vregInputNumerator, vregInputNumerator, vregInputDenominator, mask);96 MicroAPI::Div(vregInputNumerator, vregInputNumerator, vregInputDenominator, mask);
97 97 
98- MicroAPI::CompareScalar<float, CMPMODE::LT>(cmpMaskReg, vregInput, FP32_ZERO, mask);98+ MicroAPI::Compares<float, CMPMODE::LT>(cmpMaskReg, vregInput, FP32_ZERO, mask);
99 MicroAPI::Select(vregOutput, vregInputNumerator, vregOutput, cmpMaskReg);99 MicroAPI::Select(vregOutput, vregInputNumerator, vregOutput, cmpMaskReg);
100 MicroAPI::Mul(vregOutput, vregOutput, vregInput, mask);100 MicroAPI::Mul(vregOutput, vregOutput, vregInput, mask);
101 101 
102 // OpCopyOut102 // OpCopyOut
103 if constexpr (std::is_same_v<T, float>) {103 if constexpr (std::is_same_v<T, float>) {
104- MicroAPI::DataCopy<T, MicroAPI::StoreDist::DIST_NORM_B32>((__ubuf__ T*)(dstAddr + loopIdx * vlSize),104+ MicroAPI::StoreAlign<T, MicroAPI::StoreDist::DIST_NORM_B32>(
105- vregOutput, mask);105+ (__ubuf__ T*)(dstAddr + loopIdx * vlSize), vregOutput, mask);
106 } else {106 } else {
107 MicroAPI::Cast<T, float, castTrait1>(vregOutput16, vregOutput, mask);107 MicroAPI::Cast<T, float, castTrait1>(vregOutput16, vregOutput, mask);
108- MicroAPI::DataCopy<T, MicroAPI::StoreDist::DIST_PACK_B32>((__ubuf__ T*)(dstAddr + loopIdx * vlSize),108+ MicroAPI::StoreAlign<T, MicroAPI::StoreDist::DIST_PACK_B32>(
109- vregOutput16, mask);109+ (__ubuf__ T*)(dstAddr + loopIdx * vlSize), vregOutput16, mask);
110 }110 }
111 }111 }
112 }112 }
@@ -127,4 +127,4 @@ struct MishDAG {
127};127};
128 128 
129} // namespace MishDag1129} // namespace MishDag1
130-#endif // OPS_NN_ACTIVATION_MISH_KERNEL_DAG_H130+#endif // OPS_NN_ACTIVATION_MISH_KERNEL_DAG_H
Mactivation/mish_grad/op_kernel/arch35/mish_grad_dag.h+12-12
@@ -55,10 +55,10 @@ struct MishGradCustom : public Vec::ElemwiseBinaryOP<T, T, T> {
55 for (uint16_t loopIdx = 0; loopIdx < loopNum; loopIdx++) {55 for (uint16_t loopIdx = 0; loopIdx < loopNum; loopIdx++) {
56 mask = MicroAPI::UpdateMask<float, MicroAPI::RegTraitNumOne>(count);56 mask = MicroAPI::UpdateMask<float, MicroAPI::RegTraitNumOne>(count);
57 // OpCopyIn57 // OpCopyIn
58- MicroAPI::DataCopy<T, MicroAPI::LoadDist::DIST_NORM>(vregInput,58+ MicroAPI::LoadAlign<T, MicroAPI::LoadDist::DIST_NORM>(vregInput,
59- (__ubuf__ T*)(srcAddr + loopIdx * vlSize));59+ (__ubuf__ T*)(srcAddr + loopIdx * vlSize));
60- MicroAPI::DataCopy<T, MicroAPI::LoadDist::DIST_NORM>(vregInput2,60+ MicroAPI::LoadAlign<T, MicroAPI::LoadDist::DIST_NORM>(vregInput2,
61- (__ubuf__ T*)(src2Addr + loopIdx * vlSize));61+ (__ubuf__ T*)(src2Addr + loopIdx * vlSize));
62 62 
63 MicroAPI::Mul(vregInputSqr, vregInput2, vregInput2, mask); // vregInput2 = tanh(z)63 MicroAPI::Mul(vregInputSqr, vregInput2, vregInput2, mask); // vregInput2 = tanh(z)
64 MicroAPI::Adds(vregInputSqr, vregInputSqr, FP32_NEG_ONE, mask);64 MicroAPI::Adds(vregInputSqr, vregInputSqr, FP32_NEG_ONE, mask);
@@ -71,8 +71,8 @@ struct MishGradCustom : public Vec::ElemwiseBinaryOP<T, T, T> {
71 MicroAPI::Div(vregInputExp2, vregInputSqr, vregInputExp2, mask);71 MicroAPI::Div(vregInputExp2, vregInputSqr, vregInputExp2, mask);
72 MicroAPI::Sub(vregOutput, vregInput2, vregInputExp2, mask);72 MicroAPI::Sub(vregOutput, vregInput2, vregInputExp2, mask);
73 // OpCopyOut73 // OpCopyOut
74- MicroAPI::DataCopy<T, MicroAPI::StoreDist::DIST_NORM_B32>((__ubuf__ T*)(dstAddr + loopIdx * vlSize),74+ MicroAPI::StoreAlign<T, MicroAPI::StoreDist::DIST_NORM_B32>(
75- vregOutput, mask);75+ (__ubuf__ T*)(dstAddr + loopIdx * vlSize), vregOutput, mask);
76 }76 }
77 }77 }
78 }78 }
@@ -109,8 +109,8 @@ struct MishCustom : public Vec::ElemwiseUnaryOP<T, T> {
109 for (uint16_t loopIdx = 0; loopIdx < loopNum; loopIdx++) {109 for (uint16_t loopIdx = 0; loopIdx < loopNum; loopIdx++) {
110 mask = MicroAPI::UpdateMask<float, MicroAPI::RegTraitNumOne>(count);110 mask = MicroAPI::UpdateMask<float, MicroAPI::RegTraitNumOne>(count);
111 // OpCopyIn111 // OpCopyIn
112- MicroAPI::DataCopy<T, MicroAPI::LoadDist::DIST_NORM>(vregInput,112+ MicroAPI::LoadAlign<T, MicroAPI::LoadDist::DIST_NORM>(vregInput,
113- (__ubuf__ T*)(srcAddr + loopIdx * vlSize));113+ (__ubuf__ T*)(srcAddr + loopIdx * vlSize));
114 114 
115 MicroAPI::Muls(vregInputNeg, vregInput, FP32_NEG_ONE, mask);115 MicroAPI::Muls(vregInputNeg, vregInput, FP32_NEG_ONE, mask);
116 MicroAPI::Muls(vregInputNeg2, vregInput, FP32_NEG_TWO, mask);116 MicroAPI::Muls(vregInputNeg2, vregInput, FP32_NEG_TWO, mask);
@@ -130,11 +130,11 @@ struct MishCustom : public Vec::ElemwiseUnaryOP<T, T> {
130 MicroAPI::Adds(vregInputExp, vregInputMid, FP32_TWO, mask);130 MicroAPI::Adds(vregInputExp, vregInputMid, FP32_TWO, mask);
131 MicroAPI::Div(vregInputMid, vregInputMid, vregInputExp, mask);131 MicroAPI::Div(vregInputMid, vregInputMid, vregInputExp, mask);
132 132 
133- MicroAPI::CompareScalar<float, CMPMODE::LT>(cmpMaskReg, vregInput, FP32_ZERO, mask);133+ MicroAPI::Compares<float, CMPMODE::LT>(cmpMaskReg, vregInput, FP32_ZERO, mask);
134 MicroAPI::Select(vregOutput, vregInputMid, vregOutput, cmpMaskReg);134 MicroAPI::Select(vregOutput, vregInputMid, vregOutput, cmpMaskReg);
135 // OpCopyOut135 // OpCopyOut
136- MicroAPI::DataCopy<T, MicroAPI::StoreDist::DIST_NORM_B32>((__ubuf__ T*)(dstAddr + loopIdx * vlSize),136+ MicroAPI::StoreAlign<T, MicroAPI::StoreDist::DIST_NORM_B32>((__ubuf__ T*)(dstAddr + loopIdx * vlSize),
137- vregOutput, mask);137+ vregOutput, mask);
138 }138 }
139 }139 }
140#endif140#endif
@@ -181,4 +181,4 @@ struct MishGradDAG {
181 using OpDag = DAGSch<Outputs, void, MemCfg>;181 using OpDag = DAGSch<Outputs, void, MemCfg>;
182};182};
183} // namespace MishGradOp183} // namespace MishGradOp
184-#endif // CANN_CUSTOM_OPS_MISHGRAD_DAG_H184+#endif // CANN_CUSTOM_OPS_MISHGRAD_DAG_H
Mactivation/p_relu/op_kernel/arch35/p_relu_dag.h+4-4
@@ -51,8 +51,8 @@ struct PreluCustom : public Vec::ElemwiseBinaryOP<T, T, T> {
51 for (uint16_t loopIdx = 0; loopIdx < loopNum; loopIdx++) {51 for (uint16_t loopIdx = 0; loopIdx < loopNum; loopIdx++) {
52 mask = MicroAPI::UpdateMask<T, MicroAPI::RegTraitNumOne>(count);52 mask = MicroAPI::UpdateMask<T, MicroAPI::RegTraitNumOne>(count);
53 // OpCopyIn53 // OpCopyIn
54- MicroAPI::DataCopy(vregInputX, (__ubuf__ T*)(src0Addr + loopIdx * vlSize));54+ MicroAPI::LoadAlign(vregInputX, (__ubuf__ T*)(src0Addr + loopIdx * vlSize));
55- MicroAPI::DataCopy(vregInputWeight, (__ubuf__ T*)(src1Addr + loopIdx * vlSize));55+ MicroAPI::LoadAlign(vregInputWeight, (__ubuf__ T*)(src1Addr + loopIdx * vlSize));
56 56 
57 // compute57 // compute
58 MicroAPI::Mul(vregInputProd, vregInputX, vregInputWeight, mask);58 MicroAPI::Mul(vregInputProd, vregInputX, vregInputWeight, mask);
@@ -60,7 +60,7 @@ struct PreluCustom : public Vec::ElemwiseBinaryOP<T, T, T> {
60 MicroAPI::Select<T>(vregOutput, vregInputX, vregInputProd, cmpMask);60 MicroAPI::Select<T>(vregOutput, vregInputX, vregInputProd, cmpMask);
61 61 
62 // OpCopyOut62 // OpCopyOut
63- MicroAPI::DataCopy((__ubuf__ T*)(dstAddr + loopIdx * vlSize), vregOutput, mask);63+ MicroAPI::StoreAlign((__ubuf__ T*)(dstAddr + loopIdx * vlSize), vregOutput, mask);
64 }64 }
65 }65 }
66#endif66#endif
@@ -80,4 +80,4 @@ struct PreluDAG {
80};80};
81} // namespace PreluOp81} // namespace PreluOp
82 82 
83-#endif // OP_NN_ACTIVATION_P_RELU_OP_KERNEL_PRELU_DAG_H83+#endif // OP_NN_ACTIVATION_P_RELU_OP_KERNEL_PRELU_DAG_H
Mactivation/sigmoid/op_kernel/arch35/sigmoid_bf16.h+4-4
@@ -93,11 +93,11 @@ private:
93 vreg0, static_cast<float>(1), preg0);93 vreg0, static_cast<float>(1), preg0);
94 uint16_t vfLoopNum = (i0Extent + (AscendC::VECTOR_REG_WIDTH / sizeof(float)) - 1) /94 uint16_t vfLoopNum = (i0Extent + (AscendC::VECTOR_REG_WIDTH / sizeof(float)) - 1) /
95 (AscendC::VECTOR_REG_WIDTH / sizeof(float));95 (AscendC::VECTOR_REG_WIDTH / sizeof(float));
96- __local_mem__ bfloat16_t* bufferIn0Addr = (__local_mem__ bfloat16_t*)bufferIn0_.GetPhyAddr();96+ __ubuf__ bfloat16_t* bufferIn0Addr = (__ubuf__ bfloat16_t*)bufferIn0_.GetPhyAddr();
97- __local_mem__ bfloat16_t* bufferOut0Addr = (__local_mem__ bfloat16_t*)bufferOut0_.GetPhyAddr();97+ __ubuf__ bfloat16_t* bufferOut0Addr = (__ubuf__ bfloat16_t*)bufferOut0_.GetPhyAddr();
98 for (uint16_t i = 0; i < vfLoopNum; i++) {98 for (uint16_t i = 0; i < vfLoopNum; i++) {
99 preg0 = AscendC::MicroAPI::UpdateMask<float>(size);99 preg0 = AscendC::MicroAPI::UpdateMask<float>(size);
100- AscendC::MicroAPI::DataCopy<bfloat16_t, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(100+ AscendC::MicroAPI::LoadAlign<bfloat16_t, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(
101 vreg1, bufferIn0Addr + i * (AscendC::VECTOR_REG_WIDTH / sizeof(float)));101 vreg1, bufferIn0Addr + i * (AscendC::VECTOR_REG_WIDTH / sizeof(float)));
102 AscendC::MicroAPI::Cast<float, bfloat16_t, castTrait0>(vreg2, vreg1, preg0);102 AscendC::MicroAPI::Cast<float, bfloat16_t, castTrait0>(vreg2, vreg1, preg0);
103 AscendC::MicroAPI::Muls<float, float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(103 AscendC::MicroAPI::Muls<float, float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(
@@ -107,7 +107,7 @@ private:
107 vreg5, vreg4, static_cast<float>(1), preg0);107 vreg5, vreg4, static_cast<float>(1), preg0);
108 AscendC::MicroAPI::Div<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg6, vreg0, vreg5, preg0);108 AscendC::MicroAPI::Div<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg6, vreg0, vreg5, preg0);
109 AscendC::MicroAPI::Cast<bfloat16_t, float, castTrait1>(vreg7, vreg6, preg0);109 AscendC::MicroAPI::Cast<bfloat16_t, float, castTrait1>(vreg7, vreg6, preg0);
110- AscendC::MicroAPI::DataCopy<bfloat16_t, AscendC::MicroAPI::StoreDist::DIST_PACK_B32>(110+ AscendC::MicroAPI::StoreAlign<bfloat16_t, AscendC::MicroAPI::StoreDist::DIST_PACK_B32>(
111 bufferOut0Addr + i * (AscendC::VECTOR_REG_WIDTH / sizeof(float)), vreg7, preg0);111 bufferOut0Addr + i * (AscendC::VECTOR_REG_WIDTH / sizeof(float)), vreg7, preg0);
112 }112 }
113 }113 }
Mactivation/sigmoid/op_kernel/arch35/sigmoid_f16.h+4-4
@@ -93,11 +93,11 @@ private:
93 vreg0, static_cast<float>(1), preg0);93 vreg0, static_cast<float>(1), preg0);
94 uint16_t vfLoopNum = (i0Extent + (AscendC::VECTOR_REG_WIDTH / sizeof(float)) - 1) /94 uint16_t vfLoopNum = (i0Extent + (AscendC::VECTOR_REG_WIDTH / sizeof(float)) - 1) /
95 (AscendC::VECTOR_REG_WIDTH / sizeof(float));95 (AscendC::VECTOR_REG_WIDTH / sizeof(float));
96- __local_mem__ half* bufferIn0Addr = (__local_mem__ half*)bufferIn0_.GetPhyAddr();96+ __ubuf__ half* bufferIn0Addr = (__ubuf__ half*)bufferIn0_.GetPhyAddr();
97- __local_mem__ half* bufferOut0Addr = (__local_mem__ half*)bufferOut0_.GetPhyAddr();97+ __ubuf__ half* bufferOut0Addr = (__ubuf__ half*)bufferOut0_.GetPhyAddr();
98 for (uint16_t i = 0; i < vfLoopNum; i++) {98 for (uint16_t i = 0; i < vfLoopNum; i++) {
99 preg0 = AscendC::MicroAPI::UpdateMask<float>(size);99 preg0 = AscendC::MicroAPI::UpdateMask<float>(size);
100- AscendC::MicroAPI::DataCopy<half, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(100+ AscendC::MicroAPI::LoadAlign<half, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(
101 vreg1, bufferIn0Addr + i * (AscendC::VECTOR_REG_WIDTH / sizeof(float)));101 vreg1, bufferIn0Addr + i * (AscendC::VECTOR_REG_WIDTH / sizeof(float)));
102 AscendC::MicroAPI::Cast<float, half, castTrait0>(vreg2, vreg1, preg0);102 AscendC::MicroAPI::Cast<float, half, castTrait0>(vreg2, vreg1, preg0);
103 AscendC::MicroAPI::Muls<float, float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(103 AscendC::MicroAPI::Muls<float, float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(
@@ -107,7 +107,7 @@ private:
107 vreg5, vreg4, static_cast<float>(1), preg0);107 vreg5, vreg4, static_cast<float>(1), preg0);
108 AscendC::MicroAPI::Div<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg6, vreg0, vreg5, preg0);108 AscendC::MicroAPI::Div<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg6, vreg0, vreg5, preg0);
109 AscendC::MicroAPI::Cast<half, float, castTrait1>(vreg7, vreg6, preg0);109 AscendC::MicroAPI::Cast<half, float, castTrait1>(vreg7, vreg6, preg0);
110- AscendC::MicroAPI::DataCopy<half, AscendC::MicroAPI::StoreDist::DIST_PACK_B32>(110+ AscendC::MicroAPI::StoreAlign<half, AscendC::MicroAPI::StoreDist::DIST_PACK_B32>(
111 bufferOut0Addr + i * (AscendC::VECTOR_REG_WIDTH / sizeof(float)), vreg7, preg0);111 bufferOut0Addr + i * (AscendC::VECTOR_REG_WIDTH / sizeof(float)), vreg7, preg0);
112 }112 }
113 }113 }
Mactivation/sigmoid/op_kernel/arch35/sigmoid_f32.h+4-4
@@ -91,11 +91,11 @@ private:
91 vreg0, static_cast<float>(1), preg0);91 vreg0, static_cast<float>(1), preg0);
92 uint16_t vfLoopNum = (i0Extent + (AscendC::VECTOR_REG_WIDTH / sizeof(float)) - 1) /92 uint16_t vfLoopNum = (i0Extent + (AscendC::VECTOR_REG_WIDTH / sizeof(float)) - 1) /
93 (AscendC::VECTOR_REG_WIDTH / sizeof(float));93 (AscendC::VECTOR_REG_WIDTH / sizeof(float));
94- __local_mem__ float* bufferIn0Addr = (__local_mem__ float*)bufferIn0_.GetPhyAddr();94+ __ubuf__ float* bufferIn0Addr = (__ubuf__ float*)bufferIn0_.GetPhyAddr();
95- __local_mem__ float* bufferOut0Addr = (__local_mem__ float*)bufferOut0_.GetPhyAddr();95+ __ubuf__ float* bufferOut0Addr = (__ubuf__ float*)bufferOut0_.GetPhyAddr();
96 for (uint16_t i = 0; i < vfLoopNum; i++) {96 for (uint16_t i = 0; i < vfLoopNum; i++) {
97 preg0 = AscendC::MicroAPI::UpdateMask<float>(size);97 preg0 = AscendC::MicroAPI::UpdateMask<float>(size);
98- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(98+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(
99 vreg1, bufferIn0Addr + i * (AscendC::VECTOR_REG_WIDTH / sizeof(float)));99 vreg1, bufferIn0Addr + i * (AscendC::VECTOR_REG_WIDTH / sizeof(float)));
100 AscendC::MicroAPI::Muls<float, float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(100 AscendC::MicroAPI::Muls<float, float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(
101 vreg2, vreg1, static_cast<float>(-1), preg0);101 vreg2, vreg1, static_cast<float>(-1), preg0);
@@ -103,7 +103,7 @@ private:
103 AscendC::MicroAPI::Adds<float, float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(103 AscendC::MicroAPI::Adds<float, float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(
104 vreg4, vreg3, static_cast<float>(1), preg0);104 vreg4, vreg3, static_cast<float>(1), preg0);
105 AscendC::MicroAPI::Div<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg5, vreg0, vreg4, preg0);105 AscendC::MicroAPI::Div<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg5, vreg0, vreg4, preg0);
106- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::StoreDist::DIST_NORM_B32>(106+ AscendC::MicroAPI::StoreAlign<float, AscendC::MicroAPI::StoreDist::DIST_NORM_B32>(
107 bufferOut0Addr + i * (AscendC::VECTOR_REG_WIDTH / sizeof(float)), vreg5, preg0);107 bufferOut0Addr + i * (AscendC::VECTOR_REG_WIDTH / sizeof(float)), vreg5, preg0);
108 }108 }
109 }109 }
Mactivation/sigmoid_grad/op_kernel/arch35/sigmoid_grad_bf16.h+6-6
@@ -112,22 +112,22 @@ private:
112 vreg2, static_cast<float>(1.0), preg0);112 vreg2, static_cast<float>(1.0), preg0);
113 uint16_t vfLoopNum = (i0Extent + (AscendC::VECTOR_REG_WIDTH / sizeof(float)) - 1) /113 uint16_t vfLoopNum = (i0Extent + (AscendC::VECTOR_REG_WIDTH / sizeof(float)) - 1) /
114 (AscendC::VECTOR_REG_WIDTH / sizeof(float));114 (AscendC::VECTOR_REG_WIDTH / sizeof(float));
115- __local_mem__ bfloat16_t* bufferIn0Addr = (__local_mem__ bfloat16_t*)bufferIn0_.GetPhyAddr();115+ __ubuf__ bfloat16_t* bufferIn0Addr = (__ubuf__ bfloat16_t*)bufferIn0_.GetPhyAddr();
116- __local_mem__ bfloat16_t* bufferIn1Addr = (__local_mem__ bfloat16_t*)bufferIn1_.GetPhyAddr();116+ __ubuf__ bfloat16_t* bufferIn1Addr = (__ubuf__ bfloat16_t*)bufferIn1_.GetPhyAddr();
117- __local_mem__ bfloat16_t* bufferOut0Addr = (__local_mem__ bfloat16_t*)bufferOut0_.GetPhyAddr();117+ __ubuf__ bfloat16_t* bufferOut0Addr = (__ubuf__ bfloat16_t*)bufferOut0_.GetPhyAddr();
118 for (uint16_t i = 0; i < vfLoopNum; i++) {118 for (uint16_t i = 0; i < vfLoopNum; i++) {
119 preg0 = AscendC::MicroAPI::UpdateMask<float>(size);119 preg0 = AscendC::MicroAPI::UpdateMask<float>(size);
120- AscendC::MicroAPI::DataCopy<bfloat16_t, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(120+ AscendC::MicroAPI::LoadAlign<bfloat16_t, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(
121 vreg5, bufferIn1Addr + i * (AscendC::VECTOR_REG_WIDTH / sizeof(float)));121 vreg5, bufferIn1Addr + i * (AscendC::VECTOR_REG_WIDTH / sizeof(float)));
122 AscendC::MicroAPI::Cast<float, bfloat16_t, castTrait0>(vreg6, vreg5, preg0);122 AscendC::MicroAPI::Cast<float, bfloat16_t, castTrait0>(vreg6, vreg5, preg0);
123- AscendC::MicroAPI::DataCopy<bfloat16_t, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(123+ AscendC::MicroAPI::LoadAlign<bfloat16_t, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(
124 vreg0, bufferIn0Addr + i * (AscendC::VECTOR_REG_WIDTH / sizeof(float)));124 vreg0, bufferIn0Addr + i * (AscendC::VECTOR_REG_WIDTH / sizeof(float)));
125 AscendC::MicroAPI::Cast<float, bfloat16_t, castTrait0>(vreg1, vreg0, preg0);125 AscendC::MicroAPI::Cast<float, bfloat16_t, castTrait0>(vreg1, vreg0, preg0);
126 AscendC::MicroAPI::Sub<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg3, vreg2, vreg1, preg0);126 AscendC::MicroAPI::Sub<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg3, vreg2, vreg1, preg0);
127 AscendC::MicroAPI::Mul<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg4, vreg6, vreg3, preg0);127 AscendC::MicroAPI::Mul<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg4, vreg6, vreg3, preg0);
128 AscendC::MicroAPI::Mul<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg7, vreg4, vreg1, preg0);128 AscendC::MicroAPI::Mul<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg7, vreg4, vreg1, preg0);
129 AscendC::MicroAPI::Cast<bfloat16_t, float, castTrait1>(vreg8, vreg7, preg0);129 AscendC::MicroAPI::Cast<bfloat16_t, float, castTrait1>(vreg8, vreg7, preg0);
130- AscendC::MicroAPI::DataCopy<bfloat16_t, AscendC::MicroAPI::StoreDist::DIST_PACK_B32>(130+ AscendC::MicroAPI::StoreAlign<bfloat16_t, AscendC::MicroAPI::StoreDist::DIST_PACK_B32>(
131 bufferOut0Addr + i * (AscendC::VECTOR_REG_WIDTH / sizeof(float)), vreg8, preg0);131 bufferOut0Addr + i * (AscendC::VECTOR_REG_WIDTH / sizeof(float)), vreg8, preg0);
132 }132 }
133 }133 }
Mactivation/sigmoid_grad/op_kernel/arch35/sigmoid_grad_f16.h+6-6
@@ -112,22 +112,22 @@ private:
112 vreg2, static_cast<float>(1.0), preg0);112 vreg2, static_cast<float>(1.0), preg0);
113 uint16_t vfLoopNum = (i0Extent + (AscendC::VECTOR_REG_WIDTH / sizeof(float)) - 1) /113 uint16_t vfLoopNum = (i0Extent + (AscendC::VECTOR_REG_WIDTH / sizeof(float)) - 1) /
114 (AscendC::VECTOR_REG_WIDTH / sizeof(float));114 (AscendC::VECTOR_REG_WIDTH / sizeof(float));
115- __local_mem__ half* bufferIn0Addr = (__local_mem__ half*)bufferIn0_.GetPhyAddr();115+ __ubuf__ half* bufferIn0Addr = (__ubuf__ half*)bufferIn0_.GetPhyAddr();
116- __local_mem__ half* bufferIn1Addr = (__local_mem__ half*)bufferIn1_.GetPhyAddr();116+ __ubuf__ half* bufferIn1Addr = (__ubuf__ half*)bufferIn1_.GetPhyAddr();
117- __local_mem__ half* bufferOut0Addr = (__local_mem__ half*)bufferOut0_.GetPhyAddr();117+ __ubuf__ half* bufferOut0Addr = (__ubuf__ half*)bufferOut0_.GetPhyAddr();
118 for (uint16_t i = 0; i < vfLoopNum; i++) {118 for (uint16_t i = 0; i < vfLoopNum; i++) {
119 preg0 = AscendC::MicroAPI::UpdateMask<float>(size);119 preg0 = AscendC::MicroAPI::UpdateMask<float>(size);
120- AscendC::MicroAPI::DataCopy<half, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(120+ AscendC::MicroAPI::LoadAlign<half, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(
121 vreg5, bufferIn1Addr + i * (AscendC::VECTOR_REG_WIDTH / sizeof(float)));121 vreg5, bufferIn1Addr + i * (AscendC::VECTOR_REG_WIDTH / sizeof(float)));
122 AscendC::MicroAPI::Cast<float, half, castTrait0>(vreg6, vreg5, preg0);122 AscendC::MicroAPI::Cast<float, half, castTrait0>(vreg6, vreg5, preg0);
123- AscendC::MicroAPI::DataCopy<half, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(123+ AscendC::MicroAPI::LoadAlign<half, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(
124 vreg0, bufferIn0Addr + i * (AscendC::VECTOR_REG_WIDTH / sizeof(float)));124 vreg0, bufferIn0Addr + i * (AscendC::VECTOR_REG_WIDTH / sizeof(float)));
125 AscendC::MicroAPI::Cast<float, half, castTrait0>(vreg1, vreg0, preg0);125 AscendC::MicroAPI::Cast<float, half, castTrait0>(vreg1, vreg0, preg0);
126 AscendC::MicroAPI::Sub<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg3, vreg2, vreg1, preg0);126 AscendC::MicroAPI::Sub<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg3, vreg2, vreg1, preg0);
127 AscendC::MicroAPI::Mul<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg4, vreg6, vreg3, preg0);127 AscendC::MicroAPI::Mul<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg4, vreg6, vreg3, preg0);
128 AscendC::MicroAPI::Mul<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg7, vreg4, vreg1, preg0);128 AscendC::MicroAPI::Mul<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg7, vreg4, vreg1, preg0);
129 AscendC::MicroAPI::Cast<half, float, castTrait1>(vreg8, vreg7, preg0);129 AscendC::MicroAPI::Cast<half, float, castTrait1>(vreg8, vreg7, preg0);
130- AscendC::MicroAPI::DataCopy<half, AscendC::MicroAPI::StoreDist::DIST_PACK_B32>(130+ AscendC::MicroAPI::StoreAlign<half, AscendC::MicroAPI::StoreDist::DIST_PACK_B32>(
131 bufferOut0Addr + i * (AscendC::VECTOR_REG_WIDTH / sizeof(float)), vreg8, preg0);131 bufferOut0Addr + i * (AscendC::VECTOR_REG_WIDTH / sizeof(float)), vreg8, preg0);
132 }132 }
133 }133 }
Mactivation/sigmoid_grad/op_kernel/arch35/sigmoid_grad_f32.h+6-6
@@ -109,19 +109,19 @@ private:
109 vreg1, static_cast<float>(1.0), preg0);109 vreg1, static_cast<float>(1.0), preg0);
110 uint16_t vfLoopNum = (i0Extent + (AscendC::VECTOR_REG_WIDTH / sizeof(float)) - 1) /110 uint16_t vfLoopNum = (i0Extent + (AscendC::VECTOR_REG_WIDTH / sizeof(float)) - 1) /
111 (AscendC::VECTOR_REG_WIDTH / 4);111 (AscendC::VECTOR_REG_WIDTH / 4);
112- __local_mem__ float* bufferIn0Addr = (__local_mem__ float*)bufferIn0_.GetPhyAddr();112+ __ubuf__ float* bufferIn0Addr = (__ubuf__ float*)bufferIn0_.GetPhyAddr();
113- __local_mem__ float* bufferIn1Addr = (__local_mem__ float*)bufferIn1_.GetPhyAddr();113+ __ubuf__ float* bufferIn1Addr = (__ubuf__ float*)bufferIn1_.GetPhyAddr();
114- __local_mem__ float* bufferOut0Addr = (__local_mem__ float*)bufferOut0_.GetPhyAddr();114+ __ubuf__ float* bufferOut0Addr = (__ubuf__ float*)bufferOut0_.GetPhyAddr();
115 for (uint16_t i = 0; i < vfLoopNum; i++) {115 for (uint16_t i = 0; i < vfLoopNum; i++) {
116 preg0 = AscendC::MicroAPI::UpdateMask<float>(size);116 preg0 = AscendC::MicroAPI::UpdateMask<float>(size);
117- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(117+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(
118 vreg4, bufferIn1Addr + i * (AscendC::VECTOR_REG_WIDTH / sizeof(float)));118 vreg4, bufferIn1Addr + i * (AscendC::VECTOR_REG_WIDTH / sizeof(float)));
119- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(119+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(
120 vreg0, bufferIn0Addr + i * (AscendC::VECTOR_REG_WIDTH / sizeof(float)));120 vreg0, bufferIn0Addr + i * (AscendC::VECTOR_REG_WIDTH / sizeof(float)));
121 AscendC::MicroAPI::Sub<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg2, vreg1, vreg0, preg0);121 AscendC::MicroAPI::Sub<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg2, vreg1, vreg0, preg0);
122 AscendC::MicroAPI::Mul<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg3, vreg4, vreg2, preg0);122 AscendC::MicroAPI::Mul<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg3, vreg4, vreg2, preg0);
123 AscendC::MicroAPI::Mul<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg5, vreg3, vreg0, preg0);123 AscendC::MicroAPI::Mul<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg5, vreg3, vreg0, preg0);
124- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::StoreDist::DIST_NORM_B32>(124+ AscendC::MicroAPI::StoreAlign<float, AscendC::MicroAPI::StoreDist::DIST_NORM_B32>(
125 bufferOut0Addr + i * (AscendC::VECTOR_REG_WIDTH / sizeof(float)), vreg5, preg0);125 bufferOut0Addr + i * (AscendC::VECTOR_REG_WIDTH / sizeof(float)), vreg5, preg0);
126 }126 }
127 }127 }
Mactivation/silu_grad/op_kernel/arch35/silu_grad_bf16_nddma_with_loops.h+18-18
@@ -114,13 +114,13 @@ private:
114 uint16_t vfLoopNum = (ubSplitSize * tilingDataPtr_->outputStrides[tilingDataPtr_->ubSplitAxis] +114 uint16_t vfLoopNum = (ubSplitSize * tilingDataPtr_->outputStrides[tilingDataPtr_->ubSplitAxis] +
115 (AscendC::VECTOR_REG_WIDTH / 4) - 1) /115 (AscendC::VECTOR_REG_WIDTH / 4) - 1) /
116 (AscendC::VECTOR_REG_WIDTH / 4);116 (AscendC::VECTOR_REG_WIDTH / 4);
117- __local_mem__ bfloat16_t* bufferIn0Addr = (__local_mem__ bfloat16_t*)bufferIn0_.GetPhyAddr();117+ __ubuf__ bfloat16_t* bufferIn0Addr = (__ubuf__ bfloat16_t*)bufferIn0_.GetPhyAddr();
118- __local_mem__ float* bufferTmp0Addr = (__local_mem__ float*)bufferTmp0_.GetPhyAddr();118+ __ubuf__ float* bufferTmp0Addr = (__ubuf__ float*)bufferTmp0_.GetPhyAddr();
119- __local_mem__ float* bufferTmp1Addr = (__local_mem__ float*)bufferTmp1_.GetPhyAddr();119+ __ubuf__ float* bufferTmp1Addr = (__ubuf__ float*)bufferTmp1_.GetPhyAddr();
120- __local_mem__ float* bufferTmp2Addr = (__local_mem__ float*)bufferTmp2_.GetPhyAddr();120+ __ubuf__ float* bufferTmp2Addr = (__ubuf__ float*)bufferTmp2_.GetPhyAddr();
121 for (uint16_t i = 0; i < vfLoopNum; i++) {121 for (uint16_t i = 0; i < vfLoopNum; i++) {
122 preg0 = AscendC::MicroAPI::UpdateMask<float>(size);122 preg0 = AscendC::MicroAPI::UpdateMask<float>(size);
123- AscendC::MicroAPI::DataCopy<bfloat16_t, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(123+ AscendC::MicroAPI::LoadAlign<bfloat16_t, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(
124 vreg3, bufferIn0Addr + i * (AscendC::VECTOR_REG_WIDTH / 4));124 vreg3, bufferIn0Addr + i * (AscendC::VECTOR_REG_WIDTH / 4));
125 AscendC::MicroAPI::Cast<float, bfloat16_t, castTrait0>(vreg4, vreg3, preg0);125 AscendC::MicroAPI::Cast<float, bfloat16_t, castTrait0>(vreg4, vreg3, preg0);
126 AscendC::MicroAPI::Muls<float, float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(126 AscendC::MicroAPI::Muls<float, float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(
@@ -129,11 +129,11 @@ private:
129 AscendC::MicroAPI::Adds<float, float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(129 AscendC::MicroAPI::Adds<float, float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(
130 vreg7, vreg6, static_cast<float>(1), preg0);130 vreg7, vreg6, static_cast<float>(1), preg0);
131 AscendC::MicroAPI::Div<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg8, vreg2, vreg7, preg0);131 AscendC::MicroAPI::Div<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg8, vreg2, vreg7, preg0);
132- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::StoreDist::DIST_NORM_B32>(132+ AscendC::MicroAPI::StoreAlign<float, AscendC::MicroAPI::StoreDist::DIST_NORM_B32>(
133 bufferTmp0Addr + i * (AscendC::VECTOR_REG_WIDTH / 4), vreg8, preg0);133 bufferTmp0Addr + i * (AscendC::VECTOR_REG_WIDTH / 4), vreg8, preg0);
134- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::StoreDist::DIST_NORM_B32>(134+ AscendC::MicroAPI::StoreAlign<float, AscendC::MicroAPI::StoreDist::DIST_NORM_B32>(
135 bufferTmp1Addr + i * (AscendC::VECTOR_REG_WIDTH / 4), vreg2, preg0);135 bufferTmp1Addr + i * (AscendC::VECTOR_REG_WIDTH / 4), vreg2, preg0);
136- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::StoreDist::DIST_NORM_B32>(136+ AscendC::MicroAPI::StoreAlign<float, AscendC::MicroAPI::StoreDist::DIST_NORM_B32>(
137 bufferTmp2Addr + i * (AscendC::VECTOR_REG_WIDTH / 4), vreg4, preg0);137 bufferTmp2Addr + i * (AscendC::VECTOR_REG_WIDTH / 4), vreg4, preg0);
138 }138 }
139 }139 }
@@ -177,29 +177,29 @@ private:
177 uint16_t vfLoopNum = (ubSplitSize * tilingDataPtr_->outputStrides[tilingDataPtr_->ubSplitAxis] +177 uint16_t vfLoopNum = (ubSplitSize * tilingDataPtr_->outputStrides[tilingDataPtr_->ubSplitAxis] +
178 (AscendC::VECTOR_REG_WIDTH / 4) - 1) /178 (AscendC::VECTOR_REG_WIDTH / 4) - 1) /
179 (AscendC::VECTOR_REG_WIDTH / 4);179 (AscendC::VECTOR_REG_WIDTH / 4);
180- __local_mem__ bfloat16_t* bufferIn1Addr = (__local_mem__ bfloat16_t*)bufferIn1_.GetPhyAddr();180+ __ubuf__ bfloat16_t* bufferIn1Addr = (__ubuf__ bfloat16_t*)bufferIn1_.GetPhyAddr();
181- __local_mem__ bfloat16_t* bufferOut0Addr = (__local_mem__ bfloat16_t*)bufferOut0_.GetPhyAddr();181+ __ubuf__ bfloat16_t* bufferOut0Addr = (__ubuf__ bfloat16_t*)bufferOut0_.GetPhyAddr();
182- __local_mem__ float* bufferTmp0Addr = (__local_mem__ float*)bufferTmp0_.GetPhyAddr();182+ __ubuf__ float* bufferTmp0Addr = (__ubuf__ float*)bufferTmp0_.GetPhyAddr();
183- __local_mem__ float* bufferTmp1Addr = (__local_mem__ float*)bufferTmp1_.GetPhyAddr();183+ __ubuf__ float* bufferTmp1Addr = (__ubuf__ float*)bufferTmp1_.GetPhyAddr();
184- __local_mem__ float* bufferTmp2Addr = (__local_mem__ float*)bufferTmp2_.GetPhyAddr();184+ __ubuf__ float* bufferTmp2Addr = (__ubuf__ float*)bufferTmp2_.GetPhyAddr();
185 for (uint16_t i = 0; i < vfLoopNum; i++) {185 for (uint16_t i = 0; i < vfLoopNum; i++) {
186 preg0 = AscendC::MicroAPI::UpdateMask<float>(size);186 preg0 = AscendC::MicroAPI::UpdateMask<float>(size);
187- AscendC::MicroAPI::DataCopy<bfloat16_t, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(187+ AscendC::MicroAPI::LoadAlign<bfloat16_t, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(
188 vreg0, bufferIn1Addr + i * (AscendC::VECTOR_REG_WIDTH / 4));188 vreg0, bufferIn1Addr + i * (AscendC::VECTOR_REG_WIDTH / 4));
189 AscendC::MicroAPI::Cast<float, bfloat16_t, castTrait0>(vreg1, vreg0, preg0);189 AscendC::MicroAPI::Cast<float, bfloat16_t, castTrait0>(vreg1, vreg0, preg0);
190- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(190+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(
191 vreg8, bufferTmp0Addr + i * (AscendC::VECTOR_REG_WIDTH / 4));191 vreg8, bufferTmp0Addr + i * (AscendC::VECTOR_REG_WIDTH / 4));
192- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(192+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(
193 vreg2, bufferTmp1Addr + i * (AscendC::VECTOR_REG_WIDTH / 4));193 vreg2, bufferTmp1Addr + i * (AscendC::VECTOR_REG_WIDTH / 4));
194 AscendC::MicroAPI::Sub<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg9, vreg2, vreg8, preg0);194 AscendC::MicroAPI::Sub<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg9, vreg2, vreg8, preg0);
195- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(195+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(
196 vreg4, bufferTmp2Addr + i * (AscendC::VECTOR_REG_WIDTH / 4));196 vreg4, bufferTmp2Addr + i * (AscendC::VECTOR_REG_WIDTH / 4));
197 AscendC::MicroAPI::Mul<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg10, vreg9, vreg4, preg0);197 AscendC::MicroAPI::Mul<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg10, vreg9, vreg4, preg0);
198 AscendC::MicroAPI::Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg11, vreg2, vreg10, preg0);198 AscendC::MicroAPI::Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg11, vreg2, vreg10, preg0);
199 AscendC::MicroAPI::Mul<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg12, vreg8, vreg11, preg0);199 AscendC::MicroAPI::Mul<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg12, vreg8, vreg11, preg0);
200 AscendC::MicroAPI::Mul<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg13, vreg1, vreg12, preg0);200 AscendC::MicroAPI::Mul<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg13, vreg1, vreg12, preg0);
201 AscendC::MicroAPI::Cast<bfloat16_t, float, castTrait1>(vreg14, vreg13, preg0);201 AscendC::MicroAPI::Cast<bfloat16_t, float, castTrait1>(vreg14, vreg13, preg0);
202- AscendC::MicroAPI::DataCopy<bfloat16_t, AscendC::MicroAPI::StoreDist::DIST_PACK_B32>(202+ AscendC::MicroAPI::StoreAlign<bfloat16_t, AscendC::MicroAPI::StoreDist::DIST_PACK_B32>(
203 bufferOut0Addr + i * (AscendC::VECTOR_REG_WIDTH / 4), vreg14, preg0);203 bufferOut0Addr + i * (AscendC::VECTOR_REG_WIDTH / 4), vreg14, preg0);
204 }204 }
205 }205 }
Mactivation/silu_grad/op_kernel/arch35/silu_grad_bf16_nddma_without_loops.h+18-18
@@ -114,13 +114,13 @@ private:
114 uint16_t vfLoopNum = (ubSplitSize * tilingDataPtr_->outputStrides[tilingDataPtr_->ubSplitAxis] +114 uint16_t vfLoopNum = (ubSplitSize * tilingDataPtr_->outputStrides[tilingDataPtr_->ubSplitAxis] +
115 (AscendC::VECTOR_REG_WIDTH / 4) - 1) /115 (AscendC::VECTOR_REG_WIDTH / 4) - 1) /
116 (AscendC::VECTOR_REG_WIDTH / 4);116 (AscendC::VECTOR_REG_WIDTH / 4);
117- __local_mem__ bfloat16_t* bufferIn0Addr = (__local_mem__ bfloat16_t*)bufferIn0_.GetPhyAddr();117+ __ubuf__ bfloat16_t* bufferIn0Addr = (__ubuf__ bfloat16_t*)bufferIn0_.GetPhyAddr();
118- __local_mem__ float* bufferTmp0Addr = (__local_mem__ float*)bufferTmp0_.GetPhyAddr();118+ __ubuf__ float* bufferTmp0Addr = (__ubuf__ float*)bufferTmp0_.GetPhyAddr();
119- __local_mem__ float* bufferTmp1Addr = (__local_mem__ float*)bufferTmp1_.GetPhyAddr();119+ __ubuf__ float* bufferTmp1Addr = (__ubuf__ float*)bufferTmp1_.GetPhyAddr();
120- __local_mem__ float* bufferTmp2Addr = (__local_mem__ float*)bufferTmp2_.GetPhyAddr();120+ __ubuf__ float* bufferTmp2Addr = (__ubuf__ float*)bufferTmp2_.GetPhyAddr();
121 for (uint16_t i = 0; i < vfLoopNum; i++) {121 for (uint16_t i = 0; i < vfLoopNum; i++) {
122 preg0 = AscendC::MicroAPI::UpdateMask<float>(size);122 preg0 = AscendC::MicroAPI::UpdateMask<float>(size);
123- AscendC::MicroAPI::DataCopy<bfloat16_t, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(123+ AscendC::MicroAPI::LoadAlign<bfloat16_t, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(
124 vreg3, bufferIn0Addr + i * (AscendC::VECTOR_REG_WIDTH / 4));124 vreg3, bufferIn0Addr + i * (AscendC::VECTOR_REG_WIDTH / 4));
125 AscendC::MicroAPI::Cast<float, bfloat16_t, castTrait0>(vreg4, vreg3, preg0);125 AscendC::MicroAPI::Cast<float, bfloat16_t, castTrait0>(vreg4, vreg3, preg0);
126 AscendC::MicroAPI::Muls<float, float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(126 AscendC::MicroAPI::Muls<float, float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(
@@ -129,11 +129,11 @@ private:
129 AscendC::MicroAPI::Adds<float, float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(129 AscendC::MicroAPI::Adds<float, float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(
130 vreg7, vreg6, static_cast<float>(1), preg0);130 vreg7, vreg6, static_cast<float>(1), preg0);
131 AscendC::MicroAPI::Div<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg8, vreg2, vreg7, preg0);131 AscendC::MicroAPI::Div<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg8, vreg2, vreg7, preg0);
132- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::StoreDist::DIST_NORM_B32>(132+ AscendC::MicroAPI::StoreAlign<float, AscendC::MicroAPI::StoreDist::DIST_NORM_B32>(
133 bufferTmp0Addr + i * (AscendC::VECTOR_REG_WIDTH / 4), vreg8, preg0);133 bufferTmp0Addr + i * (AscendC::VECTOR_REG_WIDTH / 4), vreg8, preg0);
134- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::StoreDist::DIST_NORM_B32>(134+ AscendC::MicroAPI::StoreAlign<float, AscendC::MicroAPI::StoreDist::DIST_NORM_B32>(
135 bufferTmp1Addr + i * (AscendC::VECTOR_REG_WIDTH / 4), vreg2, preg0);135 bufferTmp1Addr + i * (AscendC::VECTOR_REG_WIDTH / 4), vreg2, preg0);
136- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::StoreDist::DIST_NORM_B32>(136+ AscendC::MicroAPI::StoreAlign<float, AscendC::MicroAPI::StoreDist::DIST_NORM_B32>(
137 bufferTmp2Addr + i * (AscendC::VECTOR_REG_WIDTH / 4), vreg4, preg0);137 bufferTmp2Addr + i * (AscendC::VECTOR_REG_WIDTH / 4), vreg4, preg0);
138 }138 }
139 }139 }
@@ -177,29 +177,29 @@ private:
177 uint16_t vfLoopNum = (ubSplitSize * tilingDataPtr_->outputStrides[tilingDataPtr_->ubSplitAxis] +177 uint16_t vfLoopNum = (ubSplitSize * tilingDataPtr_->outputStrides[tilingDataPtr_->ubSplitAxis] +
178 (AscendC::VECTOR_REG_WIDTH / 4) - 1) /178 (AscendC::VECTOR_REG_WIDTH / 4) - 1) /
179 (AscendC::VECTOR_REG_WIDTH / 4);179 (AscendC::VECTOR_REG_WIDTH / 4);
180- __local_mem__ bfloat16_t* bufferIn1Addr = (__local_mem__ bfloat16_t*)bufferIn1_.GetPhyAddr();180+ __ubuf__ bfloat16_t* bufferIn1Addr = (__ubuf__ bfloat16_t*)bufferIn1_.GetPhyAddr();
181- __local_mem__ bfloat16_t* bufferOut0Addr = (__local_mem__ bfloat16_t*)bufferOut0_.GetPhyAddr();181+ __ubuf__ bfloat16_t* bufferOut0Addr = (__ubuf__ bfloat16_t*)bufferOut0_.GetPhyAddr();
182- __local_mem__ float* bufferTmp0Addr = (__local_mem__ float*)bufferTmp0_.GetPhyAddr();182+ __ubuf__ float* bufferTmp0Addr = (__ubuf__ float*)bufferTmp0_.GetPhyAddr();
183- __local_mem__ float* bufferTmp1Addr = (__local_mem__ float*)bufferTmp1_.GetPhyAddr();183+ __ubuf__ float* bufferTmp1Addr = (__ubuf__ float*)bufferTmp1_.GetPhyAddr();
184- __local_mem__ float* bufferTmp2Addr = (__local_mem__ float*)bufferTmp2_.GetPhyAddr();184+ __ubuf__ float* bufferTmp2Addr = (__ubuf__ float*)bufferTmp2_.GetPhyAddr();
185 for (uint16_t i = 0; i < vfLoopNum; i++) {185 for (uint16_t i = 0; i < vfLoopNum; i++) {
186 preg0 = AscendC::MicroAPI::UpdateMask<float>(size);186 preg0 = AscendC::MicroAPI::UpdateMask<float>(size);
187- AscendC::MicroAPI::DataCopy<bfloat16_t, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(187+ AscendC::MicroAPI::LoadAlign<bfloat16_t, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(
188 vreg0, bufferIn1Addr + i * (AscendC::VECTOR_REG_WIDTH / 4));188 vreg0, bufferIn1Addr + i * (AscendC::VECTOR_REG_WIDTH / 4));
189 AscendC::MicroAPI::Cast<float, bfloat16_t, castTrait0>(vreg1, vreg0, preg0);189 AscendC::MicroAPI::Cast<float, bfloat16_t, castTrait0>(vreg1, vreg0, preg0);
190- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(190+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(
191 vreg8, bufferTmp0Addr + i * (AscendC::VECTOR_REG_WIDTH / 4));191 vreg8, bufferTmp0Addr + i * (AscendC::VECTOR_REG_WIDTH / 4));
192- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(192+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(
193 vreg2, bufferTmp1Addr + i * (AscendC::VECTOR_REG_WIDTH / 4));193 vreg2, bufferTmp1Addr + i * (AscendC::VECTOR_REG_WIDTH / 4));
194 AscendC::MicroAPI::Sub<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg9, vreg2, vreg8, preg0);194 AscendC::MicroAPI::Sub<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg9, vreg2, vreg8, preg0);
195- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(195+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(
196 vreg4, bufferTmp2Addr + i * (AscendC::VECTOR_REG_WIDTH / 4));196 vreg4, bufferTmp2Addr + i * (AscendC::VECTOR_REG_WIDTH / 4));
197 AscendC::MicroAPI::Mul<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg10, vreg9, vreg4, preg0);197 AscendC::MicroAPI::Mul<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg10, vreg9, vreg4, preg0);
198 AscendC::MicroAPI::Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg11, vreg2, vreg10, preg0);198 AscendC::MicroAPI::Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg11, vreg2, vreg10, preg0);
199 AscendC::MicroAPI::Mul<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg12, vreg8, vreg11, preg0);199 AscendC::MicroAPI::Mul<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg12, vreg8, vreg11, preg0);
200 AscendC::MicroAPI::Mul<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg13, vreg1, vreg12, preg0);200 AscendC::MicroAPI::Mul<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg13, vreg1, vreg12, preg0);
201 AscendC::MicroAPI::Cast<bfloat16_t, float, castTrait1>(vreg14, vreg13, preg0);201 AscendC::MicroAPI::Cast<bfloat16_t, float, castTrait1>(vreg14, vreg13, preg0);
202- AscendC::MicroAPI::DataCopy<bfloat16_t, AscendC::MicroAPI::StoreDist::DIST_PACK_B32>(202+ AscendC::MicroAPI::StoreAlign<bfloat16_t, AscendC::MicroAPI::StoreDist::DIST_PACK_B32>(
203 bufferOut0Addr + i * (AscendC::VECTOR_REG_WIDTH / 4), vreg14, preg0);203 bufferOut0Addr + i * (AscendC::VECTOR_REG_WIDTH / 4), vreg14, preg0);
204 }204 }
205 }205 }
Mactivation/silu_grad/op_kernel/arch35/silu_grad_dtype_comb_0_nddma_with_loops.h+6-6
@@ -131,15 +131,15 @@ private:
131 uint16_t vfLoopNum = (ubSplitSize * tilingDataPtr_->outputStrides[tilingDataPtr_->ubSplitAxis] +131 uint16_t vfLoopNum = (ubSplitSize * tilingDataPtr_->outputStrides[tilingDataPtr_->ubSplitAxis] +
132 (AscendC::VECTOR_REG_WIDTH / 4) - 1) /132 (AscendC::VECTOR_REG_WIDTH / 4) - 1) /
133 (AscendC::VECTOR_REG_WIDTH / 4);133 (AscendC::VECTOR_REG_WIDTH / 4);
134- __local_mem__ bfloat16_t* bufferIn1Addr = (__local_mem__ bfloat16_t*)bufferIn1_.GetPhyAddr();134+ __ubuf__ bfloat16_t* bufferIn1Addr = (__ubuf__ bfloat16_t*)bufferIn1_.GetPhyAddr();
135- __local_mem__ float* bufferOut0Addr = (__local_mem__ float*)bufferOut0_.GetPhyAddr();135+ __ubuf__ float* bufferOut0Addr = (__ubuf__ float*)bufferOut0_.GetPhyAddr();
136- __local_mem__ half* bufferIn0Addr = (__local_mem__ half*)bufferIn0_.GetPhyAddr();136+ __ubuf__ half* bufferIn0Addr = (__ubuf__ half*)bufferIn0_.GetPhyAddr();
137 for (uint16_t i = 0; i < vfLoopNum; i++) {137 for (uint16_t i = 0; i < vfLoopNum; i++) {
138 preg0 = AscendC::MicroAPI::UpdateMask<float>(size);138 preg0 = AscendC::MicroAPI::UpdateMask<float>(size);
139- AscendC::MicroAPI::DataCopy<half, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(139+ AscendC::MicroAPI::LoadAlign<half, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(
140 vreg0, bufferIn0Addr + i * (AscendC::VECTOR_REG_WIDTH / 4));140 vreg0, bufferIn0Addr + i * (AscendC::VECTOR_REG_WIDTH / 4));
141 AscendC::MicroAPI::Cast<float, half, castTrait0>(vreg1, vreg0, preg0);141 AscendC::MicroAPI::Cast<float, half, castTrait0>(vreg1, vreg0, preg0);
142- AscendC::MicroAPI::DataCopy<bfloat16_t, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(142+ AscendC::MicroAPI::LoadAlign<bfloat16_t, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(
143 vreg3, bufferIn1Addr + i * (AscendC::VECTOR_REG_WIDTH / 4));143 vreg3, bufferIn1Addr + i * (AscendC::VECTOR_REG_WIDTH / 4));
144 AscendC::MicroAPI::Cast<float, bfloat16_t, castTrait0>(vreg4, vreg3, preg0);144 AscendC::MicroAPI::Cast<float, bfloat16_t, castTrait0>(vreg4, vreg3, preg0);
145 AscendC::MicroAPI::Muls<float, float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(145 AscendC::MicroAPI::Muls<float, float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(
@@ -153,7 +153,7 @@ private:
153 AscendC::MicroAPI::Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg11, vreg2, vreg10, preg0);153 AscendC::MicroAPI::Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg11, vreg2, vreg10, preg0);
154 AscendC::MicroAPI::Mul<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg12, vreg8, vreg11, preg0);154 AscendC::MicroAPI::Mul<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg12, vreg8, vreg11, preg0);
155 AscendC::MicroAPI::Mul<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg13, vreg1, vreg12, preg0);155 AscendC::MicroAPI::Mul<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg13, vreg1, vreg12, preg0);
156- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::StoreDist::DIST_NORM_B32>(156+ AscendC::MicroAPI::StoreAlign<float, AscendC::MicroAPI::StoreDist::DIST_NORM_B32>(
157 bufferOut0Addr + i * (AscendC::VECTOR_REG_WIDTH / 4), vreg13, preg0);157 bufferOut0Addr + i * (AscendC::VECTOR_REG_WIDTH / 4), vreg13, preg0);
158 }158 }
159 }159 }
Mactivation/silu_grad/op_kernel/arch35/silu_grad_dtype_comb_0_nddma_without_loops.h+6-6
@@ -131,15 +131,15 @@ private:
131 uint16_t vfLoopNum = (ubSplitSize * tilingDataPtr_->outputStrides[tilingDataPtr_->ubSplitAxis] +131 uint16_t vfLoopNum = (ubSplitSize * tilingDataPtr_->outputStrides[tilingDataPtr_->ubSplitAxis] +
132 (AscendC::VECTOR_REG_WIDTH / 4) - 1) /132 (AscendC::VECTOR_REG_WIDTH / 4) - 1) /
133 (AscendC::VECTOR_REG_WIDTH / 4);133 (AscendC::VECTOR_REG_WIDTH / 4);
134- __local_mem__ bfloat16_t* bufferIn1Addr = (__local_mem__ bfloat16_t*)bufferIn1_.GetPhyAddr();134+ __ubuf__ bfloat16_t* bufferIn1Addr = (__ubuf__ bfloat16_t*)bufferIn1_.GetPhyAddr();
135- __local_mem__ float* bufferOut0Addr = (__local_mem__ float*)bufferOut0_.GetPhyAddr();135+ __ubuf__ float* bufferOut0Addr = (__ubuf__ float*)bufferOut0_.GetPhyAddr();
136- __local_mem__ half* bufferIn0Addr = (__local_mem__ half*)bufferIn0_.GetPhyAddr();136+ __ubuf__ half* bufferIn0Addr = (__ubuf__ half*)bufferIn0_.GetPhyAddr();
137 for (uint16_t i = 0; i < vfLoopNum; i++) {137 for (uint16_t i = 0; i < vfLoopNum; i++) {
138 preg0 = AscendC::MicroAPI::UpdateMask<float>(size);138 preg0 = AscendC::MicroAPI::UpdateMask<float>(size);
139- AscendC::MicroAPI::DataCopy<half, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(139+ AscendC::MicroAPI::LoadAlign<half, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(
140 vreg0, bufferIn0Addr + i * (AscendC::VECTOR_REG_WIDTH / 4));140 vreg0, bufferIn0Addr + i * (AscendC::VECTOR_REG_WIDTH / 4));
141 AscendC::MicroAPI::Cast<float, half, castTrait0>(vreg1, vreg0, preg0);141 AscendC::MicroAPI::Cast<float, half, castTrait0>(vreg1, vreg0, preg0);
142- AscendC::MicroAPI::DataCopy<bfloat16_t, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(142+ AscendC::MicroAPI::LoadAlign<bfloat16_t, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(
143 vreg3, bufferIn1Addr + i * (AscendC::VECTOR_REG_WIDTH / 4));143 vreg3, bufferIn1Addr + i * (AscendC::VECTOR_REG_WIDTH / 4));
144 AscendC::MicroAPI::Cast<float, bfloat16_t, castTrait0>(vreg4, vreg3, preg0);144 AscendC::MicroAPI::Cast<float, bfloat16_t, castTrait0>(vreg4, vreg3, preg0);
145 AscendC::MicroAPI::Muls<float, float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(145 AscendC::MicroAPI::Muls<float, float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(
@@ -153,7 +153,7 @@ private:
153 AscendC::MicroAPI::Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg11, vreg2, vreg10, preg0);153 AscendC::MicroAPI::Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg11, vreg2, vreg10, preg0);
154 AscendC::MicroAPI::Mul<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg12, vreg8, vreg11, preg0);154 AscendC::MicroAPI::Mul<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg12, vreg8, vreg11, preg0);
155 AscendC::MicroAPI::Mul<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg13, vreg1, vreg12, preg0);155 AscendC::MicroAPI::Mul<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg13, vreg1, vreg12, preg0);
156- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::StoreDist::DIST_NORM_B32>(156+ AscendC::MicroAPI::StoreAlign<float, AscendC::MicroAPI::StoreDist::DIST_NORM_B32>(
157 bufferOut0Addr + i * (AscendC::VECTOR_REG_WIDTH / 4), vreg13, preg0);157 bufferOut0Addr + i * (AscendC::VECTOR_REG_WIDTH / 4), vreg13, preg0);
158 }158 }
159 }159 }
Mactivation/silu_grad/op_kernel/arch35/silu_grad_dtype_comb_1_nddma_with_loops.h+6-6
@@ -129,15 +129,15 @@ private:
129 uint16_t vfLoopNum = (ubSplitSize * tilingDataPtr_->outputStrides[tilingDataPtr_->ubSplitAxis] +129 uint16_t vfLoopNum = (ubSplitSize * tilingDataPtr_->outputStrides[tilingDataPtr_->ubSplitAxis] +
130 (AscendC::VECTOR_REG_WIDTH / 4) - 1) /130 (AscendC::VECTOR_REG_WIDTH / 4) - 1) /
131 (AscendC::VECTOR_REG_WIDTH / 4);131 (AscendC::VECTOR_REG_WIDTH / 4);
132- __local_mem__ float* bufferIn1Addr = (__local_mem__ float*)bufferIn1_.GetPhyAddr();132+ __ubuf__ float* bufferIn1Addr = (__ubuf__ float*)bufferIn1_.GetPhyAddr();
133- __local_mem__ float* bufferOut0Addr = (__local_mem__ float*)bufferOut0_.GetPhyAddr();133+ __ubuf__ float* bufferOut0Addr = (__ubuf__ float*)bufferOut0_.GetPhyAddr();
134- __local_mem__ half* bufferIn0Addr = (__local_mem__ half*)bufferIn0_.GetPhyAddr();134+ __ubuf__ half* bufferIn0Addr = (__ubuf__ half*)bufferIn0_.GetPhyAddr();
135 for (uint16_t i = 0; i < vfLoopNum; i++) {135 for (uint16_t i = 0; i < vfLoopNum; i++) {
136 preg0 = AscendC::MicroAPI::UpdateMask<float>(size);136 preg0 = AscendC::MicroAPI::UpdateMask<float>(size);
137- AscendC::MicroAPI::DataCopy<half, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(137+ AscendC::MicroAPI::LoadAlign<half, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(
138 vreg0, bufferIn0Addr + i * (AscendC::VECTOR_REG_WIDTH / 4));138 vreg0, bufferIn0Addr + i * (AscendC::VECTOR_REG_WIDTH / 4));
139 AscendC::MicroAPI::Cast<float, half, castTrait0>(vreg1, vreg0, preg0);139 AscendC::MicroAPI::Cast<float, half, castTrait0>(vreg1, vreg0, preg0);
140- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(140+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(
141 vreg3, bufferIn1Addr + i * (AscendC::VECTOR_REG_WIDTH / 4));141 vreg3, bufferIn1Addr + i * (AscendC::VECTOR_REG_WIDTH / 4));
142 AscendC::MicroAPI::Muls<float, float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(142 AscendC::MicroAPI::Muls<float, float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(
143 vreg4, vreg3, static_cast<float>(-1), preg0);143 vreg4, vreg3, static_cast<float>(-1), preg0);
@@ -150,7 +150,7 @@ private:
150 AscendC::MicroAPI::Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg10, vreg2, vreg9, preg0);150 AscendC::MicroAPI::Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg10, vreg2, vreg9, preg0);
151 AscendC::MicroAPI::Mul<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg11, vreg7, vreg10, preg0);151 AscendC::MicroAPI::Mul<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg11, vreg7, vreg10, preg0);
152 AscendC::MicroAPI::Mul<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg12, vreg1, vreg11, preg0);152 AscendC::MicroAPI::Mul<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg12, vreg1, vreg11, preg0);
153- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::StoreDist::DIST_NORM_B32>(153+ AscendC::MicroAPI::StoreAlign<float, AscendC::MicroAPI::StoreDist::DIST_NORM_B32>(
154 bufferOut0Addr + i * (AscendC::VECTOR_REG_WIDTH / 4), vreg12, preg0);154 bufferOut0Addr + i * (AscendC::VECTOR_REG_WIDTH / 4), vreg12, preg0);
155 }155 }
156 }156 }
Mactivation/silu_grad/op_kernel/arch35/silu_grad_dtype_comb_1_nddma_without_loops.h+6-6
@@ -129,15 +129,15 @@ private:
129 uint16_t vfLoopNum = (ubSplitSize * tilingDataPtr_->outputStrides[tilingDataPtr_->ubSplitAxis] +129 uint16_t vfLoopNum = (ubSplitSize * tilingDataPtr_->outputStrides[tilingDataPtr_->ubSplitAxis] +
130 (AscendC::VECTOR_REG_WIDTH / 4) - 1) /130 (AscendC::VECTOR_REG_WIDTH / 4) - 1) /
131 (AscendC::VECTOR_REG_WIDTH / 4);131 (AscendC::VECTOR_REG_WIDTH / 4);
132- __local_mem__ float* bufferIn1Addr = (__local_mem__ float*)bufferIn1_.GetPhyAddr();132+ __ubuf__ float* bufferIn1Addr = (__ubuf__ float*)bufferIn1_.GetPhyAddr();
133- __local_mem__ float* bufferOut0Addr = (__local_mem__ float*)bufferOut0_.GetPhyAddr();133+ __ubuf__ float* bufferOut0Addr = (__ubuf__ float*)bufferOut0_.GetPhyAddr();
134- __local_mem__ half* bufferIn0Addr = (__local_mem__ half*)bufferIn0_.GetPhyAddr();134+ __ubuf__ half* bufferIn0Addr = (__ubuf__ half*)bufferIn0_.GetPhyAddr();
135 for (uint16_t i = 0; i < vfLoopNum; i++) {135 for (uint16_t i = 0; i < vfLoopNum; i++) {
136 preg0 = AscendC::MicroAPI::UpdateMask<float>(size);136 preg0 = AscendC::MicroAPI::UpdateMask<float>(size);
137- AscendC::MicroAPI::DataCopy<half, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(137+ AscendC::MicroAPI::LoadAlign<half, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(
138 vreg0, bufferIn0Addr + i * (AscendC::VECTOR_REG_WIDTH / 4));138 vreg0, bufferIn0Addr + i * (AscendC::VECTOR_REG_WIDTH / 4));
139 AscendC::MicroAPI::Cast<float, half, castTrait0>(vreg1, vreg0, preg0);139 AscendC::MicroAPI::Cast<float, half, castTrait0>(vreg1, vreg0, preg0);
140- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(140+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(
141 vreg3, bufferIn1Addr + i * (AscendC::VECTOR_REG_WIDTH / 4));141 vreg3, bufferIn1Addr + i * (AscendC::VECTOR_REG_WIDTH / 4));
142 AscendC::MicroAPI::Muls<float, float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(142 AscendC::MicroAPI::Muls<float, float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(
143 vreg4, vreg3, static_cast<float>(-1), preg0);143 vreg4, vreg3, static_cast<float>(-1), preg0);
@@ -150,7 +150,7 @@ private:
150 AscendC::MicroAPI::Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg10, vreg2, vreg9, preg0);150 AscendC::MicroAPI::Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg10, vreg2, vreg9, preg0);
151 AscendC::MicroAPI::Mul<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg11, vreg7, vreg10, preg0);151 AscendC::MicroAPI::Mul<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg11, vreg7, vreg10, preg0);
152 AscendC::MicroAPI::Mul<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg12, vreg1, vreg11, preg0);152 AscendC::MicroAPI::Mul<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg12, vreg1, vreg11, preg0);
153- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::StoreDist::DIST_NORM_B32>(153+ AscendC::MicroAPI::StoreAlign<float, AscendC::MicroAPI::StoreDist::DIST_NORM_B32>(
154 bufferOut0Addr + i * (AscendC::VECTOR_REG_WIDTH / 4), vreg12, preg0);154 bufferOut0Addr + i * (AscendC::VECTOR_REG_WIDTH / 4), vreg12, preg0);
155 }155 }
156 }156 }
Mactivation/silu_grad/op_kernel/arch35/silu_grad_dtype_comb_2_nddma_with_loops.h+6-6
@@ -131,15 +131,15 @@ private:
131 uint16_t vfLoopNum = (ubSplitSize * tilingDataPtr_->outputStrides[tilingDataPtr_->ubSplitAxis] +131 uint16_t vfLoopNum = (ubSplitSize * tilingDataPtr_->outputStrides[tilingDataPtr_->ubSplitAxis] +
132 (AscendC::VECTOR_REG_WIDTH / 4) - 1) /132 (AscendC::VECTOR_REG_WIDTH / 4) - 1) /
133 (AscendC::VECTOR_REG_WIDTH / 4);133 (AscendC::VECTOR_REG_WIDTH / 4);
134- __local_mem__ bfloat16_t* bufferIn0Addr = (__local_mem__ bfloat16_t*)bufferIn0_.GetPhyAddr();134+ __ubuf__ bfloat16_t* bufferIn0Addr = (__ubuf__ bfloat16_t*)bufferIn0_.GetPhyAddr();
135- __local_mem__ float* bufferOut0Addr = (__local_mem__ float*)bufferOut0_.GetPhyAddr();135+ __ubuf__ float* bufferOut0Addr = (__ubuf__ float*)bufferOut0_.GetPhyAddr();
136- __local_mem__ half* bufferIn1Addr = (__local_mem__ half*)bufferIn1_.GetPhyAddr();136+ __ubuf__ half* bufferIn1Addr = (__ubuf__ half*)bufferIn1_.GetPhyAddr();
137 for (uint16_t i = 0; i < vfLoopNum; i++) {137 for (uint16_t i = 0; i < vfLoopNum; i++) {
138 preg0 = AscendC::MicroAPI::UpdateMask<float>(size);138 preg0 = AscendC::MicroAPI::UpdateMask<float>(size);
139- AscendC::MicroAPI::DataCopy<bfloat16_t, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(139+ AscendC::MicroAPI::LoadAlign<bfloat16_t, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(
140 vreg0, bufferIn0Addr + i * (AscendC::VECTOR_REG_WIDTH / 4));140 vreg0, bufferIn0Addr + i * (AscendC::VECTOR_REG_WIDTH / 4));
141 AscendC::MicroAPI::Cast<float, bfloat16_t, castTrait0>(vreg1, vreg0, preg0);141 AscendC::MicroAPI::Cast<float, bfloat16_t, castTrait0>(vreg1, vreg0, preg0);
142- AscendC::MicroAPI::DataCopy<half, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(142+ AscendC::MicroAPI::LoadAlign<half, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(
143 vreg3, bufferIn1Addr + i * (AscendC::VECTOR_REG_WIDTH / 4));143 vreg3, bufferIn1Addr + i * (AscendC::VECTOR_REG_WIDTH / 4));
144 AscendC::MicroAPI::Cast<float, half, castTrait0>(vreg4, vreg3, preg0);144 AscendC::MicroAPI::Cast<float, half, castTrait0>(vreg4, vreg3, preg0);
145 AscendC::MicroAPI::Muls<float, float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(145 AscendC::MicroAPI::Muls<float, float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(
@@ -153,7 +153,7 @@ private:
153 AscendC::MicroAPI::Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg11, vreg2, vreg10, preg0);153 AscendC::MicroAPI::Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg11, vreg2, vreg10, preg0);
154 AscendC::MicroAPI::Mul<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg12, vreg8, vreg11, preg0);154 AscendC::MicroAPI::Mul<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg12, vreg8, vreg11, preg0);
155 AscendC::MicroAPI::Mul<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg13, vreg1, vreg12, preg0);155 AscendC::MicroAPI::Mul<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg13, vreg1, vreg12, preg0);
156- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::StoreDist::DIST_NORM_B32>(156+ AscendC::MicroAPI::StoreAlign<float, AscendC::MicroAPI::StoreDist::DIST_NORM_B32>(
157 bufferOut0Addr + i * (AscendC::VECTOR_REG_WIDTH / 4), vreg13, preg0);157 bufferOut0Addr + i * (AscendC::VECTOR_REG_WIDTH / 4), vreg13, preg0);
158 }158 }
159 }159 }
Mactivation/silu_grad/op_kernel/arch35/silu_grad_dtype_comb_2_nddma_without_loops.h+6-6
@@ -131,15 +131,15 @@ private:
131 uint16_t vfLoopNum = (ubSplitSize * tilingDataPtr_->outputStrides[tilingDataPtr_->ubSplitAxis] +131 uint16_t vfLoopNum = (ubSplitSize * tilingDataPtr_->outputStrides[tilingDataPtr_->ubSplitAxis] +
132 (AscendC::VECTOR_REG_WIDTH / 4) - 1) /132 (AscendC::VECTOR_REG_WIDTH / 4) - 1) /
133 (AscendC::VECTOR_REG_WIDTH / 4);133 (AscendC::VECTOR_REG_WIDTH / 4);
134- __local_mem__ bfloat16_t* bufferIn0Addr = (__local_mem__ bfloat16_t*)bufferIn0_.GetPhyAddr();134+ __ubuf__ bfloat16_t* bufferIn0Addr = (__ubuf__ bfloat16_t*)bufferIn0_.GetPhyAddr();
135- __local_mem__ float* bufferOut0Addr = (__local_mem__ float*)bufferOut0_.GetPhyAddr();135+ __ubuf__ float* bufferOut0Addr = (__ubuf__ float*)bufferOut0_.GetPhyAddr();
136- __local_mem__ half* bufferIn1Addr = (__local_mem__ half*)bufferIn1_.GetPhyAddr();136+ __ubuf__ half* bufferIn1Addr = (__ubuf__ half*)bufferIn1_.GetPhyAddr();
137 for (uint16_t i = 0; i < vfLoopNum; i++) {137 for (uint16_t i = 0; i < vfLoopNum; i++) {
138 preg0 = AscendC::MicroAPI::UpdateMask<float>(size);138 preg0 = AscendC::MicroAPI::UpdateMask<float>(size);
139- AscendC::MicroAPI::DataCopy<bfloat16_t, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(139+ AscendC::MicroAPI::LoadAlign<bfloat16_t, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(
140 vreg0, bufferIn0Addr + i * (AscendC::VECTOR_REG_WIDTH / 4));140 vreg0, bufferIn0Addr + i * (AscendC::VECTOR_REG_WIDTH / 4));
141 AscendC::MicroAPI::Cast<float, bfloat16_t, castTrait0>(vreg1, vreg0, preg0);141 AscendC::MicroAPI::Cast<float, bfloat16_t, castTrait0>(vreg1, vreg0, preg0);
142- AscendC::MicroAPI::DataCopy<half, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(142+ AscendC::MicroAPI::LoadAlign<half, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(
143 vreg3, bufferIn1Addr + i * (AscendC::VECTOR_REG_WIDTH / 4));143 vreg3, bufferIn1Addr + i * (AscendC::VECTOR_REG_WIDTH / 4));
144 AscendC::MicroAPI::Cast<float, half, castTrait0>(vreg4, vreg3, preg0);144 AscendC::MicroAPI::Cast<float, half, castTrait0>(vreg4, vreg3, preg0);
145 AscendC::MicroAPI::Muls<float, float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(145 AscendC::MicroAPI::Muls<float, float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(
@@ -153,7 +153,7 @@ private:
153 AscendC::MicroAPI::Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg11, vreg2, vreg10, preg0);153 AscendC::MicroAPI::Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg11, vreg2, vreg10, preg0);
154 AscendC::MicroAPI::Mul<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg12, vreg8, vreg11, preg0);154 AscendC::MicroAPI::Mul<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg12, vreg8, vreg11, preg0);
155 AscendC::MicroAPI::Mul<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg13, vreg1, vreg12, preg0);155 AscendC::MicroAPI::Mul<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg13, vreg1, vreg12, preg0);
156- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::StoreDist::DIST_NORM_B32>(156+ AscendC::MicroAPI::StoreAlign<float, AscendC::MicroAPI::StoreDist::DIST_NORM_B32>(
157 bufferOut0Addr + i * (AscendC::VECTOR_REG_WIDTH / 4), vreg13, preg0);157 bufferOut0Addr + i * (AscendC::VECTOR_REG_WIDTH / 4), vreg13, preg0);
158 }158 }
159 }159 }
Mactivation/silu_grad/op_kernel/arch35/silu_grad_dtype_comb_3_nddma_with_loops.h+6-6
@@ -129,15 +129,15 @@ private:
129 uint16_t vfLoopNum = (ubSplitSize * tilingDataPtr_->outputStrides[tilingDataPtr_->ubSplitAxis] +129 uint16_t vfLoopNum = (ubSplitSize * tilingDataPtr_->outputStrides[tilingDataPtr_->ubSplitAxis] +
130 (AscendC::VECTOR_REG_WIDTH / 4) - 1) /130 (AscendC::VECTOR_REG_WIDTH / 4) - 1) /
131 (AscendC::VECTOR_REG_WIDTH / 4);131 (AscendC::VECTOR_REG_WIDTH / 4);
132- __local_mem__ bfloat16_t* bufferIn0Addr = (__local_mem__ bfloat16_t*)bufferIn0_.GetPhyAddr();132+ __ubuf__ bfloat16_t* bufferIn0Addr = (__ubuf__ bfloat16_t*)bufferIn0_.GetPhyAddr();
133- __local_mem__ float* bufferIn1Addr = (__local_mem__ float*)bufferIn1_.GetPhyAddr();133+ __ubuf__ float* bufferIn1Addr = (__ubuf__ float*)bufferIn1_.GetPhyAddr();
134- __local_mem__ float* bufferOut0Addr = (__local_mem__ float*)bufferOut0_.GetPhyAddr();134+ __ubuf__ float* bufferOut0Addr = (__ubuf__ float*)bufferOut0_.GetPhyAddr();
135 for (uint16_t i = 0; i < vfLoopNum; i++) {135 for (uint16_t i = 0; i < vfLoopNum; i++) {
136 preg0 = AscendC::MicroAPI::UpdateMask<float>(size);136 preg0 = AscendC::MicroAPI::UpdateMask<float>(size);
137- AscendC::MicroAPI::DataCopy<bfloat16_t, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(137+ AscendC::MicroAPI::LoadAlign<bfloat16_t, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(
138 vreg0, bufferIn0Addr + i * (AscendC::VECTOR_REG_WIDTH / 4));138 vreg0, bufferIn0Addr + i * (AscendC::VECTOR_REG_WIDTH / 4));
139 AscendC::MicroAPI::Cast<float, bfloat16_t, castTrait0>(vreg1, vreg0, preg0);139 AscendC::MicroAPI::Cast<float, bfloat16_t, castTrait0>(vreg1, vreg0, preg0);
140- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(140+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(
141 vreg3, bufferIn1Addr + i * (AscendC::VECTOR_REG_WIDTH / 4));141 vreg3, bufferIn1Addr + i * (AscendC::VECTOR_REG_WIDTH / 4));
142 AscendC::MicroAPI::Muls<float, float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(142 AscendC::MicroAPI::Muls<float, float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(
143 vreg4, vreg3, static_cast<float>(-1), preg0);143 vreg4, vreg3, static_cast<float>(-1), preg0);
@@ -150,7 +150,7 @@ private:
150 AscendC::MicroAPI::Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg10, vreg2, vreg9, preg0);150 AscendC::MicroAPI::Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg10, vreg2, vreg9, preg0);
151 AscendC::MicroAPI::Mul<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg11, vreg7, vreg10, preg0);151 AscendC::MicroAPI::Mul<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg11, vreg7, vreg10, preg0);
152 AscendC::MicroAPI::Mul<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg12, vreg1, vreg11, preg0);152 AscendC::MicroAPI::Mul<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg12, vreg1, vreg11, preg0);
153- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::StoreDist::DIST_NORM_B32>(153+ AscendC::MicroAPI::StoreAlign<float, AscendC::MicroAPI::StoreDist::DIST_NORM_B32>(
154 bufferOut0Addr + i * (AscendC::VECTOR_REG_WIDTH / 4), vreg12, preg0);154 bufferOut0Addr + i * (AscendC::VECTOR_REG_WIDTH / 4), vreg12, preg0);
155 }155 }
156 }156 }
Mactivation/silu_grad/op_kernel/arch35/silu_grad_dtype_comb_3_nddma_without_loops.h+6-6
@@ -129,15 +129,15 @@ private:
129 uint16_t vfLoopNum = (ubSplitSize * tilingDataPtr_->outputStrides[tilingDataPtr_->ubSplitAxis] +129 uint16_t vfLoopNum = (ubSplitSize * tilingDataPtr_->outputStrides[tilingDataPtr_->ubSplitAxis] +
130 (AscendC::VECTOR_REG_WIDTH / 4) - 1) /130 (AscendC::VECTOR_REG_WIDTH / 4) - 1) /
131 (AscendC::VECTOR_REG_WIDTH / 4);131 (AscendC::VECTOR_REG_WIDTH / 4);
132- __local_mem__ bfloat16_t* bufferIn0Addr = (__local_mem__ bfloat16_t*)bufferIn0_.GetPhyAddr();132+ __ubuf__ bfloat16_t* bufferIn0Addr = (__ubuf__ bfloat16_t*)bufferIn0_.GetPhyAddr();
133- __local_mem__ float* bufferIn1Addr = (__local_mem__ float*)bufferIn1_.GetPhyAddr();133+ __ubuf__ float* bufferIn1Addr = (__ubuf__ float*)bufferIn1_.GetPhyAddr();
134- __local_mem__ float* bufferOut0Addr = (__local_mem__ float*)bufferOut0_.GetPhyAddr();134+ __ubuf__ float* bufferOut0Addr = (__ubuf__ float*)bufferOut0_.GetPhyAddr();
135 for (uint16_t i = 0; i < vfLoopNum; i++) {135 for (uint16_t i = 0; i < vfLoopNum; i++) {
136 preg0 = AscendC::MicroAPI::UpdateMask<float>(size);136 preg0 = AscendC::MicroAPI::UpdateMask<float>(size);
137- AscendC::MicroAPI::DataCopy<bfloat16_t, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(137+ AscendC::MicroAPI::LoadAlign<bfloat16_t, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(
138 vreg0, bufferIn0Addr + i * (AscendC::VECTOR_REG_WIDTH / 4));138 vreg0, bufferIn0Addr + i * (AscendC::VECTOR_REG_WIDTH / 4));
139 AscendC::MicroAPI::Cast<float, bfloat16_t, castTrait0>(vreg1, vreg0, preg0);139 AscendC::MicroAPI::Cast<float, bfloat16_t, castTrait0>(vreg1, vreg0, preg0);
140- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(140+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(
141 vreg3, bufferIn1Addr + i * (AscendC::VECTOR_REG_WIDTH / 4));141 vreg3, bufferIn1Addr + i * (AscendC::VECTOR_REG_WIDTH / 4));
142 AscendC::MicroAPI::Muls<float, float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(142 AscendC::MicroAPI::Muls<float, float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(
143 vreg4, vreg3, static_cast<float>(-1), preg0);143 vreg4, vreg3, static_cast<float>(-1), preg0);
@@ -150,7 +150,7 @@ private:
150 AscendC::MicroAPI::Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg10, vreg2, vreg9, preg0);150 AscendC::MicroAPI::Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg10, vreg2, vreg9, preg0);
151 AscendC::MicroAPI::Mul<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg11, vreg7, vreg10, preg0);151 AscendC::MicroAPI::Mul<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg11, vreg7, vreg10, preg0);
152 AscendC::MicroAPI::Mul<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg12, vreg1, vreg11, preg0);152 AscendC::MicroAPI::Mul<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg12, vreg1, vreg11, preg0);
153- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::StoreDist::DIST_NORM_B32>(153+ AscendC::MicroAPI::StoreAlign<float, AscendC::MicroAPI::StoreDist::DIST_NORM_B32>(
154 bufferOut0Addr + i * (AscendC::VECTOR_REG_WIDTH / 4), vreg12, preg0);154 bufferOut0Addr + i * (AscendC::VECTOR_REG_WIDTH / 4), vreg12, preg0);
155 }155 }
156 }156 }
Mactivation/silu_grad/op_kernel/arch35/silu_grad_dtype_comb_4_nddma_with_loops.h+6-6
@@ -129,14 +129,14 @@ private:
129 uint16_t vfLoopNum = (ubSplitSize * tilingDataPtr_->outputStrides[tilingDataPtr_->ubSplitAxis] +129 uint16_t vfLoopNum = (ubSplitSize * tilingDataPtr_->outputStrides[tilingDataPtr_->ubSplitAxis] +
130 (AscendC::VECTOR_REG_WIDTH / 4) - 1) /130 (AscendC::VECTOR_REG_WIDTH / 4) - 1) /
131 (AscendC::VECTOR_REG_WIDTH / 4);131 (AscendC::VECTOR_REG_WIDTH / 4);
132- __local_mem__ float* bufferIn0Addr = (__local_mem__ float*)bufferIn0_.GetPhyAddr();132+ __ubuf__ float* bufferIn0Addr = (__ubuf__ float*)bufferIn0_.GetPhyAddr();
133- __local_mem__ float* bufferOut0Addr = (__local_mem__ float*)bufferOut0_.GetPhyAddr();133+ __ubuf__ float* bufferOut0Addr = (__ubuf__ float*)bufferOut0_.GetPhyAddr();
134- __local_mem__ half* bufferIn1Addr = (__local_mem__ half*)bufferIn1_.GetPhyAddr();134+ __ubuf__ half* bufferIn1Addr = (__ubuf__ half*)bufferIn1_.GetPhyAddr();
135 for (uint16_t i = 0; i < vfLoopNum; i++) {135 for (uint16_t i = 0; i < vfLoopNum; i++) {
136 preg0 = AscendC::MicroAPI::UpdateMask<float>(size);136 preg0 = AscendC::MicroAPI::UpdateMask<float>(size);
137- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(137+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(
138 vreg0, bufferIn0Addr + i * (AscendC::VECTOR_REG_WIDTH / 4));138 vreg0, bufferIn0Addr + i * (AscendC::VECTOR_REG_WIDTH / 4));
139- AscendC::MicroAPI::DataCopy<half, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(139+ AscendC::MicroAPI::LoadAlign<half, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(
140 vreg2, bufferIn1Addr + i * (AscendC::VECTOR_REG_WIDTH / 4));140 vreg2, bufferIn1Addr + i * (AscendC::VECTOR_REG_WIDTH / 4));
141 AscendC::MicroAPI::Cast<float, half, castTrait0>(vreg3, vreg2, preg0);141 AscendC::MicroAPI::Cast<float, half, castTrait0>(vreg3, vreg2, preg0);
142 AscendC::MicroAPI::Muls<float, float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(142 AscendC::MicroAPI::Muls<float, float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(
@@ -150,7 +150,7 @@ private:
150 AscendC::MicroAPI::Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg10, vreg1, vreg9, preg0);150 AscendC::MicroAPI::Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg10, vreg1, vreg9, preg0);
151 AscendC::MicroAPI::Mul<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg11, vreg7, vreg10, preg0);151 AscendC::MicroAPI::Mul<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg11, vreg7, vreg10, preg0);
152 AscendC::MicroAPI::Mul<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg12, vreg0, vreg11, preg0);152 AscendC::MicroAPI::Mul<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg12, vreg0, vreg11, preg0);
153- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::StoreDist::DIST_NORM_B32>(153+ AscendC::MicroAPI::StoreAlign<float, AscendC::MicroAPI::StoreDist::DIST_NORM_B32>(
154 bufferOut0Addr + i * (AscendC::VECTOR_REG_WIDTH / 4), vreg12, preg0);154 bufferOut0Addr + i * (AscendC::VECTOR_REG_WIDTH / 4), vreg12, preg0);
155 }155 }
156 }156 }
Mactivation/silu_grad/op_kernel/arch35/silu_grad_dtype_comb_4_nddma_without_loops.h+6-6
@@ -129,14 +129,14 @@ private:
129 uint16_t vfLoopNum = (ubSplitSize * tilingDataPtr_->outputStrides[tilingDataPtr_->ubSplitAxis] +129 uint16_t vfLoopNum = (ubSplitSize * tilingDataPtr_->outputStrides[tilingDataPtr_->ubSplitAxis] +
130 (AscendC::VECTOR_REG_WIDTH / 4) - 1) /130 (AscendC::VECTOR_REG_WIDTH / 4) - 1) /
131 (AscendC::VECTOR_REG_WIDTH / 4);131 (AscendC::VECTOR_REG_WIDTH / 4);
132- __local_mem__ float* bufferIn0Addr = (__local_mem__ float*)bufferIn0_.GetPhyAddr();132+ __ubuf__ float* bufferIn0Addr = (__ubuf__ float*)bufferIn0_.GetPhyAddr();
133- __local_mem__ float* bufferOut0Addr = (__local_mem__ float*)bufferOut0_.GetPhyAddr();133+ __ubuf__ float* bufferOut0Addr = (__ubuf__ float*)bufferOut0_.GetPhyAddr();
134- __local_mem__ half* bufferIn1Addr = (__local_mem__ half*)bufferIn1_.GetPhyAddr();134+ __ubuf__ half* bufferIn1Addr = (__ubuf__ half*)bufferIn1_.GetPhyAddr();
135 for (uint16_t i = 0; i < vfLoopNum; i++) {135 for (uint16_t i = 0; i < vfLoopNum; i++) {
136 preg0 = AscendC::MicroAPI::UpdateMask<float>(size);136 preg0 = AscendC::MicroAPI::UpdateMask<float>(size);
137- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(137+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(
138 vreg0, bufferIn0Addr + i * (AscendC::VECTOR_REG_WIDTH / 4));138 vreg0, bufferIn0Addr + i * (AscendC::VECTOR_REG_WIDTH / 4));
139- AscendC::MicroAPI::DataCopy<half, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(139+ AscendC::MicroAPI::LoadAlign<half, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(
140 vreg2, bufferIn1Addr + i * (AscendC::VECTOR_REG_WIDTH / 4));140 vreg2, bufferIn1Addr + i * (AscendC::VECTOR_REG_WIDTH / 4));
141 AscendC::MicroAPI::Cast<float, half, castTrait0>(vreg3, vreg2, preg0);141 AscendC::MicroAPI::Cast<float, half, castTrait0>(vreg3, vreg2, preg0);
142 AscendC::MicroAPI::Muls<float, float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(142 AscendC::MicroAPI::Muls<float, float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(
@@ -150,7 +150,7 @@ private:
150 AscendC::MicroAPI::Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg10, vreg1, vreg9, preg0);150 AscendC::MicroAPI::Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg10, vreg1, vreg9, preg0);
151 AscendC::MicroAPI::Mul<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg11, vreg7, vreg10, preg0);151 AscendC::MicroAPI::Mul<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg11, vreg7, vreg10, preg0);
152 AscendC::MicroAPI::Mul<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg12, vreg0, vreg11, preg0);152 AscendC::MicroAPI::Mul<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg12, vreg0, vreg11, preg0);
153- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::StoreDist::DIST_NORM_B32>(153+ AscendC::MicroAPI::StoreAlign<float, AscendC::MicroAPI::StoreDist::DIST_NORM_B32>(
154 bufferOut0Addr + i * (AscendC::VECTOR_REG_WIDTH / 4), vreg12, preg0);154 bufferOut0Addr + i * (AscendC::VECTOR_REG_WIDTH / 4), vreg12, preg0);
155 }155 }
156 }156 }
Mactivation/silu_grad/op_kernel/arch35/silu_grad_dtype_comb_5_nddma_with_loops.h+6-6
@@ -129,14 +129,14 @@ private:
129 uint16_t vfLoopNum = (ubSplitSize * tilingDataPtr_->outputStrides[tilingDataPtr_->ubSplitAxis] +129 uint16_t vfLoopNum = (ubSplitSize * tilingDataPtr_->outputStrides[tilingDataPtr_->ubSplitAxis] +
130 (AscendC::VECTOR_REG_WIDTH / 4) - 1) /130 (AscendC::VECTOR_REG_WIDTH / 4) - 1) /
131 (AscendC::VECTOR_REG_WIDTH / 4);131 (AscendC::VECTOR_REG_WIDTH / 4);
132- __local_mem__ bfloat16_t* bufferIn1Addr = (__local_mem__ bfloat16_t*)bufferIn1_.GetPhyAddr();132+ __ubuf__ bfloat16_t* bufferIn1Addr = (__ubuf__ bfloat16_t*)bufferIn1_.GetPhyAddr();
133- __local_mem__ float* bufferIn0Addr = (__local_mem__ float*)bufferIn0_.GetPhyAddr();133+ __ubuf__ float* bufferIn0Addr = (__ubuf__ float*)bufferIn0_.GetPhyAddr();
134- __local_mem__ float* bufferOut0Addr = (__local_mem__ float*)bufferOut0_.GetPhyAddr();134+ __ubuf__ float* bufferOut0Addr = (__ubuf__ float*)bufferOut0_.GetPhyAddr();
135 for (uint16_t i = 0; i < vfLoopNum; i++) {135 for (uint16_t i = 0; i < vfLoopNum; i++) {
136 preg0 = AscendC::MicroAPI::UpdateMask<float>(size);136 preg0 = AscendC::MicroAPI::UpdateMask<float>(size);
137- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(137+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(
138 vreg0, bufferIn0Addr + i * (AscendC::VECTOR_REG_WIDTH / 4));138 vreg0, bufferIn0Addr + i * (AscendC::VECTOR_REG_WIDTH / 4));
139- AscendC::MicroAPI::DataCopy<bfloat16_t, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(139+ AscendC::MicroAPI::LoadAlign<bfloat16_t, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(
140 vreg2, bufferIn1Addr + i * (AscendC::VECTOR_REG_WIDTH / 4));140 vreg2, bufferIn1Addr + i * (AscendC::VECTOR_REG_WIDTH / 4));
141 AscendC::MicroAPI::Cast<float, bfloat16_t, castTrait0>(vreg3, vreg2, preg0);141 AscendC::MicroAPI::Cast<float, bfloat16_t, castTrait0>(vreg3, vreg2, preg0);
142 AscendC::MicroAPI::Muls<float, float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(142 AscendC::MicroAPI::Muls<float, float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(
@@ -150,7 +150,7 @@ private:
150 AscendC::MicroAPI::Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg10, vreg1, vreg9, preg0);150 AscendC::MicroAPI::Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg10, vreg1, vreg9, preg0);
151 AscendC::MicroAPI::Mul<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg11, vreg7, vreg10, preg0);151 AscendC::MicroAPI::Mul<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg11, vreg7, vreg10, preg0);
152 AscendC::MicroAPI::Mul<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg12, vreg0, vreg11, preg0);152 AscendC::MicroAPI::Mul<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg12, vreg0, vreg11, preg0);
153- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::StoreDist::DIST_NORM_B32>(153+ AscendC::MicroAPI::StoreAlign<float, AscendC::MicroAPI::StoreDist::DIST_NORM_B32>(
154 bufferOut0Addr + i * (AscendC::VECTOR_REG_WIDTH / 4), vreg12, preg0);154 bufferOut0Addr + i * (AscendC::VECTOR_REG_WIDTH / 4), vreg12, preg0);
155 }155 }
156 }156 }
Mactivation/silu_grad/op_kernel/arch35/silu_grad_dtype_comb_5_nddma_without_loops.h+6-6
@@ -129,14 +129,14 @@ private:
129 uint16_t vfLoopNum = (ubSplitSize * tilingDataPtr_->outputStrides[tilingDataPtr_->ubSplitAxis] +129 uint16_t vfLoopNum = (ubSplitSize * tilingDataPtr_->outputStrides[tilingDataPtr_->ubSplitAxis] +
130 (AscendC::VECTOR_REG_WIDTH / 4) - 1) /130 (AscendC::VECTOR_REG_WIDTH / 4) - 1) /
131 (AscendC::VECTOR_REG_WIDTH / 4);131 (AscendC::VECTOR_REG_WIDTH / 4);
132- __local_mem__ bfloat16_t* bufferIn1Addr = (__local_mem__ bfloat16_t*)bufferIn1_.GetPhyAddr();132+ __ubuf__ bfloat16_t* bufferIn1Addr = (__ubuf__ bfloat16_t*)bufferIn1_.GetPhyAddr();
133- __local_mem__ float* bufferIn0Addr = (__local_mem__ float*)bufferIn0_.GetPhyAddr();133+ __ubuf__ float* bufferIn0Addr = (__ubuf__ float*)bufferIn0_.GetPhyAddr();
134- __local_mem__ float* bufferOut0Addr = (__local_mem__ float*)bufferOut0_.GetPhyAddr();134+ __ubuf__ float* bufferOut0Addr = (__ubuf__ float*)bufferOut0_.GetPhyAddr();
135 for (uint16_t i = 0; i < vfLoopNum; i++) {135 for (uint16_t i = 0; i < vfLoopNum; i++) {
136 preg0 = AscendC::MicroAPI::UpdateMask<float>(size);136 preg0 = AscendC::MicroAPI::UpdateMask<float>(size);
137- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(137+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(
138 vreg0, bufferIn0Addr + i * (AscendC::VECTOR_REG_WIDTH / 4));138 vreg0, bufferIn0Addr + i * (AscendC::VECTOR_REG_WIDTH / 4));
139- AscendC::MicroAPI::DataCopy<bfloat16_t, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(139+ AscendC::MicroAPI::LoadAlign<bfloat16_t, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(
140 vreg2, bufferIn1Addr + i * (AscendC::VECTOR_REG_WIDTH / 4));140 vreg2, bufferIn1Addr + i * (AscendC::VECTOR_REG_WIDTH / 4));
141 AscendC::MicroAPI::Cast<float, bfloat16_t, castTrait0>(vreg3, vreg2, preg0);141 AscendC::MicroAPI::Cast<float, bfloat16_t, castTrait0>(vreg3, vreg2, preg0);
142 AscendC::MicroAPI::Muls<float, float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(142 AscendC::MicroAPI::Muls<float, float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(
@@ -150,7 +150,7 @@ private:
150 AscendC::MicroAPI::Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg10, vreg1, vreg9, preg0);150 AscendC::MicroAPI::Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg10, vreg1, vreg9, preg0);
151 AscendC::MicroAPI::Mul<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg11, vreg7, vreg10, preg0);151 AscendC::MicroAPI::Mul<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg11, vreg7, vreg10, preg0);
152 AscendC::MicroAPI::Mul<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg12, vreg0, vreg11, preg0);152 AscendC::MicroAPI::Mul<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg12, vreg0, vreg11, preg0);
153- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::StoreDist::DIST_NORM_B32>(153+ AscendC::MicroAPI::StoreAlign<float, AscendC::MicroAPI::StoreDist::DIST_NORM_B32>(
154 bufferOut0Addr + i * (AscendC::VECTOR_REG_WIDTH / 4), vreg12, preg0);154 bufferOut0Addr + i * (AscendC::VECTOR_REG_WIDTH / 4), vreg12, preg0);
155 }155 }
156 }156 }
Mactivation/silu_grad/op_kernel/arch35/silu_grad_f16_nddma_with_loops.h+18-18
@@ -114,13 +114,13 @@ private:
114 uint16_t vfLoopNum = (ubSplitSize * tilingDataPtr_->outputStrides[tilingDataPtr_->ubSplitAxis] +114 uint16_t vfLoopNum = (ubSplitSize * tilingDataPtr_->outputStrides[tilingDataPtr_->ubSplitAxis] +
115 (AscendC::VECTOR_REG_WIDTH / 4) - 1) /115 (AscendC::VECTOR_REG_WIDTH / 4) - 1) /
116 (AscendC::VECTOR_REG_WIDTH / 4);116 (AscendC::VECTOR_REG_WIDTH / 4);
117- __local_mem__ float* bufferTmp0Addr = (__local_mem__ float*)bufferTmp0_.GetPhyAddr();117+ __ubuf__ float* bufferTmp0Addr = (__ubuf__ float*)bufferTmp0_.GetPhyAddr();
118- __local_mem__ float* bufferTmp1Addr = (__local_mem__ float*)bufferTmp1_.GetPhyAddr();118+ __ubuf__ float* bufferTmp1Addr = (__ubuf__ float*)bufferTmp1_.GetPhyAddr();
119- __local_mem__ float* bufferTmp2Addr = (__local_mem__ float*)bufferTmp2_.GetPhyAddr();119+ __ubuf__ float* bufferTmp2Addr = (__ubuf__ float*)bufferTmp2_.GetPhyAddr();
120- __local_mem__ half* bufferIn0Addr = (__local_mem__ half*)bufferIn0_.GetPhyAddr();120+ __ubuf__ half* bufferIn0Addr = (__ubuf__ half*)bufferIn0_.GetPhyAddr();
121 for (uint16_t i = 0; i < vfLoopNum; i++) {121 for (uint16_t i = 0; i < vfLoopNum; i++) {
122 preg0 = AscendC::MicroAPI::UpdateMask<float>(size);122 preg0 = AscendC::MicroAPI::UpdateMask<float>(size);
123- AscendC::MicroAPI::DataCopy<half, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(123+ AscendC::MicroAPI::LoadAlign<half, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(
124 vreg3, bufferIn0Addr + i * (AscendC::VECTOR_REG_WIDTH / 4));124 vreg3, bufferIn0Addr + i * (AscendC::VECTOR_REG_WIDTH / 4));
125 AscendC::MicroAPI::Cast<float, half, castTrait0>(vreg4, vreg3, preg0);125 AscendC::MicroAPI::Cast<float, half, castTrait0>(vreg4, vreg3, preg0);
126 AscendC::MicroAPI::Muls<float, float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(126 AscendC::MicroAPI::Muls<float, float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(
@@ -129,11 +129,11 @@ private:
129 AscendC::MicroAPI::Adds<float, float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(129 AscendC::MicroAPI::Adds<float, float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(
130 vreg7, vreg6, static_cast<float>(1), preg0);130 vreg7, vreg6, static_cast<float>(1), preg0);
131 AscendC::MicroAPI::Div<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg8, vreg2, vreg7, preg0);131 AscendC::MicroAPI::Div<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg8, vreg2, vreg7, preg0);
132- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::StoreDist::DIST_NORM_B32>(132+ AscendC::MicroAPI::StoreAlign<float, AscendC::MicroAPI::StoreDist::DIST_NORM_B32>(
133 bufferTmp0Addr + i * (AscendC::VECTOR_REG_WIDTH / 4), vreg8, preg0);133 bufferTmp0Addr + i * (AscendC::VECTOR_REG_WIDTH / 4), vreg8, preg0);
134- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::StoreDist::DIST_NORM_B32>(134+ AscendC::MicroAPI::StoreAlign<float, AscendC::MicroAPI::StoreDist::DIST_NORM_B32>(
135 bufferTmp1Addr + i * (AscendC::VECTOR_REG_WIDTH / 4), vreg2, preg0);135 bufferTmp1Addr + i * (AscendC::VECTOR_REG_WIDTH / 4), vreg2, preg0);
136- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::StoreDist::DIST_NORM_B32>(136+ AscendC::MicroAPI::StoreAlign<float, AscendC::MicroAPI::StoreDist::DIST_NORM_B32>(
137 bufferTmp2Addr + i * (AscendC::VECTOR_REG_WIDTH / 4), vreg4, preg0);137 bufferTmp2Addr + i * (AscendC::VECTOR_REG_WIDTH / 4), vreg4, preg0);
138 }138 }
139 }139 }
@@ -177,29 +177,29 @@ private:
177 uint16_t vfLoopNum = (ubSplitSize * tilingDataPtr_->outputStrides[tilingDataPtr_->ubSplitAxis] +177 uint16_t vfLoopNum = (ubSplitSize * tilingDataPtr_->outputStrides[tilingDataPtr_->ubSplitAxis] +
178 (AscendC::VECTOR_REG_WIDTH / 4) - 1) /178 (AscendC::VECTOR_REG_WIDTH / 4) - 1) /
179 (AscendC::VECTOR_REG_WIDTH / 4);179 (AscendC::VECTOR_REG_WIDTH / 4);
180- __local_mem__ float* bufferTmp0Addr = (__local_mem__ float*)bufferTmp0_.GetPhyAddr();180+ __ubuf__ float* bufferTmp0Addr = (__ubuf__ float*)bufferTmp0_.GetPhyAddr();
181- __local_mem__ float* bufferTmp1Addr = (__local_mem__ float*)bufferTmp1_.GetPhyAddr();181+ __ubuf__ float* bufferTmp1Addr = (__ubuf__ float*)bufferTmp1_.GetPhyAddr();
182- __local_mem__ float* bufferTmp2Addr = (__local_mem__ float*)bufferTmp2_.GetPhyAddr();182+ __ubuf__ float* bufferTmp2Addr = (__ubuf__ float*)bufferTmp2_.GetPhyAddr();
183- __local_mem__ half* bufferIn1Addr = (__local_mem__ half*)bufferIn1_.GetPhyAddr();183+ __ubuf__ half* bufferIn1Addr = (__ubuf__ half*)bufferIn1_.GetPhyAddr();
184- __local_mem__ half* bufferOut0Addr = (__local_mem__ half*)bufferOut0_.GetPhyAddr();184+ __ubuf__ half* bufferOut0Addr = (__ubuf__ half*)bufferOut0_.GetPhyAddr();
185 for (uint16_t i = 0; i < vfLoopNum; i++) {185 for (uint16_t i = 0; i < vfLoopNum; i++) {
186 preg0 = AscendC::MicroAPI::UpdateMask<float>(size);186 preg0 = AscendC::MicroAPI::UpdateMask<float>(size);
187- AscendC::MicroAPI::DataCopy<half, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(187+ AscendC::MicroAPI::LoadAlign<half, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(
188 vreg0, bufferIn1Addr + i * (AscendC::VECTOR_REG_WIDTH / 4));188 vreg0, bufferIn1Addr + i * (AscendC::VECTOR_REG_WIDTH / 4));
189 AscendC::MicroAPI::Cast<float, half, castTrait0>(vreg1, vreg0, preg0);189 AscendC::MicroAPI::Cast<float, half, castTrait0>(vreg1, vreg0, preg0);
190- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(190+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(
191 vreg8, bufferTmp0Addr + i * (AscendC::VECTOR_REG_WIDTH / 4));191 vreg8, bufferTmp0Addr + i * (AscendC::VECTOR_REG_WIDTH / 4));
192- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(192+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(
193 vreg2, bufferTmp1Addr + i * (AscendC::VECTOR_REG_WIDTH / 4));193 vreg2, bufferTmp1Addr + i * (AscendC::VECTOR_REG_WIDTH / 4));
194 AscendC::MicroAPI::Sub<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg9, vreg2, vreg8, preg0);194 AscendC::MicroAPI::Sub<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg9, vreg2, vreg8, preg0);
195- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(195+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(
196 vreg4, bufferTmp2Addr + i * (AscendC::VECTOR_REG_WIDTH / 4));196 vreg4, bufferTmp2Addr + i * (AscendC::VECTOR_REG_WIDTH / 4));
197 AscendC::MicroAPI::Mul<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg10, vreg9, vreg4, preg0);197 AscendC::MicroAPI::Mul<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg10, vreg9, vreg4, preg0);
198 AscendC::MicroAPI::Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg11, vreg2, vreg10, preg0);198 AscendC::MicroAPI::Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg11, vreg2, vreg10, preg0);
199 AscendC::MicroAPI::Mul<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg12, vreg8, vreg11, preg0);199 AscendC::MicroAPI::Mul<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg12, vreg8, vreg11, preg0);
200 AscendC::MicroAPI::Mul<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg13, vreg1, vreg12, preg0);200 AscendC::MicroAPI::Mul<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg13, vreg1, vreg12, preg0);
201 AscendC::MicroAPI::Cast<half, float, castTrait1>(vreg14, vreg13, preg0);201 AscendC::MicroAPI::Cast<half, float, castTrait1>(vreg14, vreg13, preg0);
202- AscendC::MicroAPI::DataCopy<half, AscendC::MicroAPI::StoreDist::DIST_PACK_B32>(202+ AscendC::MicroAPI::StoreAlign<half, AscendC::MicroAPI::StoreDist::DIST_PACK_B32>(
203 bufferOut0Addr + i * (AscendC::VECTOR_REG_WIDTH / 4), vreg14, preg0);203 bufferOut0Addr + i * (AscendC::VECTOR_REG_WIDTH / 4), vreg14, preg0);
204 }204 }
205 }205 }
Mactivation/silu_grad/op_kernel/arch35/silu_grad_f16_nddma_without_loops.h+18-18
@@ -114,13 +114,13 @@ private:
114 uint16_t vfLoopNum = (ubSplitSize * tilingDataPtr_->outputStrides[tilingDataPtr_->ubSplitAxis] +114 uint16_t vfLoopNum = (ubSplitSize * tilingDataPtr_->outputStrides[tilingDataPtr_->ubSplitAxis] +
115 (AscendC::VECTOR_REG_WIDTH / 4) - 1) /115 (AscendC::VECTOR_REG_WIDTH / 4) - 1) /
116 (AscendC::VECTOR_REG_WIDTH / 4);116 (AscendC::VECTOR_REG_WIDTH / 4);
117- __local_mem__ float* bufferTmp0Addr = (__local_mem__ float*)bufferTmp0_.GetPhyAddr();117+ __ubuf__ float* bufferTmp0Addr = (__ubuf__ float*)bufferTmp0_.GetPhyAddr();
118- __local_mem__ float* bufferTmp1Addr = (__local_mem__ float*)bufferTmp1_.GetPhyAddr();118+ __ubuf__ float* bufferTmp1Addr = (__ubuf__ float*)bufferTmp1_.GetPhyAddr();
119- __local_mem__ float* bufferTmp2Addr = (__local_mem__ float*)bufferTmp2_.GetPhyAddr();119+ __ubuf__ float* bufferTmp2Addr = (__ubuf__ float*)bufferTmp2_.GetPhyAddr();
120- __local_mem__ half* bufferIn0Addr = (__local_mem__ half*)bufferIn0_.GetPhyAddr();120+ __ubuf__ half* bufferIn0Addr = (__ubuf__ half*)bufferIn0_.GetPhyAddr();
121 for (uint16_t i = 0; i < vfLoopNum; i++) {121 for (uint16_t i = 0; i < vfLoopNum; i++) {
122 preg0 = AscendC::MicroAPI::UpdateMask<float>(size);122 preg0 = AscendC::MicroAPI::UpdateMask<float>(size);
123- AscendC::MicroAPI::DataCopy<half, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(123+ AscendC::MicroAPI::LoadAlign<half, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(
124 vreg3, bufferIn0Addr + i * (AscendC::VECTOR_REG_WIDTH / 4));124 vreg3, bufferIn0Addr + i * (AscendC::VECTOR_REG_WIDTH / 4));
125 AscendC::MicroAPI::Cast<float, half, castTrait0>(vreg4, vreg3, preg0);125 AscendC::MicroAPI::Cast<float, half, castTrait0>(vreg4, vreg3, preg0);
126 AscendC::MicroAPI::Muls<float, float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(126 AscendC::MicroAPI::Muls<float, float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(
@@ -129,11 +129,11 @@ private:
129 AscendC::MicroAPI::Adds<float, float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(129 AscendC::MicroAPI::Adds<float, float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(
130 vreg7, vreg6, static_cast<float>(1), preg0);130 vreg7, vreg6, static_cast<float>(1), preg0);
131 AscendC::MicroAPI::Div<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg8, vreg2, vreg7, preg0);131 AscendC::MicroAPI::Div<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg8, vreg2, vreg7, preg0);
132- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::StoreDist::DIST_NORM_B32>(132+ AscendC::MicroAPI::StoreAlign<float, AscendC::MicroAPI::StoreDist::DIST_NORM_B32>(
133 bufferTmp0Addr + i * (AscendC::VECTOR_REG_WIDTH / 4), vreg8, preg0);133 bufferTmp0Addr + i * (AscendC::VECTOR_REG_WIDTH / 4), vreg8, preg0);
134- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::StoreDist::DIST_NORM_B32>(134+ AscendC::MicroAPI::StoreAlign<float, AscendC::MicroAPI::StoreDist::DIST_NORM_B32>(
135 bufferTmp1Addr + i * (AscendC::VECTOR_REG_WIDTH / 4), vreg2, preg0);135 bufferTmp1Addr + i * (AscendC::VECTOR_REG_WIDTH / 4), vreg2, preg0);
136- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::StoreDist::DIST_NORM_B32>(136+ AscendC::MicroAPI::StoreAlign<float, AscendC::MicroAPI::StoreDist::DIST_NORM_B32>(
137 bufferTmp2Addr + i * (AscendC::VECTOR_REG_WIDTH / 4), vreg4, preg0);137 bufferTmp2Addr + i * (AscendC::VECTOR_REG_WIDTH / 4), vreg4, preg0);
138 }138 }
139 }139 }
@@ -177,29 +177,29 @@ private:
177 uint16_t vfLoopNum = (ubSplitSize * tilingDataPtr_->outputStrides[tilingDataPtr_->ubSplitAxis] +177 uint16_t vfLoopNum = (ubSplitSize * tilingDataPtr_->outputStrides[tilingDataPtr_->ubSplitAxis] +
178 (AscendC::VECTOR_REG_WIDTH / 4) - 1) /178 (AscendC::VECTOR_REG_WIDTH / 4) - 1) /
179 (AscendC::VECTOR_REG_WIDTH / 4);179 (AscendC::VECTOR_REG_WIDTH / 4);
180- __local_mem__ float* bufferTmp0Addr = (__local_mem__ float*)bufferTmp0_.GetPhyAddr();180+ __ubuf__ float* bufferTmp0Addr = (__ubuf__ float*)bufferTmp0_.GetPhyAddr();
181- __local_mem__ float* bufferTmp1Addr = (__local_mem__ float*)bufferTmp1_.GetPhyAddr();181+ __ubuf__ float* bufferTmp1Addr = (__ubuf__ float*)bufferTmp1_.GetPhyAddr();
182- __local_mem__ float* bufferTmp2Addr = (__local_mem__ float*)bufferTmp2_.GetPhyAddr();182+ __ubuf__ float* bufferTmp2Addr = (__ubuf__ float*)bufferTmp2_.GetPhyAddr();
183- __local_mem__ half* bufferIn1Addr = (__local_mem__ half*)bufferIn1_.GetPhyAddr();183+ __ubuf__ half* bufferIn1Addr = (__ubuf__ half*)bufferIn1_.GetPhyAddr();
184- __local_mem__ half* bufferOut0Addr = (__local_mem__ half*)bufferOut0_.GetPhyAddr();184+ __ubuf__ half* bufferOut0Addr = (__ubuf__ half*)bufferOut0_.GetPhyAddr();
185 for (uint16_t i = 0; i < vfLoopNum; i++) {185 for (uint16_t i = 0; i < vfLoopNum; i++) {
186 preg0 = AscendC::MicroAPI::UpdateMask<float>(size);186 preg0 = AscendC::MicroAPI::UpdateMask<float>(size);
187- AscendC::MicroAPI::DataCopy<half, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(187+ AscendC::MicroAPI::LoadAlign<half, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(
188 vreg0, bufferIn1Addr + i * (AscendC::VECTOR_REG_WIDTH / 4));188 vreg0, bufferIn1Addr + i * (AscendC::VECTOR_REG_WIDTH / 4));
189 AscendC::MicroAPI::Cast<float, half, castTrait0>(vreg1, vreg0, preg0);189 AscendC::MicroAPI::Cast<float, half, castTrait0>(vreg1, vreg0, preg0);
190- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(190+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(
191 vreg8, bufferTmp0Addr + i * (AscendC::VECTOR_REG_WIDTH / 4));191 vreg8, bufferTmp0Addr + i * (AscendC::VECTOR_REG_WIDTH / 4));
192- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(192+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(
193 vreg2, bufferTmp1Addr + i * (AscendC::VECTOR_REG_WIDTH / 4));193 vreg2, bufferTmp1Addr + i * (AscendC::VECTOR_REG_WIDTH / 4));
194 AscendC::MicroAPI::Sub<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg9, vreg2, vreg8, preg0);194 AscendC::MicroAPI::Sub<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg9, vreg2, vreg8, preg0);
195- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(195+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(
196 vreg4, bufferTmp2Addr + i * (AscendC::VECTOR_REG_WIDTH / 4));196 vreg4, bufferTmp2Addr + i * (AscendC::VECTOR_REG_WIDTH / 4));
197 AscendC::MicroAPI::Mul<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg10, vreg9, vreg4, preg0);197 AscendC::MicroAPI::Mul<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg10, vreg9, vreg4, preg0);
198 AscendC::MicroAPI::Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg11, vreg2, vreg10, preg0);198 AscendC::MicroAPI::Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg11, vreg2, vreg10, preg0);
199 AscendC::MicroAPI::Mul<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg12, vreg8, vreg11, preg0);199 AscendC::MicroAPI::Mul<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg12, vreg8, vreg11, preg0);
200 AscendC::MicroAPI::Mul<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg13, vreg1, vreg12, preg0);200 AscendC::MicroAPI::Mul<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg13, vreg1, vreg12, preg0);
201 AscendC::MicroAPI::Cast<half, float, castTrait1>(vreg14, vreg13, preg0);201 AscendC::MicroAPI::Cast<half, float, castTrait1>(vreg14, vreg13, preg0);
202- AscendC::MicroAPI::DataCopy<half, AscendC::MicroAPI::StoreDist::DIST_PACK_B32>(202+ AscendC::MicroAPI::StoreAlign<half, AscendC::MicroAPI::StoreDist::DIST_PACK_B32>(
203 bufferOut0Addr + i * (AscendC::VECTOR_REG_WIDTH / 4), vreg14, preg0);203 bufferOut0Addr + i * (AscendC::VECTOR_REG_WIDTH / 4), vreg14, preg0);
204 }204 }
205 }205 }
Mactivation/silu_grad/op_kernel/arch35/silu_grad_f32_nddma_with_loops.h+6-6
@@ -127,14 +127,14 @@ private:
127 uint16_t vfLoopNum = (ubSplitSize * tilingDataPtr_->outputStrides[tilingDataPtr_->ubSplitAxis] +127 uint16_t vfLoopNum = (ubSplitSize * tilingDataPtr_->outputStrides[tilingDataPtr_->ubSplitAxis] +
128 (AscendC::VECTOR_REG_WIDTH / 4) - 1) /128 (AscendC::VECTOR_REG_WIDTH / 4) - 1) /
129 (AscendC::VECTOR_REG_WIDTH / 4);129 (AscendC::VECTOR_REG_WIDTH / 4);
130- __local_mem__ float* bufferIn0Addr = (__local_mem__ float*)bufferIn0_.GetPhyAddr();130+ __ubuf__ float* bufferIn0Addr = (__ubuf__ float*)bufferIn0_.GetPhyAddr();
131- __local_mem__ float* bufferIn1Addr = (__local_mem__ float*)bufferIn1_.GetPhyAddr();131+ __ubuf__ float* bufferIn1Addr = (__ubuf__ float*)bufferIn1_.GetPhyAddr();
132- __local_mem__ float* bufferOut0Addr = (__local_mem__ float*)bufferOut0_.GetPhyAddr();132+ __ubuf__ float* bufferOut0Addr = (__ubuf__ float*)bufferOut0_.GetPhyAddr();
133 for (uint16_t i = 0; i < vfLoopNum; i++) {133 for (uint16_t i = 0; i < vfLoopNum; i++) {
134 preg0 = AscendC::MicroAPI::UpdateMask<float>(size);134 preg0 = AscendC::MicroAPI::UpdateMask<float>(size);
135- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(135+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(
136 vreg0, bufferIn0Addr + i * (AscendC::VECTOR_REG_WIDTH / 4));136 vreg0, bufferIn0Addr + i * (AscendC::VECTOR_REG_WIDTH / 4));
137- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(137+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(
138 vreg2, bufferIn1Addr + i * (AscendC::VECTOR_REG_WIDTH / 4));138 vreg2, bufferIn1Addr + i * (AscendC::VECTOR_REG_WIDTH / 4));
139 AscendC::MicroAPI::Muls<float, float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(139 AscendC::MicroAPI::Muls<float, float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(
140 vreg3, vreg2, static_cast<float>(-1), preg0);140 vreg3, vreg2, static_cast<float>(-1), preg0);
@@ -147,7 +147,7 @@ private:
147 AscendC::MicroAPI::Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg9, vreg1, vreg8, preg0);147 AscendC::MicroAPI::Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg9, vreg1, vreg8, preg0);
148 AscendC::MicroAPI::Mul<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg10, vreg6, vreg9, preg0);148 AscendC::MicroAPI::Mul<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg10, vreg6, vreg9, preg0);
149 AscendC::MicroAPI::Mul<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg11, vreg0, vreg10, preg0);149 AscendC::MicroAPI::Mul<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg11, vreg0, vreg10, preg0);
150- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::StoreDist::DIST_NORM_B32>(150+ AscendC::MicroAPI::StoreAlign<float, AscendC::MicroAPI::StoreDist::DIST_NORM_B32>(
151 bufferOut0Addr + i * (AscendC::VECTOR_REG_WIDTH / 4), vreg11, preg0);151 bufferOut0Addr + i * (AscendC::VECTOR_REG_WIDTH / 4), vreg11, preg0);
152 }152 }
153 }153 }
Mactivation/silu_grad/op_kernel/arch35/silu_grad_f32_nddma_without_loops.h+6-6
@@ -127,14 +127,14 @@ private:
127 uint16_t vfLoopNum = (ubSplitSize * tilingDataPtr_->outputStrides[tilingDataPtr_->ubSplitAxis] +127 uint16_t vfLoopNum = (ubSplitSize * tilingDataPtr_->outputStrides[tilingDataPtr_->ubSplitAxis] +
128 (AscendC::VECTOR_REG_WIDTH / 4) - 1) /128 (AscendC::VECTOR_REG_WIDTH / 4) - 1) /
129 (AscendC::VECTOR_REG_WIDTH / 4);129 (AscendC::VECTOR_REG_WIDTH / 4);
130- __local_mem__ float* bufferIn0Addr = (__local_mem__ float*)bufferIn0_.GetPhyAddr();130+ __ubuf__ float* bufferIn0Addr = (__ubuf__ float*)bufferIn0_.GetPhyAddr();
131- __local_mem__ float* bufferIn1Addr = (__local_mem__ float*)bufferIn1_.GetPhyAddr();131+ __ubuf__ float* bufferIn1Addr = (__ubuf__ float*)bufferIn1_.GetPhyAddr();
132- __local_mem__ float* bufferOut0Addr = (__local_mem__ float*)bufferOut0_.GetPhyAddr();132+ __ubuf__ float* bufferOut0Addr = (__ubuf__ float*)bufferOut0_.GetPhyAddr();
133 for (uint16_t i = 0; i < vfLoopNum; i++) {133 for (uint16_t i = 0; i < vfLoopNum; i++) {
134 preg0 = AscendC::MicroAPI::UpdateMask<float>(size);134 preg0 = AscendC::MicroAPI::UpdateMask<float>(size);
135- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(135+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(
136 vreg0, bufferIn0Addr + i * (AscendC::VECTOR_REG_WIDTH / 4));136 vreg0, bufferIn0Addr + i * (AscendC::VECTOR_REG_WIDTH / 4));
137- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(137+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(
138 vreg2, bufferIn1Addr + i * (AscendC::VECTOR_REG_WIDTH / 4));138 vreg2, bufferIn1Addr + i * (AscendC::VECTOR_REG_WIDTH / 4));
139 AscendC::MicroAPI::Muls<float, float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(139 AscendC::MicroAPI::Muls<float, float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(
140 vreg3, vreg2, static_cast<float>(-1), preg0);140 vreg3, vreg2, static_cast<float>(-1), preg0);
@@ -147,7 +147,7 @@ private:
147 AscendC::MicroAPI::Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg9, vreg1, vreg8, preg0);147 AscendC::MicroAPI::Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg9, vreg1, vreg8, preg0);
148 AscendC::MicroAPI::Mul<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg10, vreg6, vreg9, preg0);148 AscendC::MicroAPI::Mul<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg10, vreg6, vreg9, preg0);
149 AscendC::MicroAPI::Mul<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg11, vreg0, vreg10, preg0);149 AscendC::MicroAPI::Mul<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg11, vreg0, vreg10, preg0);
150- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::StoreDist::DIST_NORM_B32>(150+ AscendC::MicroAPI::StoreAlign<float, AscendC::MicroAPI::StoreDist::DIST_NORM_B32>(
151 bufferOut0Addr + i * (AscendC::VECTOR_REG_WIDTH / 4), vreg11, preg0);151 bufferOut0Addr + i * (AscendC::VECTOR_REG_WIDTH / 4), vreg11, preg0);
152 }152 }
153 }153 }
Mactivation/softmax_grad_ext/op_kernel/arch35/softmax_grad_ext_ar_full_load.h+63-63
@@ -60,10 +60,10 @@ private:
60 60 
61 __aicore__ inline void CopyOutY(int64_t ubA, int64_t offset);61 __aicore__ inline void CopyOutY(int64_t ubA, int64_t offset);
62 62 
63- __aicore__ inline void StoreTensorForDtypeTOut(__local_mem__ T* dst, RegTensor<float>& src, MaskReg& preg,63+ __aicore__ inline void StoreTensorForDtypeTOut(__ubuf__ T* dst, RegTensor<float>& src, MaskReg& preg,
64 uint32_t offset);64 uint32_t offset);
65 65 
66- __aicore__ inline void LoadTensorForDtypeTIn(__local_mem__ T* src, RegTensor<float>& dst, MaskReg& preg,66+ __aicore__ inline void LoadTensorForDtypeTIn(__ubuf__ T* src, RegTensor<float>& dst, MaskReg& preg,
67 uint32_t offset);67 uint32_t offset);
68 68 
69private:69private:
@@ -160,10 +160,10 @@ __aicore__ inline void SoftmaxGradExtAR<T>::NormComputeSmallR(const int64_t aSiz
160 160 
161 uint16_t loopTimes = aSize;161 uint16_t loopTimes = aSize;
162 if (rSize <= VL_FP32) {162 if (rSize <= VL_FP32) {
163- __local_mem__ T* dst = (__local_mem__ T*)dstTensor.GetPhyAddr();163+ __ubuf__ T* dst = (__ubuf__ T*)dstTensor.GetPhyAddr();
164- __local_mem__ T* x0 = (__local_mem__ T*)x0Tensor.GetPhyAddr();164+ __ubuf__ T* x0 = (__ubuf__ T*)x0Tensor.GetPhyAddr();
165- __local_mem__ T* x1 = (__local_mem__ T*)x1Tensor.GetPhyAddr();165+ __ubuf__ T* x1 = (__ubuf__ T*)x1Tensor.GetPhyAddr();
166- __local_mem__ T* x2 = (__local_mem__ T*)x2Tensor.GetPhyAddr();166+ __ubuf__ T* x2 = (__ubuf__ T*)x2Tensor.GetPhyAddr();
167 167 
168 __VEC_SCOPE__168 __VEC_SCOPE__
169 {169 {
@@ -179,7 +179,7 @@ __aicore__ inline void SoftmaxGradExtAR<T>::NormComputeSmallR(const int64_t aSiz
179 LoadTensorForDtypeTIn(x2, reg3, pMask, i * rAligned);179 LoadTensorForDtypeTIn(x2, reg3, pMask, i * rAligned);
180 Mul(reg2, reg0, reg1, pMask);180 Mul(reg2, reg0, reg1, pMask);
181 181 
182- ReduceSum(reg2, reg2, pMask);182+ Reduce<ReduceType::SUM>(reg2, reg2, pMask);
183 Duplicate(reg2, reg2, pFull); // 广播第一个元素183 Duplicate(reg2, reg2, pFull); // 广播第一个元素
184 184 
185 Mul(reg0, reg0, reg1, pMask);185 Mul(reg0, reg0, reg1, pMask);
@@ -191,13 +191,13 @@ __aicore__ inline void SoftmaxGradExtAR<T>::NormComputeSmallR(const int64_t aSiz
191 }191 }
192 }192 }
193 } else {193 } else {
194- __local_mem__ T* dst = (__local_mem__ T*)dstTensor.GetPhyAddr();194+ __ubuf__ T* dst = (__ubuf__ T*)dstTensor.GetPhyAddr();
195- __local_mem__ T* x0 = (__local_mem__ T*)x0Tensor.GetPhyAddr();195+ __ubuf__ T* x0 = (__ubuf__ T*)x0Tensor.GetPhyAddr();
196- __local_mem__ T* x1 = (__local_mem__ T*)x1Tensor.GetPhyAddr();196+ __ubuf__ T* x1 = (__ubuf__ T*)x1Tensor.GetPhyAddr();
197- __local_mem__ T* x2 = (__local_mem__ T*)x2Tensor.GetPhyAddr();197+ __ubuf__ T* x2 = (__ubuf__ T*)x2Tensor.GetPhyAddr();
198- __local_mem__ T* x0_1 = (__local_mem__ T*)x0Tensor.GetPhyAddr() + VL_FP32;198+ __ubuf__ T* x0_1 = (__ubuf__ T*)x0Tensor.GetPhyAddr() + VL_FP32;
199- __local_mem__ T* x1_1 = (__local_mem__ T*)x1Tensor.GetPhyAddr() + VL_FP32;199+ __ubuf__ T* x1_1 = (__ubuf__ T*)x1Tensor.GetPhyAddr() + VL_FP32;
200- __local_mem__ T* x2_1 = (__local_mem__ T*)x2Tensor.GetPhyAddr() + VL_FP32;200+ __ubuf__ T* x2_1 = (__ubuf__ T*)x2Tensor.GetPhyAddr() + VL_FP32;
201 201 
202 __VEC_SCOPE__202 __VEC_SCOPE__
203 {203 {
@@ -219,8 +219,8 @@ __aicore__ inline void SoftmaxGradExtAR<T>::NormComputeSmallR(const int64_t aSiz
219 Mul(reg2_1, reg0_1, reg1_1, pMask);219 Mul(reg2_1, reg0_1, reg1_1, pMask);
220 220 
221 Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(reg2_1, reg2, reg2_1, pMask);221 Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(reg2_1, reg2, reg2_1, pMask);
222- Copy<float, AscendC::MicroAPI::MaskMergeMode::MERGING>(reg2, reg2_1, pMask);222+ Move<float, AscendC::MicroAPI::MaskMergeMode::MERGING>(reg2, reg2_1, pMask);
223- ReduceSum(reg2, reg2, pFull);223+ Reduce<ReduceType::SUM>(reg2, reg2, pFull);
224 Duplicate(reg2, reg2, pFull);224 Duplicate(reg2, reg2, pFull);
225 225 
226 Mul(reg0, reg0, reg1, pFull);226 Mul(reg0, reg0, reg1, pFull);
@@ -276,27 +276,27 @@ __aicore__ inline void SoftmaxGradExtAR<T>::NormCompute(const int64_t aSize)
276 int64_t tailSrcBOffset = floorVLCount * VL_FP32;276 int64_t tailSrcBOffset = floorVLCount * VL_FP32;
277 int64_t unFoldSrcOffset = (mainFoldLoopTimes + tailFoldLoopTimes) * VL_FP32; // 计算偏移量277 int64_t unFoldSrcOffset = (mainFoldLoopTimes + tailFoldLoopTimes) * VL_FP32; // 计算偏移量
278 278 
279- __local_mem__ float* dst = (__local_mem__ float*)reduceSumTempTensor.GetPhyAddr();279+ __ubuf__ float* dst = (__ubuf__ float*)reduceSumTempTensor.GetPhyAddr();
280- __local_mem__ T* foldX0A = (__local_mem__ T*)x0Tensor.GetPhyAddr();280+ __ubuf__ T* foldX0A = (__ubuf__ T*)x0Tensor.GetPhyAddr();
281- __local_mem__ T* foldX0B = (__local_mem__ T*)x0Tensor.GetPhyAddr() + foldSrcBOffset;281+ __ubuf__ T* foldX0B = (__ubuf__ T*)x0Tensor.GetPhyAddr() + foldSrcBOffset;
282- __local_mem__ T* tailX0A = (__local_mem__ T*)x0Tensor.GetPhyAddr() + tailSrcAOffset;282+ __ubuf__ T* tailX0A = (__ubuf__ T*)x0Tensor.GetPhyAddr() + tailSrcAOffset;
283- __local_mem__ T* tailX0B = (__local_mem__ T*)x0Tensor.GetPhyAddr() + tailSrcBOffset;283+ __ubuf__ T* tailX0B = (__ubuf__ T*)x0Tensor.GetPhyAddr() + tailSrcBOffset;
284- __local_mem__ T* unFoldX0 = (__local_mem__ T*)x0Tensor.GetPhyAddr() + unFoldSrcOffset; // 获取局部内存指针284+ __ubuf__ T* unFoldX0 = (__ubuf__ T*)x0Tensor.GetPhyAddr() + unFoldSrcOffset; // 获取局部内存指针
285 285 
286- __local_mem__ T* foldX1A = (__local_mem__ T*)x1Tensor.GetPhyAddr();286+ __ubuf__ T* foldX1A = (__ubuf__ T*)x1Tensor.GetPhyAddr();
287- __local_mem__ T* foldX1B = (__local_mem__ T*)x1Tensor.GetPhyAddr() + foldSrcBOffset;287+ __ubuf__ T* foldX1B = (__ubuf__ T*)x1Tensor.GetPhyAddr() + foldSrcBOffset;
288- __local_mem__ T* tailX1A = (__local_mem__ T*)x1Tensor.GetPhyAddr() + tailSrcAOffset;288+ __ubuf__ T* tailX1A = (__ubuf__ T*)x1Tensor.GetPhyAddr() + tailSrcAOffset;
289- __local_mem__ T* tailX1B = (__local_mem__ T*)x1Tensor.GetPhyAddr() + tailSrcBOffset;289+ __ubuf__ T* tailX1B = (__ubuf__ T*)x1Tensor.GetPhyAddr() + tailSrcBOffset;
290- __local_mem__ T* unFoldX1 = (__local_mem__ T*)x1Tensor.GetPhyAddr() + unFoldSrcOffset;290+ __ubuf__ T* unFoldX1 = (__ubuf__ T*)x1Tensor.GetPhyAddr() + unFoldSrcOffset;
291 291 
292 __VEC_SCOPE__292 __VEC_SCOPE__
293 {293 {
294 AscendC::MicroAPI::MaskReg pFull = AscendC::MicroAPI::CreateMask<294 AscendC::MicroAPI::MaskReg pFull = AscendC::MicroAPI::CreateMask<
295 float, AscendC::MicroAPI::MaskPattern::ALL>(); // 创建一个全掩码pFull，表示所有元素都参与计算295 float, AscendC::MicroAPI::MaskPattern::ALL>(); // 创建一个全掩码pFull，表示所有元素都参与计算
296- AscendC::MicroAPI::UnalignReg UReg; // 创建一个未对齐寄存器Ureg，用于处理非对齐内存访问296+ AscendC::MicroAPI::UnalignRegForStore UReg; // 创建一个未对齐寄存器Ureg，用于处理非对齐内存访问
297 297 
298 for (uint16_t i = 0; i < outerLoopTimes; i++) { // 外部循环298 for (uint16_t i = 0; i < outerLoopTimes; i++) { // 外部循环
299- dst = (__local_mem__ float*)reduceSumTempTensor.GetPhyAddr() +299+ dst = (__ubuf__ float*)reduceSumTempTensor.GetPhyAddr() +
300 i * outerLoopDstStride; // 每次更新目标地址，指向当前外循环迭代的起始位置300 i * outerLoopDstStride; // 每次更新目标地址，指向当前外循环迭代的起始位置
301 for (uint16_t j = 0; j < mainFoldLoopTimes; j++) { // 主折叠循环301 for (uint16_t j = 0; j < mainFoldLoopTimes; j++) { // 主折叠循环
302 AscendC::MicroAPI::RegTensor<float> reg0, reg1, reg0_1, reg1_1, reg2, reg2_1;302 AscendC::MicroAPI::RegTensor<float> reg0, reg1, reg0_1, reg1_1, reg2, reg2_1;
@@ -312,9 +312,9 @@ __aicore__ inline void SoftmaxGradExtAR<T>::NormCompute(const int64_t aSize)
312 312 
313 Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(313 Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(
314 reg2, reg2, reg2_1, pFull); // 将结果相加存储在reg2，使用zeroing模式，未参与计算的元素置为0314 reg2, reg2, reg2_1, pFull); // 将结果相加存储在reg2，使用zeroing模式，未参与计算的元素置为0
315- ReduceSum(reg2, reg2, pFull); // 对reg2中的数据进行求和315+ Reduce<ReduceType::SUM>(reg2, reg2, pFull); // 对reg2中的数据进行求和
316- AscendC::MicroAPI::DataCopyUnAlign((__local_mem__ float*&)dst, reg2, UReg,316+ AscendC::MicroAPI::StoreUnAlign((__ubuf__ float*&)dst, reg2, UReg,
317- 1); // 将结果拷贝到dst，使用未对齐拷贝317+ 1); // 将结果拷贝到dst，使用未对齐拷贝
318 }318 }
319 for (uint16_t j = 0; j < tailFoldLoopTimes; j++) { // 尾部折叠循环319 for (uint16_t j = 0; j < tailFoldLoopTimes; j++) { // 尾部折叠循环
320 uint32_t count = static_cast<uint32_t>(tailFoldElemCount);320 uint32_t count = static_cast<uint32_t>(tailFoldElemCount);
@@ -331,9 +331,9 @@ __aicore__ inline void SoftmaxGradExtAR<T>::NormCompute(const int64_t aSize)
331 Mul(reg2_1, reg1, reg1_1, pMask);331 Mul(reg2_1, reg1, reg1_1, pMask);
332 332 
333 Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(reg2_1, reg2, reg2_1, pMask);333 Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(reg2_1, reg2, reg2_1, pMask);
334- Copy<float, AscendC::MicroAPI::MaskMergeMode::MERGING>(reg2, reg2_1, pMask);334+ Move<float, AscendC::MicroAPI::MaskMergeMode::MERGING>(reg2, reg2_1, pMask);
335- ReduceSum(reg2, reg2, pFull);335+ Reduce<ReduceType::SUM>(reg2, reg2, pFull);
336- AscendC::MicroAPI::DataCopyUnAlign((__local_mem__ float*&)dst, reg2, UReg, 1);336+ AscendC::MicroAPI::StoreUnAlign((__ubuf__ float*&)dst, reg2, UReg, 1);
337 }337 }
338 for (uint16_t j = 0; j < unFoldLoopTimes; j++) { // 非折叠循环338 for (uint16_t j = 0; j < unFoldLoopTimes; j++) { // 非折叠循环
339 AscendC::MicroAPI::RegTensor<float> reg0, reg1, reg0_1;339 AscendC::MicroAPI::RegTensor<float> reg0, reg1, reg0_1;
@@ -341,10 +341,10 @@ __aicore__ inline void SoftmaxGradExtAR<T>::NormCompute(const int64_t aSize)
341 LoadTensorForDtypeTIn(unFoldX1, reg0_1, pFull, i * outerLoopStride + j * innerLoopStride);341 LoadTensorForDtypeTIn(unFoldX1, reg0_1, pFull, i * outerLoopStride + j * innerLoopStride);
342 342 
343 Mul(reg1, reg0, reg0_1, pFull);343 Mul(reg1, reg0, reg0_1, pFull);
344- ReduceSum(reg1, reg1, pFull);344+ Reduce<ReduceType::SUM>(reg1, reg1, pFull);
345- AscendC::MicroAPI::DataCopyUnAlign((__local_mem__ float*&)dst, reg1, UReg, 1);345+ AscendC::MicroAPI::StoreUnAlign((__ubuf__ float*&)dst, reg1, UReg, 1);
346 }346 }
347- AscendC::MicroAPI::DataCopyUnAlignPost((__local_mem__ float*&)dst, UReg, 0);347+ AscendC::MicroAPI::StoreUnAlignPost((__ubuf__ float*&)dst, UReg, 0);
348 }348 }
349 }349 }
350 NormComputePost(dstTensor, x0Tensor, x1Tensor, x2Tensor, reduceSumTempTensor, aSize, foldPoint, outerLoopDstStride);350 NormComputePost(dstTensor, x0Tensor, x1Tensor, x2Tensor, reduceSumTempTensor, aSize, foldPoint, outerLoopDstStride);
@@ -374,11 +374,11 @@ __aicore__ inline void SoftmaxGradExtAR<T>::NormComputePost(
374 uint16_t oriRAligned = tl_->rAligned;374 uint16_t oriRAligned = tl_->rAligned;
375 375 
376 if (rSize <= VL_FP32) {376 if (rSize <= VL_FP32) {
377- __local_mem__ T* dst = (__local_mem__ T*)dstTensor.GetPhyAddr();377+ __ubuf__ T* dst = (__ubuf__ T*)dstTensor.GetPhyAddr();
378- __local_mem__ T* x0 = (__local_mem__ T*)x0Tensor.GetPhyAddr();378+ __ubuf__ T* x0 = (__ubuf__ T*)x0Tensor.GetPhyAddr();
379- __local_mem__ T* x1 = (__local_mem__ T*)x1Tensor.GetPhyAddr();379+ __ubuf__ T* x1 = (__ubuf__ T*)x1Tensor.GetPhyAddr();
380- __local_mem__ T* x2 = (__local_mem__ T*)x2Tensor.GetPhyAddr();380+ __ubuf__ T* x2 = (__ubuf__ T*)x2Tensor.GetPhyAddr();
381- __local_mem__ float* sumTmp = (__local_mem__ float*)binAddTmpTensor.GetPhyAddr();381+ __ubuf__ float* sumTmp = (__ubuf__ float*)binAddTmpTensor.GetPhyAddr();
382 382 
383 __VEC_SCOPE__383 __VEC_SCOPE__
384 {384 {
@@ -390,8 +390,8 @@ __aicore__ inline void SoftmaxGradExtAR<T>::NormComputePost(
390 float, AscendC::MicroAPI::MaskPattern::ALL>(); // 创建一个掩码寄存器pFull，并将其设置为全掩码模式390 float, AscendC::MicroAPI::MaskPattern::ALL>(); // 创建一个掩码寄存器pFull，并将其设置为全掩码模式
391 AscendC::MicroAPI::MaskReg maskOri;391 AscendC::MicroAPI::MaskReg maskOri;
392 for (uint16_t i = 0; i < loopTimes; i++) {392 for (uint16_t i = 0; i < loopTimes; i++) {
393- DataCopy(reg0, (__local_mem__ float*)sumTmp + i * stride);393+ Reg::LoadAlign(reg0, (__ubuf__ float*)sumTmp + i * stride);
394- ReduceSum(reg1, reg0, pMask);394+ Reduce<ReduceType::SUM>(reg1, reg0, pMask);
395 Duplicate(reg2, reg1, pFull);395 Duplicate(reg2, reg1, pFull);
396 396 
397 uint32_t sreg0 = static_cast<uint32_t>(oriR);397 uint32_t sreg0 = static_cast<uint32_t>(oriR);
@@ -410,13 +410,13 @@ __aicore__ inline void SoftmaxGradExtAR<T>::NormComputePost(
410 }410 }
411 }411 }
412 } else {412 } else {
413- __local_mem__ T* dst = (__local_mem__ T*)dstTensor.GetPhyAddr();413+ __ubuf__ T* dst = (__ubuf__ T*)dstTensor.GetPhyAddr();
414- __local_mem__ float* sumTmpA = (__local_mem__ float*)binAddTmpTensor.GetPhyAddr();414+ __ubuf__ float* sumTmpA = (__ubuf__ float*)binAddTmpTensor.GetPhyAddr();
415- __local_mem__ float* sumTmpB = (__local_mem__ float*)binAddTmpTensor.GetPhyAddr() + VL_FP32;415+ __ubuf__ float* sumTmpB = (__ubuf__ float*)binAddTmpTensor.GetPhyAddr() + VL_FP32;
416 416 
417- __local_mem__ T* x0 = (__local_mem__ T*)x0Tensor.GetPhyAddr();417+ __ubuf__ T* x0 = (__ubuf__ T*)x0Tensor.GetPhyAddr();
418- __local_mem__ T* x1 = (__local_mem__ T*)x1Tensor.GetPhyAddr();418+ __ubuf__ T* x1 = (__ubuf__ T*)x1Tensor.GetPhyAddr();
419- __local_mem__ T* x2 = (__local_mem__ T*)x2Tensor.GetPhyAddr();419+ __ubuf__ T* x2 = (__ubuf__ T*)x2Tensor.GetPhyAddr();
420 420 
421 __VEC_SCOPE__421 __VEC_SCOPE__
422 {422 {
@@ -428,11 +428,11 @@ __aicore__ inline void SoftmaxGradExtAR<T>::NormComputePost(
428 float, AscendC::MicroAPI::MaskPattern::ALL>(); // 创建一个掩码寄存器pFull，并将其设置为全掩码模式428 float, AscendC::MicroAPI::MaskPattern::ALL>(); // 创建一个掩码寄存器pFull，并将其设置为全掩码模式
429 AscendC::MicroAPI::MaskReg maskOri;429 AscendC::MicroAPI::MaskReg maskOri;
430 for (uint16_t i = 0; i < loopTimes; i++) {430 for (uint16_t i = 0; i < loopTimes; i++) {
431- DataCopy(reg0, (__local_mem__ float*)sumTmpA + i * stride);431+ Reg::LoadAlign(reg0, (__ubuf__ float*)sumTmpA + i * stride);
432- DataCopy(reg1, (__local_mem__ float*)sumTmpB + i * stride);432+ Reg::LoadAlign(reg1, (__ubuf__ float*)sumTmpB + i * stride);
433 Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(reg1, reg0, reg1, pMask);433 Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(reg1, reg0, reg1, pMask);
434- Copy<float, AscendC::MicroAPI::MaskMergeMode::MERGING>(reg0, reg1, pMask);434+ Move<float, AscendC::MicroAPI::MaskMergeMode::MERGING>(reg0, reg1, pMask);
435- ReduceSum(reg2, reg0, pFull);435+ Reduce<ReduceType::SUM>(reg2, reg0, pFull);
436 Duplicate(reg2, reg2, pFull);436 Duplicate(reg2, reg2, pFull);
437 uint32_t sreg0 = static_cast<uint32_t>(oriR);437 uint32_t sreg0 = static_cast<uint32_t>(oriR);
438 for (uint16_t j = 0; j < rLoopCount; j++) {438 for (uint16_t j = 0; j < rLoopCount; j++) {
@@ -455,14 +455,14 @@ __aicore__ inline void SoftmaxGradExtAR<T>::NormComputePost(
455}455}
456 456 
457template <typename T>457template <typename T>
458-__aicore__ inline void SoftmaxGradExtAR<T>::LoadTensorForDtypeTIn(__local_mem__ T* src, RegTensor<float>& dst,458+__aicore__ inline void SoftmaxGradExtAR<T>::LoadTensorForDtypeTIn(__ubuf__ T* src, RegTensor<float>& dst, MaskReg& preg,
459- MaskReg& preg, uint32_t offset)459+ uint32_t offset)
460{460{
461 if constexpr (IsSameType<T, float>::value) {461 if constexpr (IsSameType<T, float>::value) {
462- DataCopy<float, LoadDist::DIST_NORM>(dst, (__local_mem__ float*)src + offset);462+ Reg::LoadAlign<float, LoadDist::DIST_NORM>(dst, (__ubuf__ float*)src + offset);
463 } else { // fp16、bf16463 } else { // fp16、bf16
464 RegTensor<T> xFp16;464 RegTensor<T> xFp16;
465- DataCopy<T, LoadDist::DIST_UNPACK_B16>(xFp16, ((__local_mem__ T*)src + offset));465+ Reg::LoadAlign<T, LoadDist::DIST_UNPACK_B16>(xFp16, ((__ubuf__ T*)src + offset));
466 Cast<float, T, castTraitFp16ToFp32>(dst, xFp16, preg);466 Cast<float, T, castTraitFp16ToFp32>(dst, xFp16, preg);
467 }467 }
468}468}
@@ -503,16 +503,16 @@ __aicore__ inline void SoftmaxGradExtAR<T>::CopyInX(int64_t ubA, int64_t offset)
503}503}
504 504 
505template <typename T>505template <typename T>
506-__aicore__ inline void SoftmaxGradExtAR<T>::StoreTensorForDtypeTOut(__local_mem__ T* dst,506+__aicore__ inline void SoftmaxGradExtAR<T>::StoreTensorForDtypeTOut(__ubuf__ T* dst,
507 AscendC::MicroAPI::RegTensor<float>& src,507 AscendC::MicroAPI::RegTensor<float>& src,
508 AscendC::MicroAPI::MaskReg& preg, uint32_t offset)508 AscendC::MicroAPI::MaskReg& preg, uint32_t offset)
509{509{
510 if constexpr (IsSameType<T, float>::value) {510 if constexpr (IsSameType<T, float>::value) {
511- DataCopy<T, AscendC::MicroAPI::StoreDist::DIST_NORM>(dst + offset, src, preg);511+ Reg::StoreAlign<T, AscendC::MicroAPI::StoreDist::DIST_NORM>(dst + offset, src, preg);
512 } else {512 } else {
513 AscendC::MicroAPI::RegTensor<T> xFp16;513 AscendC::MicroAPI::RegTensor<T> xFp16;
514 Cast<T, float, castTraitFp32ToFp16>(xFp16, src, preg);514 Cast<T, float, castTraitFp32ToFp16>(xFp16, src, preg);
515- DataCopy<T, AscendC::MicroAPI::StoreDist::DIST_PACK_B32>(dst + offset, xFp16, preg);515+ Reg::StoreAlign<T, AscendC::MicroAPI::StoreDist::DIST_PACK_B32>(dst + offset, xFp16, preg);
516 }516 }
517}517}
518 518 
@@ -531,4 +531,4 @@ __aicore__ inline void SoftmaxGradExtAR<T>::CopyOutY(int64_t ubA, int64_t offset
531}531}
532 532 
533} // namespace SoftmaxGradExt533} // namespace SoftmaxGradExt
534-#endif // SOFTMAX_GRAD_AR_FULL_LOAD_H534+#endif // SOFTMAX_GRAD_AR_FULL_LOAD_H
Mactivation/softmax_grad_ext/op_kernel/arch35/softmax_grad_ext_ar_recompute.h+74-74
@@ -48,8 +48,8 @@ private:
48 __aicore__ inline void CalcOutVF(uint32_t ubFactor);48 __aicore__ inline void CalcOutVF(uint32_t ubFactor);
49 __aicore__ inline void CalcOutVFSameShape(uint32_t ubFactor);49 __aicore__ inline void CalcOutVFSameShape(uint32_t ubFactor);
50 50 
51- __aicore__ inline void MainBlockVF(__local_mem__ float* dst, uint32_t ubFactor);51+ __aicore__ inline void MainBlockVF(__ubuf__ float* dst, uint32_t ubFactor);
52- __aicore__ inline void FoldBlockVF(__local_mem__ float* dst, uint32_t ubFactor);52+ __aicore__ inline void FoldBlockVF(__ubuf__ float* dst, uint32_t ubFactor);
53 53 
54 __aicore__ inline void LastReduceSum(const LocalTensor<float>& dstTensor, const LocalTensor<float>& srcTensor,54 __aicore__ inline void LastReduceSum(const LocalTensor<float>& dstTensor, const LocalTensor<float>& srcTensor,
55 const LocalTensor<float>& reduceSumTempTensor, const int64_t aSize,55 const LocalTensor<float>& reduceSumTempTensor, const int64_t aSize,
@@ -58,9 +58,9 @@ private:
58 __aicore__ inline void LastReduceSumSmallR(const LocalTensor<float>& dstTensor, const LocalTensor<float>& srcTensor,58 __aicore__ inline void LastReduceSumSmallR(const LocalTensor<float>& dstTensor, const LocalTensor<float>& srcTensor,
59 const int64_t aSize, const int64_t rSize, const int64_t stride);59 const int64_t aSize, const int64_t rSize, const int64_t stride);
60 60 
61- __aicore__ inline void LoadTensorForDtypeT(__local_mem__ T* src, AscendC::MicroAPI::RegTensor<float>& dst,61+ __aicore__ inline void LoadTensorForDtypeT(__ubuf__ T* src, AscendC::MicroAPI::RegTensor<float>& dst,
62 AscendC::MicroAPI::MaskReg& pregMask, uint32_t offset);62 AscendC::MicroAPI::MaskReg& pregMask, uint32_t offset);
63- __aicore__ inline void StoreTensorForDtypeTOut(__local_mem__ T* dst, AscendC::MicroAPI::RegTensor<float>& src,63+ __aicore__ inline void StoreTensorForDtypeTOut(__ubuf__ T* dst, AscendC::MicroAPI::RegTensor<float>& src,
64 AscendC::MicroAPI::MaskReg& preg, uint32_t offset);64 AscendC::MicroAPI::MaskReg& preg, uint32_t offset);
65 65 
66 __aicore__ inline void CopyInX(int64_t xGmOffset, uint32_t ubFactor);66 __aicore__ inline void CopyInX(int64_t xGmOffset, uint32_t ubFactor);
@@ -188,7 +188,7 @@ __aicore__ inline void SoftmaxGradExtARRecompute<T>::CalcReduceSum(int64_t xDimO
188 LocalTensor<float> xSum = xSumBuffer_.Get<float>(); // 大小为32B188 LocalTensor<float> xSum = xSumBuffer_.Get<float>(); // 大小为32B
189 189 
190 LocalTensor<float> xTmp = yQueue_.AllocTensor<float>(); // 复用y做二分累加190 LocalTensor<float> xTmp = yQueue_.AllocTensor<float>(); // 复用y做二分累加
191- __local_mem__ float* xTmpLocal = (__local_mem__ float*)xTmp.GetPhyAddr();191+ __ubuf__ float* xTmpLocal = (__ubuf__ float*)xTmp.GetPhyAddr();
192 192 
193 // ub间累加fold折叠到main193 // ub间累加fold折叠到main
194 for (uint64_t basicBlockIdx = 0; basicBlockIdx < tl_->basicBlockLoop; basicBlockIdx++) {194 for (uint64_t basicBlockIdx = 0; basicBlockIdx < tl_->basicBlockLoop; basicBlockIdx++) {
@@ -228,13 +228,13 @@ __aicore__ inline void SoftmaxGradExtARRecompute<T>::CalcReduceSum(int64_t xDimO
228 228 
229// cast + mul，cast成FP32之后求gradi * x0i229// cast + mul，cast成FP32之后求gradi * x0i
230template <typename T>230template <typename T>
231-__aicore__ inline void SoftmaxGradExtARRecompute<T>::MainBlockVF(__local_mem__ float* dst, uint32_t ubFactor)231+__aicore__ inline void SoftmaxGradExtARRecompute<T>::MainBlockVF(__ubuf__ float* dst, uint32_t ubFactor)
232{232{
233 LocalTensor<T> x0 = x0Queue_.DeQue<T>();233 LocalTensor<T> x0 = x0Queue_.DeQue<T>();
234 LocalTensor<T> grad = gradQueue_.DeQue<T>();234 LocalTensor<T> grad = gradQueue_.DeQue<T>();
235 235 
236- __local_mem__ T* x0Local = (__local_mem__ T*)x0.GetPhyAddr();236+ __ubuf__ T* x0Local = (__ubuf__ T*)x0.GetPhyAddr();
237- __local_mem__ T* gradLocal = (__local_mem__ T*)grad.GetPhyAddr();237+ __ubuf__ T* gradLocal = (__ubuf__ T*)grad.GetPhyAddr();
238 238 
239 __VEC_SCOPE__239 __VEC_SCOPE__
240 {240 {
@@ -251,7 +251,7 @@ __aicore__ inline void SoftmaxGradExtARRecompute<T>::MainBlockVF(__local_mem__ f
251 251 
252 Mul(reg0, reg0, reg1, pregMask);252 Mul(reg0, reg0, reg1, pregMask);
253 253 
254- AscendC::MicroAPI::DataCopy(dst + offset, reg0, pregMask);254+ AscendC::MicroAPI::StoreAlign(dst + offset, reg0, pregMask);
255 }255 }
256 }256 }
257 x0Queue_.FreeTensor(x0);257 x0Queue_.FreeTensor(x0);
@@ -260,12 +260,12 @@ __aicore__ inline void SoftmaxGradExtARRecompute<T>::MainBlockVF(__local_mem__ f
260 260 
261// fold块的cast + mul261// fold块的cast + mul
262template <typename T>262template <typename T>
263-__aicore__ inline void SoftmaxGradExtARRecompute<T>::FoldBlockVF(__local_mem__ float* dst, uint32_t ubFactor)263+__aicore__ inline void SoftmaxGradExtARRecompute<T>::FoldBlockVF(__ubuf__ float* dst, uint32_t ubFactor)
264{264{
265 LocalTensor<T> x0 = x0Queue_.DeQue<T>();265 LocalTensor<T> x0 = x0Queue_.DeQue<T>();
266 LocalTensor<T> grad = gradQueue_.DeQue<T>();266 LocalTensor<T> grad = gradQueue_.DeQue<T>();
267- __local_mem__ T* x0Local = (__local_mem__ T*)x0.GetPhyAddr();267+ __ubuf__ T* x0Local = (__ubuf__ T*)x0.GetPhyAddr();
268- __local_mem__ T* gradLocal = (__local_mem__ T*)grad.GetPhyAddr();268+ __ubuf__ T* gradLocal = (__ubuf__ T*)grad.GetPhyAddr();
269 269 
270 __VEC_SCOPE__270 __VEC_SCOPE__
271 {271 {
@@ -285,12 +285,12 @@ __aicore__ inline void SoftmaxGradExtARRecompute<T>::FoldBlockVF(__local_mem__ f
285 285 
286 Mul(reg0, reg1, reg0, pregMask);286 Mul(reg0, reg1, reg0, pregMask);
287 287 
288- AscendC::MicroAPI::DataCopy(reg1, dst + offset);288+ AscendC::MicroAPI::LoadAlign(reg1, dst + offset);
289 289 
290 AscendC::MicroAPI::Add(reg0, reg1, reg0, pregMask);290 AscendC::MicroAPI::Add(reg0, reg1, reg0, pregMask);
291- AscendC::MicroAPI::Copy<float, AscendC::MicroAPI::MaskMergeMode::MERGING>(reg1, reg0, pregMask);291+ AscendC::MicroAPI::Move<float, AscendC::MicroAPI::MaskMergeMode::MERGING>(reg1, reg0, pregMask);
292 292 
293- AscendC::MicroAPI::DataCopy(dst + offset, reg1, maskFull);293+ AscendC::MicroAPI::StoreAlign(dst + offset, reg1, maskFull);
294 }294 }
295 }295 }
296 296 
@@ -309,11 +309,11 @@ __aicore__ inline void SoftmaxGradExtARRecompute<T>::CalcOutVF(uint32_t ubFactor
309 309 
310 Duplicate<T>(x1, x1, ubFactor);310 Duplicate<T>(x1, x1, ubFactor);
311 311 
312- __local_mem__ float* xSumLocal = (__local_mem__ float*)xSumTensor_.GetPhyAddr();312+ __ubuf__ float* xSumLocal = (__ubuf__ float*)xSumTensor_.GetPhyAddr();
313- __local_mem__ T* x0Local = (__local_mem__ T*)x0.GetPhyAddr();313+ __ubuf__ T* x0Local = (__ubuf__ T*)x0.GetPhyAddr();
314- __local_mem__ T* x1Local = (__local_mem__ T*)x1.GetPhyAddr();314+ __ubuf__ T* x1Local = (__ubuf__ T*)x1.GetPhyAddr();
315- __local_mem__ T* gradLocal = (__local_mem__ T*)grad.GetPhyAddr();315+ __ubuf__ T* gradLocal = (__ubuf__ T*)grad.GetPhyAddr();
316- __local_mem__ T* yLocal = (__local_mem__ T*)y.GetPhyAddr();316+ __ubuf__ T* yLocal = (__ubuf__ T*)y.GetPhyAddr();
317 317 
318 __VEC_SCOPE__318 __VEC_SCOPE__
319 {319 {
@@ -323,7 +323,7 @@ __aicore__ inline void SoftmaxGradExtARRecompute<T>::CalcOutVF(uint32_t ubFactor
323 uint32_t sreg = ubFactor;323 uint32_t sreg = ubFactor;
324 uint16_t loopTimes = CeilDivision(ubFactor, VL_FP32);324 uint16_t loopTimes = CeilDivision(ubFactor, VL_FP32);
325 325 
326- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_BRC_B32>(sumReg, xSumLocal);326+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_BRC_B32>(sumReg, xSumLocal);
327 for (uint16_t j = 0; j < loopTimes; j++) {327 for (uint16_t j = 0; j < loopTimes; j++) {
328 pregMask = AscendC::MicroAPI::UpdateMask<float>(sreg);328 pregMask = AscendC::MicroAPI::UpdateMask<float>(sreg);
329 uint32_t offset = j * VL_FP32;329 uint32_t offset = j * VL_FP32;
@@ -357,11 +357,11 @@ __aicore__ inline void SoftmaxGradExtARRecompute<T>::CalcOutVFSameShape(uint32_t
357 LocalTensor<T> grad = gradQueue_.DeQue<T>();357 LocalTensor<T> grad = gradQueue_.DeQue<T>();
358 LocalTensor<T> y = yQueue_.AllocTensor<T>();358 LocalTensor<T> y = yQueue_.AllocTensor<T>();
359 359 
360- __local_mem__ float* xSumLocal = (__local_mem__ float*)xSumTensor_.GetPhyAddr();360+ __ubuf__ float* xSumLocal = (__ubuf__ float*)xSumTensor_.GetPhyAddr();
361- __local_mem__ T* x0Local = (__local_mem__ T*)x0.GetPhyAddr();361+ __ubuf__ T* x0Local = (__ubuf__ T*)x0.GetPhyAddr();
362- __local_mem__ T* x1Local = (__local_mem__ T*)x1.GetPhyAddr();362+ __ubuf__ T* x1Local = (__ubuf__ T*)x1.GetPhyAddr();
363- __local_mem__ T* gradLocal = (__local_mem__ T*)grad.GetPhyAddr();363+ __ubuf__ T* gradLocal = (__ubuf__ T*)grad.GetPhyAddr();
364- __local_mem__ T* yLocal = (__local_mem__ T*)y.GetPhyAddr();364+ __ubuf__ T* yLocal = (__ubuf__ T*)y.GetPhyAddr();
365 365 
366 __VEC_SCOPE__366 __VEC_SCOPE__
367 {367 {
@@ -371,7 +371,7 @@ __aicore__ inline void SoftmaxGradExtARRecompute<T>::CalcOutVFSameShape(uint32_t
371 uint32_t sreg = ubFactor;371 uint32_t sreg = ubFactor;
372 uint16_t loopTimes = CeilDivision(ubFactor, VL_FP32);372 uint16_t loopTimes = CeilDivision(ubFactor, VL_FP32);
373 373 
374- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_BRC_B32>(sumReg, xSumLocal);374+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_BRC_B32>(sumReg, xSumLocal);
375 for (uint16_t j = 0; j < loopTimes; j++) {375 for (uint16_t j = 0; j < loopTimes; j++) {
376 pregMask = AscendC::MicroAPI::UpdateMask<float>(sreg);376 pregMask = AscendC::MicroAPI::UpdateMask<float>(sreg);
377 uint32_t offset = j * VL_FP32;377 uint32_t offset = j * VL_FP32;
@@ -398,33 +398,33 @@ __aicore__ inline void SoftmaxGradExtARRecompute<T>::CalcOutVFSameShape(uint32_t
398 398 
399// 从UB搬到寄存器的同时转换数据类型399// 从UB搬到寄存器的同时转换数据类型
400template <typename T>400template <typename T>
401-__aicore__ inline void SoftmaxGradExtARRecompute<T>::LoadTensorForDtypeT(__local_mem__ T* src,401+__aicore__ inline void SoftmaxGradExtARRecompute<T>::LoadTensorForDtypeT(__ubuf__ T* src,
402 AscendC::MicroAPI::RegTensor<float>& dst,402 AscendC::MicroAPI::RegTensor<float>& dst,
403 AscendC::MicroAPI::MaskReg& pregMask,403 AscendC::MicroAPI::MaskReg& pregMask,
404 uint32_t offset)404 uint32_t offset)
405{405{
406 if constexpr (IsSameType<T, float>::value) {406 if constexpr (IsSameType<T, float>::value) {
407- DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(dst, (__local_mem__ float*)src + offset);407+ Reg::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(dst, (__ubuf__ float*)src + offset);
408 } else { // fp16、bf16408 } else { // fp16、bf16
409 AscendC::MicroAPI::RegTensor<T> xFp16;409 AscendC::MicroAPI::RegTensor<T> xFp16;
410- DataCopy<T, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(xFp16, ((__local_mem__ T*)src + offset));410+ Reg::LoadAlign<T, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(xFp16, ((__ubuf__ T*)src + offset));
411 Cast<float, T, castTraitFp16ToFp32>(dst, xFp16, pregMask);411 Cast<float, T, castTraitFp16ToFp32>(dst, xFp16, pregMask);
412 }412 }
413}413}
414 414 
415// 从寄存器搬到UB的同时转换为原来数据类型415// 从寄存器搬到UB的同时转换为原来数据类型
416template <typename T>416template <typename T>
417-__aicore__ inline void SoftmaxGradExtARRecompute<T>::StoreTensorForDtypeTOut(__local_mem__ T* dst,417+__aicore__ inline void SoftmaxGradExtARRecompute<T>::StoreTensorForDtypeTOut(__ubuf__ T* dst,
418 AscendC::MicroAPI::RegTensor<float>& src,418 AscendC::MicroAPI::RegTensor<float>& src,
419 AscendC::MicroAPI::MaskReg& preg,419 AscendC::MicroAPI::MaskReg& preg,
420 uint32_t offset)420 uint32_t offset)
421{421{
422 if constexpr (IsSameType<T, float>::value) {422 if constexpr (IsSameType<T, float>::value) {
423- DataCopy<T, AscendC::MicroAPI::StoreDist::DIST_NORM>(dst + offset, src, preg);423+ Reg::StoreAlign<T, AscendC::MicroAPI::StoreDist::DIST_NORM>(dst + offset, src, preg);
424 } else {424 } else {
425 AscendC::MicroAPI::RegTensor<T> xFp16;425 AscendC::MicroAPI::RegTensor<T> xFp16;
426 Cast<T, float, castTraitFp32ToFp16>(xFp16, src, preg);426 Cast<T, float, castTraitFp32ToFp16>(xFp16, src, preg);
427- DataCopy<T, AscendC::MicroAPI::StoreDist::DIST_PACK_B32>(dst + offset, xFp16, preg);427+ Reg::StoreAlign<T, AscendC::MicroAPI::StoreDist::DIST_PACK_B32>(dst + offset, xFp16, preg);
428 }428 }
429}429}
430 430 
@@ -556,43 +556,43 @@ __aicore__ inline void SoftmaxGradExtARRecompute<T>::LastReduceSum(const LocalTe
556 556 
557 __VEC_SCOPE__557 __VEC_SCOPE__
558 {558 {
559- __local_mem__ float* dst = (__local_mem__ float*)reduceSumTempTensor.GetPhyAddr();559+ __ubuf__ float* dst = (__ubuf__ float*)reduceSumTempTensor.GetPhyAddr();
560- __local_mem__ float* foldSrcA = (__local_mem__ float*)srcTensor.GetPhyAddr();560+ __ubuf__ float* foldSrcA = (__ubuf__ float*)srcTensor.GetPhyAddr();
561- __local_mem__ float* foldSrcB = (__local_mem__ float*)srcTensor.GetPhyAddr() + foldSrcBOffset;561+ __ubuf__ float* foldSrcB = (__ubuf__ float*)srcTensor.GetPhyAddr() + foldSrcBOffset;
562- __local_mem__ float* tailSrcA = (__local_mem__ float*)srcTensor.GetPhyAddr() + tailSrcAOffset;562+ __ubuf__ float* tailSrcA = (__ubuf__ float*)srcTensor.GetPhyAddr() + tailSrcAOffset;
563- __local_mem__ float* tailSrcB = (__local_mem__ float*)srcTensor.GetPhyAddr() + tailSrcBOffset;563+ __ubuf__ float* tailSrcB = (__ubuf__ float*)srcTensor.GetPhyAddr() + tailSrcBOffset;
564- __local_mem__ float* unFoldSrc = (__local_mem__ float*)srcTensor.GetPhyAddr() + unFoldSrcOffset;564+ __ubuf__ float* unFoldSrc = (__ubuf__ float*)srcTensor.GetPhyAddr() + unFoldSrcOffset;
565 AscendC::MicroAPI::MaskReg pFull = AscendC::MicroAPI::CreateMask<float, AscendC::MicroAPI::MaskPattern::ALL>();565 AscendC::MicroAPI::MaskReg pFull = AscendC::MicroAPI::CreateMask<float, AscendC::MicroAPI::MaskPattern::ALL>();
566- AscendC::MicroAPI::UnalignReg UReg;566+ AscendC::MicroAPI::UnalignRegForStore UReg;
567 567 
568 for (uint16_t i = 0; i < outerLoopTimes; ++i) {568 for (uint16_t i = 0; i < outerLoopTimes; ++i) {
569- dst = (__local_mem__ float*)reduceSumTempTensor.GetPhyAddr() + i * outerLoopDstStride;569+ dst = (__ubuf__ float*)reduceSumTempTensor.GetPhyAddr() + i * outerLoopDstStride;
570 for (uint16_t j = 0; j < mainFoldLoopTimes; ++j) {570 for (uint16_t j = 0; j < mainFoldLoopTimes; ++j) {
571 AscendC::MicroAPI::RegTensor<float> aReg, bReg, cReg, dReg;571 AscendC::MicroAPI::RegTensor<float> aReg, bReg, cReg, dReg;
572- DataCopy(aReg, (__local_mem__ float*)foldSrcA + i * outerLoopStride + j * innerLoopStride);572+ Reg::LoadAlign(aReg, (__ubuf__ float*)foldSrcA + i * outerLoopStride + j * innerLoopStride);
573- DataCopy(bReg, (__local_mem__ float*)foldSrcB + i * outerLoopStride + j * innerLoopStride);573+ Reg::LoadAlign(bReg, (__ubuf__ float*)foldSrcB + i * outerLoopStride + j * innerLoopStride);
574 Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(cReg, aReg, bReg, pFull);574 Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(cReg, aReg, bReg, pFull);
575- ReduceSum(dReg, cReg, pFull);575+ Reduce<ReduceType::SUM>(dReg, cReg, pFull);
576- AscendC::MicroAPI::DataCopyUnAlign((__local_mem__ float*&)dst, dReg, UReg, 1);576+ AscendC::MicroAPI::StoreUnAlign((__ubuf__ float*&)dst, dReg, UReg, 1);
577 }577 }
578 for (uint16_t j = 0; j < tailFoldLoopTimes; ++j) {578 for (uint16_t j = 0; j < tailFoldLoopTimes; ++j) {
579 uint32_t count = static_cast<uint32_t>(tailFoldElemCount);579 uint32_t count = static_cast<uint32_t>(tailFoldElemCount);
580 AscendC::MicroAPI::RegTensor<float> aReg, bReg, cReg;580 AscendC::MicroAPI::RegTensor<float> aReg, bReg, cReg;
581 AscendC::MicroAPI::MaskReg pMask = AscendC::MicroAPI::UpdateMask<float>(count);581 AscendC::MicroAPI::MaskReg pMask = AscendC::MicroAPI::UpdateMask<float>(count);
582- DataCopy(aReg, (__local_mem__ float*)tailSrcA + i * outerLoopStride + j * innerLoopStride);582+ Reg::LoadAlign(aReg, (__ubuf__ float*)tailSrcA + i * outerLoopStride + j * innerLoopStride);
583- DataCopy(bReg, (__local_mem__ float*)tailSrcB + i * outerLoopStride + j * innerLoopStride);583+ Reg::LoadAlign(bReg, (__ubuf__ float*)tailSrcB + i * outerLoopStride + j * innerLoopStride);
584 Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(cReg, aReg, bReg, pMask);584 Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(cReg, aReg, bReg, pMask);
585- Copy<float, AscendC::MicroAPI::MaskMergeMode::MERGING>(aReg, cReg, pMask);585+ Move<float, AscendC::MicroAPI::MaskMergeMode::MERGING>(aReg, cReg, pMask);
586- ReduceSum(bReg, aReg, pFull);586+ Reduce<ReduceType::SUM>(bReg, aReg, pFull);
587- AscendC::MicroAPI::DataCopyUnAlign((__local_mem__ float*&)dst, bReg, UReg, 1);587+ AscendC::MicroAPI::StoreUnAlign((__ubuf__ float*&)dst, bReg, UReg, 1);
588 }588 }
589 for (uint16_t j = 0; j < unFoldLoopTimes; ++j) {589 for (uint16_t j = 0; j < unFoldLoopTimes; ++j) {
590 AscendC::MicroAPI::RegTensor<float> aReg, bReg;590 AscendC::MicroAPI::RegTensor<float> aReg, bReg;
591- DataCopy(aReg, (__local_mem__ float*)unFoldSrc + i * outerLoopStride + j * innerLoopStride);591+ Reg::LoadAlign(aReg, (__ubuf__ float*)unFoldSrc + i * outerLoopStride + j * innerLoopStride);
592- ReduceSum(bReg, aReg, pFull);592+ Reduce<ReduceType::SUM>(bReg, aReg, pFull);
593- AscendC::MicroAPI::DataCopyUnAlign((__local_mem__ float*&)dst, bReg, UReg, 1);593+ AscendC::MicroAPI::StoreUnAlign((__ubuf__ float*&)dst, bReg, UReg, 1);
594 }594 }
595- AscendC::MicroAPI::DataCopyUnAlignPost((__local_mem__ float*&)dst, UReg, 0);595+ AscendC::MicroAPI::StoreUnAlignPost((__ubuf__ float*&)dst, UReg, 0);
596 }596 }
597 }597 }
598 LastReduceSumSmallR(dstTensor, reduceSumTempTensor, aSize, foldPoint, outerLoopDstStride);598 LastReduceSumSmallR(dstTensor, reduceSumTempTensor, aSize, foldPoint, outerLoopDstStride);
@@ -617,8 +617,8 @@ __aicore__ inline void SoftmaxGradExtARRecompute<T>::LastReduceSumSmallR(const L
617 617 
618 uint16_t loopTimes = aSize;618 uint16_t loopTimes = aSize;
619 if (rSize <= VL_FP32) {619 if (rSize <= VL_FP32) {
620- __local_mem__ float* dst = (__local_mem__ float*)dstTensor.GetPhyAddr();620+ __ubuf__ float* dst = (__ubuf__ float*)dstTensor.GetPhyAddr();
621- __local_mem__ float* src = (__local_mem__ float*)srcTensor.GetPhyAddr();621+ __ubuf__ float* src = (__ubuf__ float*)srcTensor.GetPhyAddr();
622 622 
623 __VEC_SCOPE__623 __VEC_SCOPE__
624 {624 {
@@ -628,20 +628,20 @@ __aicore__ inline void SoftmaxGradExtARRecompute<T>::LastReduceSumSmallR(const L
628 628 
629 AscendC::MicroAPI::MaskReg pMask = AscendC::MicroAPI::UpdateMask<float>(count);629 AscendC::MicroAPI::MaskReg pMask = AscendC::MicroAPI::UpdateMask<float>(count);
630 AscendC::MicroAPI::MaskReg maskOne = AscendC::MicroAPI::UpdateMask<float>(constOne);630 AscendC::MicroAPI::MaskReg maskOne = AscendC::MicroAPI::UpdateMask<float>(constOne);
631- AscendC::MicroAPI::UnalignReg UReg;631+ AscendC::MicroAPI::UnalignRegForStore UReg;
632 for (uint16_t i = 0; i < loopTimes; ++i) {632 for (uint16_t i = 0; i < loopTimes; ++i) {
633- AscendC::MicroAPI::DataCopy(aReg, (__local_mem__ float*)src + i * stride);633+ AscendC::MicroAPI::LoadAlign(aReg, (__ubuf__ float*)src + i * stride);
634- AscendC::MicroAPI::ReduceSum(bReg, aReg, pMask);634+ AscendC::MicroAPI::Reduce<ReduceType::SUM>(bReg, aReg, pMask);
635- AscendC::MicroAPI::DataCopy(sumReg, dst);635+ AscendC::MicroAPI::LoadAlign(sumReg, dst);
636 AscendC::MicroAPI::Add(bReg, bReg, sumReg, maskOne);636 AscendC::MicroAPI::Add(bReg, bReg, sumReg, maskOne);
637- AscendC::MicroAPI::DataCopyUnAlign((__local_mem__ float*&)dst, bReg, UReg, 1);637+ AscendC::MicroAPI::StoreUnAlign((__ubuf__ float*&)dst, bReg, UReg, 1);
638 }638 }
639- AscendC::MicroAPI::DataCopyUnAlignPost((__local_mem__ float*&)dst, UReg, 0);639+ AscendC::MicroAPI::StoreUnAlignPost((__ubuf__ float*&)dst, UReg, 0);
640 }640 }
641 } else {641 } else {
642- __local_mem__ float* dst = (__local_mem__ float*)dstTensor.GetPhyAddr();642+ __ubuf__ float* dst = (__ubuf__ float*)dstTensor.GetPhyAddr();
643- __local_mem__ float* src0 = (__local_mem__ float*)srcTensor.GetPhyAddr();643+ __ubuf__ float* src0 = (__ubuf__ float*)srcTensor.GetPhyAddr();
644- __local_mem__ float* src1 = (__local_mem__ float*)srcTensor.GetPhyAddr() + VL_FP32;644+ __ubuf__ float* src1 = (__ubuf__ float*)srcTensor.GetPhyAddr() + VL_FP32;
645 645 
646 __VEC_SCOPE__646 __VEC_SCOPE__
647 {647 {
@@ -649,25 +649,25 @@ __aicore__ inline void SoftmaxGradExtARRecompute<T>::LastReduceSumSmallR(const L
649 uint32_t constOne = 1;649 uint32_t constOne = 1;
650 AscendC::MicroAPI::RegTensor<float> aReg, bReg, cReg, sumReg;650 AscendC::MicroAPI::RegTensor<float> aReg, bReg, cReg, sumReg;
651 651 
652- AscendC::MicroAPI::UnalignReg UReg;652+ AscendC::MicroAPI::UnalignRegForStore UReg;
653 AscendC::MicroAPI::MaskReg pMask = AscendC::MicroAPI::UpdateMask<float>(count);653 AscendC::MicroAPI::MaskReg pMask = AscendC::MicroAPI::UpdateMask<float>(count);
654 AscendC::MicroAPI::MaskReg maskOne = AscendC::MicroAPI::UpdateMask<float>(constOne);654 AscendC::MicroAPI::MaskReg maskOne = AscendC::MicroAPI::UpdateMask<float>(constOne);
655 AscendC::MicroAPI::MaskReg655 AscendC::MicroAPI::MaskReg
656 pFull = AscendC::MicroAPI::CreateMask<float, AscendC::MicroAPI::MaskPattern::ALL>();656 pFull = AscendC::MicroAPI::CreateMask<float, AscendC::MicroAPI::MaskPattern::ALL>();
657 for (uint16_t i = 0; i < loopTimes; ++i) {657 for (uint16_t i = 0; i < loopTimes; ++i) {
658- AscendC::MicroAPI::DataCopy(aReg, (__local_mem__ float*)src0 + i * stride);658+ AscendC::MicroAPI::LoadAlign(aReg, (__ubuf__ float*)src0 + i * stride);
659- AscendC::MicroAPI::DataCopy(bReg, (__local_mem__ float*)src1 + i * stride);659+ AscendC::MicroAPI::LoadAlign(bReg, (__ubuf__ float*)src1 + i * stride);
660 AscendC::MicroAPI::Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(cReg, aReg, bReg, pMask);660 AscendC::MicroAPI::Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(cReg, aReg, bReg, pMask);
661- AscendC::MicroAPI::Copy<float, AscendC::MicroAPI::MaskMergeMode::MERGING>(aReg, cReg, pMask);661+ AscendC::MicroAPI::Move<float, AscendC::MicroAPI::MaskMergeMode::MERGING>(aReg, cReg, pMask);
662- AscendC::MicroAPI::ReduceSum(bReg, aReg, pFull);662+ AscendC::MicroAPI::Reduce<ReduceType::SUM>(bReg, aReg, pFull);
663- AscendC::MicroAPI::DataCopy(sumReg, dst);663+ AscendC::MicroAPI::LoadAlign(sumReg, dst);
664 AscendC::MicroAPI::Add(bReg, bReg, sumReg, maskOne);664 AscendC::MicroAPI::Add(bReg, bReg, sumReg, maskOne);
665- AscendC::MicroAPI::DataCopyUnAlign((__local_mem__ float*&)dst, bReg, UReg, 1);665+ AscendC::MicroAPI::StoreUnAlign((__ubuf__ float*&)dst, bReg, UReg, 1);
666 }666 }
667- AscendC::MicroAPI::DataCopyUnAlignPost((__local_mem__ float*&)dst, UReg, 0);667+ AscendC::MicroAPI::StoreUnAlignPost((__ubuf__ float*&)dst, UReg, 0);
668 }668 }
669 }669 }
670}670}
671 671 
672} // namespace SoftmaxGradExt672} // namespace SoftmaxGradExt
673-#endif // SOFTMAX_GRAD_EXT_AR_RECOMPUTE_H673+#endif // SOFTMAX_GRAD_EXT_AR_RECOMPUTE_H
Mactivation/softmax_grad_ext/op_kernel/arch35/softmax_grad_ext_ar_small_r.h+33-33
@@ -240,24 +240,24 @@ __aicore__ inline void SoftmaxGradExtARSmallR<T>::CalcReduceSum(const __ubuf__ T
240 for (uint16_t k = 0; k < loopA0Num; k++) {240 for (uint16_t k = 0; k < loopA0Num; k++) {
241 pregMask = MicroAPI::UpdateMask<float>(sreg);241 pregMask = MicroAPI::UpdateMask<float>(sreg);
242 uint32_t xOffset = i * aLen + k * VL_FP32;242 uint32_t xOffset = i * aLen + k * VL_FP32;
243- MicroAPI::DataCopy<float, MicroAPI::LoadDist::DIST_NORM>(vregSum,243+ MicroAPI::LoadAlign<float, MicroAPI::LoadDist::DIST_NORM>(vregSum,
244- (__ubuf__ float*)xSumLocalAddr + k * VL_FP32);244+ (__ubuf__ float*)xSumLocalAddr + k * VL_FP32);
245 if constexpr (xIsFp32_) {245 if constexpr (xIsFp32_) {
246- MicroAPI::DataCopy<float, MicroAPI::LoadDist::DIST_NORM>(vregX0,246+ MicroAPI::LoadAlign<float, MicroAPI::LoadDist::DIST_NORM>(vregX0,
247- (__ubuf__ float*)gradLocalAddr + xOffset);247+ (__ubuf__ float*)gradLocalAddr + xOffset);
248- MicroAPI::DataCopy<float, MicroAPI::LoadDist::DIST_NORM>(vregX1,248+ MicroAPI::LoadAlign<float, MicroAPI::LoadDist::DIST_NORM>(vregX1,
249- (__ubuf__ float*)x1LocalAddr + xOffset);249+ (__ubuf__ float*)x1LocalAddr + xOffset);
250 } else { // fp16, bf16250 } else { // fp16, bf16
251- MicroAPI::DataCopy<T, MicroAPI::LoadDist::DIST_UNPACK_B16>(vregX0B16,251+ MicroAPI::LoadAlign<T, MicroAPI::LoadDist::DIST_UNPACK_B16>(vregX0B16,
252- ((__ubuf__ T*)gradLocalAddr + xOffset));252+ ((__ubuf__ T*)gradLocalAddr + xOffset));
253- MicroAPI::DataCopy<T, MicroAPI::LoadDist::DIST_UNPACK_B16>(vregX1B16,253+ MicroAPI::LoadAlign<T, MicroAPI::LoadDist::DIST_UNPACK_B16>(vregX1B16,
254- ((__ubuf__ T*)x1LocalAddr + xOffset));254+ ((__ubuf__ T*)x1LocalAddr + xOffset));
255 MicroAPI::Cast<float, T, castTraitFp16ToFp32>(vregX0, vregX0B16, pregMask);255 MicroAPI::Cast<float, T, castTraitFp16ToFp32>(vregX0, vregX0B16, pregMask);
256 MicroAPI::Cast<float, T, castTraitFp16ToFp32>(vregX1, vregX1B16, pregMask);256 MicroAPI::Cast<float, T, castTraitFp16ToFp32>(vregX1, vregX1B16, pregMask);
257 }257 }
258 MicroAPI::MulAddDst(vregSum, vregX0, vregX1, pregMask);258 MicroAPI::MulAddDst(vregSum, vregX0, vregX1, pregMask);
259- MicroAPI::DataCopy<float, MicroAPI::StoreDist::DIST_NORM>((__ubuf__ float*)xSumLocalAddr + k * VL_FP32,259+ MicroAPI::StoreAlign<float, MicroAPI::StoreDist::DIST_NORM>(
260- vregSum, pregMask);260+ (__ubuf__ float*)xSumLocalAddr + k * VL_FP32, vregSum, pregMask);
261 }261 }
262 }262 }
263 }263 }
@@ -295,22 +295,22 @@ __aicore__ inline void SoftmaxGradExtARSmallR<T>::CalcOutput(const __ubuf__ T* g
295 for (uint16_t k = 0; k < loopA0Num; k++) {295 for (uint16_t k = 0; k < loopA0Num; k++) {
296 pregMask = MicroAPI::UpdateMask<float>(sreg);296 pregMask = MicroAPI::UpdateMask<float>(sreg);
297 uint32_t xOffset = i * aLen + k * VL_FP32;297 uint32_t xOffset = i * aLen + k * VL_FP32;
298- MicroAPI::DataCopy<float, MicroAPI::LoadDist::DIST_NORM>(vregSum,298+ MicroAPI::LoadAlign<float, MicroAPI::LoadDist::DIST_NORM>(vregSum,
299- (__ubuf__ float*)xSumLocalAddr + k * VL_FP32);299+ (__ubuf__ float*)xSumLocalAddr + k * VL_FP32);
300 if constexpr (xIsFp32_) {300 if constexpr (xIsFp32_) {
301- MicroAPI::DataCopy<float, MicroAPI::LoadDist::DIST_NORM>(vregX0,301+ MicroAPI::LoadAlign<float, MicroAPI::LoadDist::DIST_NORM>(vregX0,
302- (__ubuf__ float*)gradLocalAddr + xOffset);302+ (__ubuf__ float*)gradLocalAddr + xOffset);
303- MicroAPI::DataCopy<float, MicroAPI::LoadDist::DIST_NORM>(vregX1,303+ MicroAPI::LoadAlign<float, MicroAPI::LoadDist::DIST_NORM>(vregX1,
304- (__ubuf__ float*)x1LocalAddr + xOffset);304+ (__ubuf__ float*)x1LocalAddr + xOffset);
305- MicroAPI::DataCopy<float, MicroAPI::LoadDist::DIST_NORM>(vregX2,305+ MicroAPI::LoadAlign<float, MicroAPI::LoadDist::DIST_NORM>(vregX2,
306- (__ubuf__ float*)x2LocalAddr + xOffset);306+ (__ubuf__ float*)x2LocalAddr + xOffset);
307 } else {307 } else {
308- MicroAPI::DataCopy<T, MicroAPI::LoadDist::DIST_UNPACK_B16>(vregX0B16,308+ MicroAPI::LoadAlign<T, MicroAPI::LoadDist::DIST_UNPACK_B16>(vregX0B16,
309- ((__ubuf__ T*)gradLocalAddr + xOffset));309+ ((__ubuf__ T*)gradLocalAddr + xOffset));
310- MicroAPI::DataCopy<T, MicroAPI::LoadDist::DIST_UNPACK_B16>(vregX1B16,310+ MicroAPI::LoadAlign<T, MicroAPI::LoadDist::DIST_UNPACK_B16>(vregX1B16,
311- ((__ubuf__ T*)x1LocalAddr + xOffset));311+ ((__ubuf__ T*)x1LocalAddr + xOffset));
312- MicroAPI::DataCopy<T, MicroAPI::LoadDist::DIST_UNPACK_B16>(vregX2B16,312+ MicroAPI::LoadAlign<T, MicroAPI::LoadDist::DIST_UNPACK_B16>(vregX2B16,
313- ((__ubuf__ T*)x2LocalAddr + xOffset));313+ ((__ubuf__ T*)x2LocalAddr + xOffset));
314 MicroAPI::Cast<float, T, castTraitFp16ToFp32>(vregX0, vregX0B16, pregMask);314 MicroAPI::Cast<float, T, castTraitFp16ToFp32>(vregX0, vregX0B16, pregMask);
315 MicroAPI::Cast<float, T, castTraitFp16ToFp32>(vregX1, vregX1B16, pregMask);315 MicroAPI::Cast<float, T, castTraitFp16ToFp32>(vregX1, vregX1B16, pregMask);
316 MicroAPI::Cast<float, T, castTraitFp16ToFp32>(vregX2, vregX2B16, pregMask);316 MicroAPI::Cast<float, T, castTraitFp16ToFp32>(vregX2, vregX2B16, pregMask);
@@ -320,11 +320,11 @@ __aicore__ inline void SoftmaxGradExtARSmallR<T>::CalcOutput(const __ubuf__ T* g
320 MicroAPI::Sub(vregSub, vregMul0, vregMul1, pregMask);320 MicroAPI::Sub(vregSub, vregMul0, vregMul1, pregMask);
321 MicroAPI::Mul(vregResult, vregSub, vregX2, pregMask);321 MicroAPI::Mul(vregResult, vregSub, vregX2, pregMask);
322 if constexpr (xIsFp32_) {322 if constexpr (xIsFp32_) {
323- MicroAPI::DataCopy((__ubuf__ float*)yTempLocalAddr + xOffset, vregResult, pregMask);323+ MicroAPI::StoreAlign((__ubuf__ float*)yTempLocalAddr + xOffset, vregResult, pregMask);
324 } else { // fp16、bf16324 } else { // fp16、bf16
325 MicroAPI::Cast<T, float, castTraitFp32ToFp16>(vregResultB16, vregResult, pregMask);325 MicroAPI::Cast<T, float, castTraitFp32ToFp16>(vregResultB16, vregResult, pregMask);
326- MicroAPI::DataCopy<T, MicroAPI::StoreDist::DIST_PACK_B32>((__ubuf__ T*)yTempLocalAddr + xOffset,326+ MicroAPI::StoreAlign<T, MicroAPI::StoreDist::DIST_PACK_B32>((__ubuf__ T*)yTempLocalAddr + xOffset,
327- vregResultB16, pregMask);327+ vregResultB16, pregMask);
328 }328 }
329 }329 }
330 }330 }
@@ -406,9 +406,9 @@ __aicore__ inline void SoftmaxGradExtARSmallR<T>::CalcTransposeB16(LocalTensor<T
406template <typename T>406template <typename T>
407__aicore__ inline void SoftmaxGradExtARSmallR<T>::CopyInAndTransPose(int64_t xGmOffset, uint32_t curTileALen)407__aicore__ inline void SoftmaxGradExtARSmallR<T>::CopyInAndTransPose(int64_t xGmOffset, uint32_t curTileALen)
408{408{
409- static constexpr MultiCopyConfig config = {false};409+ static constexpr NdDmaConfig config = {false};
410 uint32_t aAligned = ops::CeilAlign(curTileALen, static_cast<uint32_t>(aTileBase_));410 uint32_t aAligned = ops::CeilAlign(curTileALen, static_cast<uint32_t>(aTileBase_));
411- MultiCopyLoopInfo<CONST_TWO> copyLoopInfo;411+ NdDmaLoopInfo<CONST_TWO> copyLoopInfo;
412 copyLoopInfo.loopSrcStride[0] = 1;412 copyLoopInfo.loopSrcStride[0] = 1;
413 copyLoopInfo.loopSrcStride[1] = rLen_;413 copyLoopInfo.loopSrcStride[1] = rLen_;
414 copyLoopInfo.loopDstStride[0] = aAligned;414 copyLoopInfo.loopDstStride[0] = aAligned;
@@ -417,7 +417,7 @@ __aicore__ inline void SoftmaxGradExtARSmallR<T>::CopyInAndTransPose(int64_t xGm
417 copyLoopInfo.loopSize[1] = curTileALen;417 copyLoopInfo.loopSize[1] = curTileALen;
418 copyLoopInfo.loopRpSize[0] = aAligned - curTileALen;418 copyLoopInfo.loopRpSize[0] = aAligned - curTileALen;
419 copyLoopInfo.loopRpSize[1] = 0;419 copyLoopInfo.loopRpSize[1] = 0;
420- MultiCopyParams<T, CONST_TWO> params = {copyLoopInfo, 0};420+ NdDmaParams<T, CONST_TWO> params = {copyLoopInfo, 0};
421 421 
422 LocalTensor<T> gradLocal_ = gradQueue_.AllocTensor<T>();422 LocalTensor<T> gradLocal_ = gradQueue_.AllocTensor<T>();
423 DataCopy<T, CONST_TWO, config>(gradLocal_, gradGm_[xGmOffset], params);423 DataCopy<T, CONST_TWO, config>(gradLocal_, gradGm_[xGmOffset], params);
@@ -457,4 +457,4 @@ __aicore__ inline void SoftmaxGradExtARSmallR<T>::CopyOutY(int64_t yGmOffset, ui
457 457 
458} // namespace SoftmaxGradExt458} // namespace SoftmaxGradExt
459 459 
460-#endif460+#endif
Mactivation/softmax_grad_ext/op_kernel/arch35/softmax_grad_ext_base.h+92-92
@@ -112,40 +112,40 @@ __aicore__ inline void SoftmaxGradExtBase::LastReduceSumSmallR(const LocalTensor
112 if (rSize <= VL_FP32) {112 if (rSize <= VL_FP32) {
113 __VEC_SCOPE__113 __VEC_SCOPE__
114 {114 {
115- __local_mem__ float* dst = (__local_mem__ float*)dstTensor.GetPhyAddr();115+ __ubuf__ float* dst = (__ubuf__ float*)dstTensor.GetPhyAddr();
116- __local_mem__ float* src = (__local_mem__ float*)srcTensor.GetPhyAddr();116+ __ubuf__ float* src = (__ubuf__ float*)srcTensor.GetPhyAddr();
117 uint32_t count = static_cast<uint32_t>(rSize);117 uint32_t count = static_cast<uint32_t>(rSize);
118 AscendC::MicroAPI::RegTensor<float> aReg, bReg;118 AscendC::MicroAPI::RegTensor<float> aReg, bReg;
119 AscendC::MicroAPI::MaskReg pMask = AscendC::MicroAPI::UpdateMask<float>(count);119 AscendC::MicroAPI::MaskReg pMask = AscendC::MicroAPI::UpdateMask<float>(count);
120- AscendC::MicroAPI::UnalignReg UReg;120+ AscendC::MicroAPI::UnalignRegForStore UReg;
121 for (uint16_t i = 0; i < loopTimes; ++i) {121 for (uint16_t i = 0; i < loopTimes; ++i) {
122- DataCopy(aReg, (__local_mem__ float*)src + i * stride);122+ Reg::LoadAlign(aReg, (__ubuf__ float*)src + i * stride);
123- ReduceSum(bReg, aReg, pMask);123+ Reduce<ReduceType::SUM>(bReg, aReg, pMask);
124- AscendC::MicroAPI::DataCopyUnAlign((__local_mem__ float*&)dst, bReg, UReg, 1);124+ AscendC::MicroAPI::StoreUnAlign((__ubuf__ float*&)dst, bReg, UReg, 1);
125 }125 }
126- AscendC::MicroAPI::DataCopyUnAlignPost((__local_mem__ float*&)dst, UReg, 0);126+ AscendC::MicroAPI::StoreUnAlignPost((__ubuf__ float*&)dst, UReg, 0);
127 }127 }
128 } else {128 } else {
129 __VEC_SCOPE__129 __VEC_SCOPE__
130 {130 {
131- __local_mem__ float* dst = (__local_mem__ float*)dstTensor.GetPhyAddr();131+ __ubuf__ float* dst = (__ubuf__ float*)dstTensor.GetPhyAddr();
132- __local_mem__ float* src0 = (__local_mem__ float*)srcTensor.GetPhyAddr();132+ __ubuf__ float* src0 = (__ubuf__ float*)srcTensor.GetPhyAddr();
133- __local_mem__ float* src1 = (__local_mem__ float*)srcTensor.GetPhyAddr() + VL_FP32;133+ __ubuf__ float* src1 = (__ubuf__ float*)srcTensor.GetPhyAddr() + VL_FP32;
134 uint32_t count = static_cast<uint32_t>(rSize - VL_FP32);134 uint32_t count = static_cast<uint32_t>(rSize - VL_FP32);
135 AscendC::MicroAPI::RegTensor<float> aReg, bReg, cReg;135 AscendC::MicroAPI::RegTensor<float> aReg, bReg, cReg;
136- AscendC::MicroAPI::UnalignReg UReg;136+ AscendC::MicroAPI::UnalignRegForStore UReg;
137 AscendC::MicroAPI::MaskReg pMask = AscendC::MicroAPI::UpdateMask<float>(count);137 AscendC::MicroAPI::MaskReg pMask = AscendC::MicroAPI::UpdateMask<float>(count);
138 AscendC::MicroAPI::MaskReg138 AscendC::MicroAPI::MaskReg
139 pFull = AscendC::MicroAPI::CreateMask<float, AscendC::MicroAPI::MaskPattern::ALL>();139 pFull = AscendC::MicroAPI::CreateMask<float, AscendC::MicroAPI::MaskPattern::ALL>();
140 for (uint16_t i = 0; i < loopTimes; ++i) {140 for (uint16_t i = 0; i < loopTimes; ++i) {
141- DataCopy(aReg, (__local_mem__ float*)src0 + i * stride);141+ Reg::LoadAlign(aReg, (__ubuf__ float*)src0 + i * stride);
142- DataCopy(bReg, (__local_mem__ float*)src1 + i * stride);142+ Reg::LoadAlign(bReg, (__ubuf__ float*)src1 + i * stride);
143 Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(cReg, aReg, bReg, pMask);143 Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(cReg, aReg, bReg, pMask);
144- Copy<float, AscendC::MicroAPI::MaskMergeMode::MERGING>(aReg, cReg, pMask);144+ Move<float, AscendC::MicroAPI::MaskMergeMode::MERGING>(aReg, cReg, pMask);
145- ReduceSum(bReg, aReg, pFull);145+ Reduce<ReduceType::SUM>(bReg, aReg, pFull);
146- AscendC::MicroAPI::DataCopyUnAlign((__local_mem__ float*&)dst, bReg, UReg, 1);146+ AscendC::MicroAPI::StoreUnAlign((__ubuf__ float*&)dst, bReg, UReg, 1);
147 }147 }
148- AscendC::MicroAPI::DataCopyUnAlignPost((__local_mem__ float*&)dst, UReg, 0);148+ AscendC::MicroAPI::StoreUnAlignPost((__ubuf__ float*&)dst, UReg, 0);
149 }149 }
150 }150 }
151}151}
@@ -190,43 +190,43 @@ __aicore__ inline void SoftmaxGradExtBase::LastReduceSum(const LocalTensor<float
190 190 
191 __VEC_SCOPE__191 __VEC_SCOPE__
192 {192 {
193- __local_mem__ float* dst = (__local_mem__ float*)reduceSumTempTensor.GetPhyAddr();193+ __ubuf__ float* dst = (__ubuf__ float*)reduceSumTempTensor.GetPhyAddr();
194- __local_mem__ float* foldSrcA = (__local_mem__ float*)srcTensor.GetPhyAddr();194+ __ubuf__ float* foldSrcA = (__ubuf__ float*)srcTensor.GetPhyAddr();
195- __local_mem__ float* foldSrcB = (__local_mem__ float*)srcTensor.GetPhyAddr() + foldSrcBOffset;195+ __ubuf__ float* foldSrcB = (__ubuf__ float*)srcTensor.GetPhyAddr() + foldSrcBOffset;
196- __local_mem__ float* tailSrcA = (__local_mem__ float*)srcTensor.GetPhyAddr() + tailSrcAOffset;196+ __ubuf__ float* tailSrcA = (__ubuf__ float*)srcTensor.GetPhyAddr() + tailSrcAOffset;
197- __local_mem__ float* tailSrcB = (__local_mem__ float*)srcTensor.GetPhyAddr() + tailSrcBOffset;197+ __ubuf__ float* tailSrcB = (__ubuf__ float*)srcTensor.GetPhyAddr() + tailSrcBOffset;
198- __local_mem__ float* unFoldSrc = (__local_mem__ float*)srcTensor.GetPhyAddr() + unFoldSrcOffset;198+ __ubuf__ float* unFoldSrc = (__ubuf__ float*)srcTensor.GetPhyAddr() + unFoldSrcOffset;
199 AscendC::MicroAPI::MaskReg pFull = AscendC::MicroAPI::CreateMask<float, AscendC::MicroAPI::MaskPattern::ALL>();199 AscendC::MicroAPI::MaskReg pFull = AscendC::MicroAPI::CreateMask<float, AscendC::MicroAPI::MaskPattern::ALL>();
200- AscendC::MicroAPI::UnalignReg UReg;200+ AscendC::MicroAPI::UnalignRegForStore UReg;
201 201 
202 for (uint16_t i = 0; i < outerLoopTimes; ++i) {202 for (uint16_t i = 0; i < outerLoopTimes; ++i) {
203- dst = (__local_mem__ float*)reduceSumTempTensor.GetPhyAddr() + i * outerLoopDstStride;203+ dst = (__ubuf__ float*)reduceSumTempTensor.GetPhyAddr() + i * outerLoopDstStride;
204 for (uint16_t j = 0; j < mainFoldLoopTimes; ++j) {204 for (uint16_t j = 0; j < mainFoldLoopTimes; ++j) {
205 AscendC::MicroAPI::RegTensor<float> aReg, bReg, cReg, dReg;205 AscendC::MicroAPI::RegTensor<float> aReg, bReg, cReg, dReg;
206- DataCopy(aReg, (__local_mem__ float*)foldSrcA + i * outerLoopStride + j * innerLoopStride);206+ Reg::LoadAlign(aReg, (__ubuf__ float*)foldSrcA + i * outerLoopStride + j * innerLoopStride);
207- DataCopy(bReg, (__local_mem__ float*)foldSrcB + i * outerLoopStride + j * innerLoopStride);207+ Reg::LoadAlign(bReg, (__ubuf__ float*)foldSrcB + i * outerLoopStride + j * innerLoopStride);
208 Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(cReg, aReg, bReg, pFull);208 Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(cReg, aReg, bReg, pFull);
209- ReduceSum(dReg, cReg, pFull);209+ Reduce<ReduceType::SUM>(dReg, cReg, pFull);
210- AscendC::MicroAPI::DataCopyUnAlign((__local_mem__ float*&)dst, dReg, UReg, 1);210+ AscendC::MicroAPI::StoreUnAlign((__ubuf__ float*&)dst, dReg, UReg, 1);
211 }211 }
212 for (uint16_t j = 0; j < tailFoldLoopTimes; ++j) {212 for (uint16_t j = 0; j < tailFoldLoopTimes; ++j) {
213 uint32_t count = static_cast<uint32_t>(tailFoldElemCount);213 uint32_t count = static_cast<uint32_t>(tailFoldElemCount);
214 AscendC::MicroAPI::RegTensor<float> aReg, bReg, cReg;214 AscendC::MicroAPI::RegTensor<float> aReg, bReg, cReg;
215 AscendC::MicroAPI::MaskReg pMask = AscendC::MicroAPI::UpdateMask<float>(count);215 AscendC::MicroAPI::MaskReg pMask = AscendC::MicroAPI::UpdateMask<float>(count);
216- DataCopy(aReg, (__local_mem__ float*)tailSrcA + i * outerLoopStride + j * innerLoopStride);216+ Reg::LoadAlign(aReg, (__ubuf__ float*)tailSrcA + i * outerLoopStride + j * innerLoopStride);
217- DataCopy(bReg, (__local_mem__ float*)tailSrcB + i * outerLoopStride + j * innerLoopStride);217+ Reg::LoadAlign(bReg, (__ubuf__ float*)tailSrcB + i * outerLoopStride + j * innerLoopStride);
218 Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(cReg, aReg, bReg, pMask);218 Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(cReg, aReg, bReg, pMask);
219- Copy<float, AscendC::MicroAPI::MaskMergeMode::MERGING>(aReg, cReg, pMask);219+ Move<float, AscendC::MicroAPI::MaskMergeMode::MERGING>(aReg, cReg, pMask);
220- ReduceSum(bReg, aReg, pFull);220+ Reduce<ReduceType::SUM>(bReg, aReg, pFull);
221- AscendC::MicroAPI::DataCopyUnAlign((__local_mem__ float*&)dst, bReg, UReg, 1);221+ AscendC::MicroAPI::StoreUnAlign((__ubuf__ float*&)dst, bReg, UReg, 1);
222 }222 }
223 for (uint16_t j = 0; j < unFoldLoopTimes; ++j) {223 for (uint16_t j = 0; j < unFoldLoopTimes; ++j) {
224 AscendC::MicroAPI::RegTensor<float> aReg, bReg;224 AscendC::MicroAPI::RegTensor<float> aReg, bReg;
225- DataCopy(aReg, (__local_mem__ float*)unFoldSrc + i * outerLoopStride + j * innerLoopStride);225+ Reg::LoadAlign(aReg, (__ubuf__ float*)unFoldSrc + i * outerLoopStride + j * innerLoopStride);
226- ReduceSum(bReg, aReg, pFull);226+ Reduce<ReduceType::SUM>(bReg, aReg, pFull);
227- AscendC::MicroAPI::DataCopyUnAlign((__local_mem__ float*&)dst, bReg, UReg, 1);227+ AscendC::MicroAPI::StoreUnAlign((__ubuf__ float*&)dst, bReg, UReg, 1);
228 }228 }
229- AscendC::MicroAPI::DataCopyUnAlignPost((__local_mem__ float*&)dst, UReg, 0);229+ AscendC::MicroAPI::StoreUnAlignPost((__ubuf__ float*&)dst, UReg, 0);
230 }230 }
231 }231 }
232 LastReduceSumSmallR(dstTensor, reduceSumTempTensor, aSize, foldPoint, outerLoopDstStride);232 LastReduceSumSmallR(dstTensor, reduceSumTempTensor, aSize, foldPoint, outerLoopDstStride);
@@ -234,13 +234,13 @@ __aicore__ inline void SoftmaxGradExtBase::LastReduceSum(const LocalTensor<float
234 234 
235template <uint32_t RSize, int32_t TailCount = -1, int32_t Index = 0, int32_t Depth = 1>235template <uint32_t RSize, int32_t TailCount = -1, int32_t Index = 0, int32_t Depth = 1>
236struct NlastDichotomyAdd {236struct NlastDichotomyAdd {
237- __aicore__ static inline void LoadAndAccumulate(AscendC::MicroAPI::RegTensor<float>& acc,237+ __aicore__ static inline void LoadAndAccumulate(AscendC::MicroAPI::RegTensor<float>& acc, __ubuf__ float*& srcA,
238- __local_mem__ float*& srcA, __local_mem__ float*& srcB,238+ __ubuf__ float*& srcB, AscendC::MicroAPI::MaskReg& pMask,
239- AscendC::MicroAPI::MaskReg& pMask, uint32_t stride)239+ uint32_t stride)
240 {240 {
241 AscendC::MicroAPI::RegTensor<float> aReg, bReg;241 AscendC::MicroAPI::RegTensor<float> aReg, bReg;
242- __local_mem__ float* srcAOffset = srcA + stride * CONST_TWO;242+ __ubuf__ float* srcAOffset = srcA + stride * CONST_TWO;
243- __local_mem__ float* srcBOffset = srcB + stride * CONST_TWO;243+ __ubuf__ float* srcBOffset = srcB + stride * CONST_TWO;
244 if constexpr (TailCount <= 0) {244 if constexpr (TailCount <= 0) {
245 NlastDichotomyAdd<(RSize + 1) / CONST_TWO>::LoadAndAccumulate(aReg, srcA, srcAOffset, pMask,245 NlastDichotomyAdd<(RSize + 1) / CONST_TWO>::LoadAndAccumulate(aReg, srcA, srcAOffset, pMask,
246 stride * CONST_TWO);246 stride * CONST_TWO);
@@ -248,13 +248,13 @@ struct NlastDichotomyAdd {
248 }248 }
249 Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(acc, aReg, bReg, pMask);249 Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(acc, aReg, bReg, pMask);
250 }250 }
251- __aicore__ static inline void LoadAndAccumulate(AscendC::MicroAPI::RegTensor<float>& acc,251+ __aicore__ static inline void LoadAndAccumulate(AscendC::MicroAPI::RegTensor<float>& acc, __ubuf__ float*& srcA,
252- __local_mem__ float*& srcA, __local_mem__ float*& srcB,252+ __ubuf__ float*& srcB, AscendC::MicroAPI::MaskReg& pMask,
253- AscendC::MicroAPI::MaskReg& pMask, uint32_t stride, uint32_t offset)253+ uint32_t stride, uint32_t offset)
254 {254 {
255 AscendC::MicroAPI::RegTensor<float> aReg, bReg;255 AscendC::MicroAPI::RegTensor<float> aReg, bReg;
256- __local_mem__ float* srcAOffset = srcA + stride * CONST_TWO;256+ __ubuf__ float* srcAOffset = srcA + stride * CONST_TWO;
257- __local_mem__ float* srcBOffset = srcB + stride * CONST_TWO;257+ __ubuf__ float* srcBOffset = srcB + stride * CONST_TWO;
258 if constexpr (TailCount <= 0) {258 if constexpr (TailCount <= 0) {
259 NlastDichotomyAdd<(RSize + 1) / CONST_TWO>::LoadAndAccumulate(aReg, srcA, srcAOffset, pMask,259 NlastDichotomyAdd<(RSize + 1) / CONST_TWO>::LoadAndAccumulate(aReg, srcA, srcAOffset, pMask,
260 stride * CONST_TWO, offset);260 stride * CONST_TWO, offset);
@@ -272,49 +272,49 @@ struct NlastDichotomyAdd {
272 272 
273template <int32_t TailCount, int32_t Index, int32_t Depth>273template <int32_t TailCount, int32_t Index, int32_t Depth>
274struct NlastDichotomyAdd<CONST_TWO, TailCount, Index, Depth> {274struct NlastDichotomyAdd<CONST_TWO, TailCount, Index, Depth> {
275- __aicore__ static inline void LoadAndAccumulate(AscendC::MicroAPI::RegTensor<float>& acc,275+ __aicore__ static inline void LoadAndAccumulate(AscendC::MicroAPI::RegTensor<float>& acc, __ubuf__ float*& srcA,
276- __local_mem__ float*& srcA, __local_mem__ float*& srcB,276+ __ubuf__ float*& srcB, AscendC::MicroAPI::MaskReg& pMask,
277- AscendC::MicroAPI::MaskReg& pMask, uint32_t stride)277+ uint32_t stride)
278 {278 {
279 AscendC::MicroAPI::RegTensor<float> aReg, bReg;279 AscendC::MicroAPI::RegTensor<float> aReg, bReg;
280- DataCopy(aReg, (__local_mem__ float*)srcA);280+ Reg::LoadAlign(aReg, (__ubuf__ float*)srcA);
281- DataCopy(bReg, (__local_mem__ float*)srcB);281+ Reg::LoadAlign(bReg, (__ubuf__ float*)srcB);
282 Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(acc, aReg, bReg, pMask);282 Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(acc, aReg, bReg, pMask);
283 }283 }
284- __aicore__ static inline void LoadAndAccumulate(AscendC::MicroAPI::RegTensor<float>& acc,284+ __aicore__ static inline void LoadAndAccumulate(AscendC::MicroAPI::RegTensor<float>& acc, __ubuf__ float*& srcA,
285- __local_mem__ float*& srcA, __local_mem__ float*& srcB,285+ __ubuf__ float*& srcB, AscendC::MicroAPI::MaskReg& pMask,
286- AscendC::MicroAPI::MaskReg& pMask, uint32_t stride, uint32_t offset)286+ uint32_t stride, uint32_t offset)
287 {287 {
288 if constexpr (TailCount <= 0) {288 if constexpr (TailCount <= 0) {
289 AscendC::MicroAPI::RegTensor<float> aReg, bReg, cReg;289 AscendC::MicroAPI::RegTensor<float> aReg, bReg, cReg;
290- DataCopy(aReg, (__local_mem__ float*)srcA);290+ Reg::LoadAlign(aReg, (__ubuf__ float*)srcA);
291- DataCopy(bReg, (__local_mem__ float*)srcA + offset);291+ Reg::LoadAlign(bReg, (__ubuf__ float*)srcA + offset);
292 Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(aReg, aReg, bReg, pMask);292 Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(aReg, aReg, bReg, pMask);
293- DataCopy(bReg, (__local_mem__ float*)srcB);293+ Reg::LoadAlign(bReg, (__ubuf__ float*)srcB);
294- DataCopy(cReg, (__local_mem__ float*)srcB + offset);294+ Reg::LoadAlign(cReg, (__ubuf__ float*)srcB + offset);
295 Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(bReg, bReg, cReg, pMask);295 Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(bReg, bReg, cReg, pMask);
296 Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(acc, aReg, bReg, pMask);296 Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(acc, aReg, bReg, pMask);
297 } else {297 } else {
298 if constexpr (Index + Depth < TailCount) {298 if constexpr (Index + Depth < TailCount) {
299 AscendC::MicroAPI::RegTensor<float> aReg, bReg, cReg;299 AscendC::MicroAPI::RegTensor<float> aReg, bReg, cReg;
300- DataCopy(aReg, (__local_mem__ float*)srcA);300+ Reg::LoadAlign(aReg, (__ubuf__ float*)srcA);
301- DataCopy(bReg, (__local_mem__ float*)srcA + offset);301+ Reg::LoadAlign(bReg, (__ubuf__ float*)srcA + offset);
302 Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(aReg, aReg, bReg, pMask);302 Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(aReg, aReg, bReg, pMask);
303- DataCopy(bReg, (__local_mem__ float*)srcB);303+ Reg::LoadAlign(bReg, (__ubuf__ float*)srcB);
304- DataCopy(cReg, (__local_mem__ float*)srcB + offset);304+ Reg::LoadAlign(cReg, (__ubuf__ float*)srcB + offset);
305 Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(bReg, bReg, cReg, pMask);305 Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(bReg, bReg, cReg, pMask);
306 Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(acc, aReg, bReg, pMask);306 Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(acc, aReg, bReg, pMask);
307 } else if constexpr (Index < TailCount) {307 } else if constexpr (Index < TailCount) {
308 AscendC::MicroAPI::RegTensor<float> aReg, bReg;308 AscendC::MicroAPI::RegTensor<float> aReg, bReg;
309- DataCopy(aReg, (__local_mem__ float*)srcA);309+ Reg::LoadAlign(aReg, (__ubuf__ float*)srcA);
310- DataCopy(bReg, (__local_mem__ float*)srcA + offset);310+ Reg::LoadAlign(bReg, (__ubuf__ float*)srcA + offset);
311 Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(aReg, aReg, bReg, pMask);311 Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(aReg, aReg, bReg, pMask);
312- DataCopy(bReg, (__local_mem__ float*)srcB);312+ Reg::LoadAlign(bReg, (__ubuf__ float*)srcB);
313 Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(acc, aReg, bReg, pMask);313 Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(acc, aReg, bReg, pMask);
314 } else {314 } else {
315 AscendC::MicroAPI::RegTensor<float> aReg, bReg;315 AscendC::MicroAPI::RegTensor<float> aReg, bReg;
316- DataCopy(aReg, (__local_mem__ float*)srcA);316+ Reg::LoadAlign(aReg, (__ubuf__ float*)srcA);
317- DataCopy(bReg, (__local_mem__ float*)srcB);317+ Reg::LoadAlign(bReg, (__ubuf__ float*)srcB);
318 Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(acc, aReg, bReg, pMask);318 Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(acc, aReg, bReg, pMask);
319 }319 }
320 }320 }
@@ -323,25 +323,25 @@ struct NlastDichotomyAdd<CONST_TWO, TailCount, Index, Depth> {
323 323 
324template <>324template <>
325struct NlastDichotomyAdd<CONST_TWO> {325struct NlastDichotomyAdd<CONST_TWO> {
326- __aicore__ static inline void LoadAndAccumulate(AscendC::MicroAPI::RegTensor<float>& acc,326+ __aicore__ static inline void LoadAndAccumulate(AscendC::MicroAPI::RegTensor<float>& acc, __ubuf__ float*& srcA,
327- __local_mem__ float*& srcA, __local_mem__ float*& srcB,327+ __ubuf__ float*& srcB, AscendC::MicroAPI::MaskReg& pMask,
328- AscendC::MicroAPI::MaskReg& pMask, uint32_t stride)328+ uint32_t stride)
329 {329 {
330 AscendC::MicroAPI::RegTensor<float> aReg, bReg;330 AscendC::MicroAPI::RegTensor<float> aReg, bReg;
331- DataCopy(aReg, (__local_mem__ float*)srcA);331+ Reg::LoadAlign(aReg, (__ubuf__ float*)srcA);
332- DataCopy(bReg, (__local_mem__ float*)srcB);332+ Reg::LoadAlign(bReg, (__ubuf__ float*)srcB);
333 Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(acc, aReg, bReg, pMask);333 Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(acc, aReg, bReg, pMask);
334 }334 }
335- __aicore__ static inline void LoadAndAccumulate(AscendC::MicroAPI::RegTensor<float>& acc,335+ __aicore__ static inline void LoadAndAccumulate(AscendC::MicroAPI::RegTensor<float>& acc, __ubuf__ float*& srcA,
336- __local_mem__ float*& srcA, __local_mem__ float*& srcB,336+ __ubuf__ float*& srcB, AscendC::MicroAPI::MaskReg& pMask,
337- AscendC::MicroAPI::MaskReg& pMask, uint32_t stride, uint32_t offset)337+ uint32_t stride, uint32_t offset)
338 {338 {
339 AscendC::MicroAPI::RegTensor<float> aReg, bReg, cReg;339 AscendC::MicroAPI::RegTensor<float> aReg, bReg, cReg;
340- DataCopy(aReg, (__local_mem__ float*)srcA);340+ Reg::LoadAlign(aReg, (__ubuf__ float*)srcA);
341- DataCopy(bReg, (__local_mem__ float*)srcA + offset);341+ Reg::LoadAlign(bReg, (__ubuf__ float*)srcA + offset);
342 Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(aReg, aReg, bReg, pMask);342 Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(aReg, aReg, bReg, pMask);
343- DataCopy(bReg, (__local_mem__ float*)srcB);343+ Reg::LoadAlign(bReg, (__ubuf__ float*)srcB);
344- DataCopy(cReg, (__local_mem__ float*)srcB + offset);344+ Reg::LoadAlign(cReg, (__ubuf__ float*)srcB + offset);
345 Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(bReg, bReg, cReg, pMask);345 Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(bReg, bReg, cReg, pMask);
346 Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(acc, aReg, bReg, pMask);346 Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(acc, aReg, bReg, pMask);
347 }347 }
@@ -349,11 +349,11 @@ struct NlastDichotomyAdd<CONST_TWO> {
349 349 
350template <>350template <>
351struct NlastDichotomyAdd<1> {351struct NlastDichotomyAdd<1> {
352- __aicore__ static inline void LoadAndAccumulate(AscendC::MicroAPI::RegTensor<float>& acc,352+ __aicore__ static inline void LoadAndAccumulate(AscendC::MicroAPI::RegTensor<float>& acc, __ubuf__ float*& srcA,
353- __local_mem__ float*& srcA, __local_mem__ float*& srcB,353+ __ubuf__ float*& srcB, AscendC::MicroAPI::MaskReg& pMask,
354- AscendC::MicroAPI::MaskReg& pMask, uint32_t stride)354+ uint32_t stride)
355 {355 {
356- DataCopy(acc, (__local_mem__ float*)srcA);356+ Reg::LoadAlign(acc, (__ubuf__ float*)srcA);
357 }357 }
358};358};
359 359 
@@ -369,22 +369,22 @@ __aicore__ inline void SoftmaxGradExtBase::UpdateCache(const LocalTensor<float>&
369 uint32_t innerLoopStride = stride;369 uint32_t innerLoopStride = stride;
370 __VEC_SCOPE__370 __VEC_SCOPE__
371 {371 {
372- __local_mem__ float* dst = (__local_mem__ float*)dstTensor.GetPhyAddr();372+ __ubuf__ float* dst = (__ubuf__ float*)dstTensor.GetPhyAddr();
373- __local_mem__ float* cah = (__local_mem__ float*)dstTensor.GetPhyAddr() + cacheID * stride;373+ __ubuf__ float* cah = (__ubuf__ float*)dstTensor.GetPhyAddr() + cacheID * stride;
374- __local_mem__ float* src = (__local_mem__ float*)srcTensor.GetPhyAddr();374+ __ubuf__ float* src = (__ubuf__ float*)srcTensor.GetPhyAddr();
375 uint32_t sreg = static_cast<uint32_t>(count);375 uint32_t sreg = static_cast<uint32_t>(count);
376 AscendC::MicroAPI::RegTensor<float> aReg, bReg;376 AscendC::MicroAPI::RegTensor<float> aReg, bReg;
377 AscendC::MicroAPI::MaskReg pMask;377 AscendC::MicroAPI::MaskReg pMask;
378 for (uint16_t i = 0; i < outerLoopTimes; ++i) {378 for (uint16_t i = 0; i < outerLoopTimes; ++i) {
379 pMask = AscendC::MicroAPI::UpdateMask<float>(sreg);379 pMask = AscendC::MicroAPI::UpdateMask<float>(sreg);
380- DataCopy(aReg, (__local_mem__ float*)src + i * outerLoopStride);380+ Reg::LoadAlign(aReg, (__ubuf__ float*)src + i * outerLoopStride);
381 for (uint16_t j = 0; j < innerLoopTimes; ++j) {381 for (uint16_t j = 0; j < innerLoopTimes; ++j) {
382- DataCopy(bReg, (__local_mem__ float*)dst + i * outerLoopStride + j * innerLoopStride);382+ Reg::LoadAlign(bReg, (__ubuf__ float*)dst + i * outerLoopStride + j * innerLoopStride);
383 Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(aReg, aReg, bReg, pMask);383 Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(aReg, aReg, bReg, pMask);
384 }384 }
385- DataCopy((__local_mem__ float*)cah + i * outerLoopStride, aReg, pMask);385+ Reg::StoreAlign((__ubuf__ float*)cah + i * outerLoopStride, aReg, pMask);
386 }386 }
387 }387 }
388}388}
389} // namespace SoftmaxGradExt389} // namespace SoftmaxGradExt
390-#endif390+#endif
Mactivation/swi_glu_grad/op_kernel/arch35/swi_glu_grad_base.h+17-17
@@ -43,9 +43,9 @@ public:
43 __aicore__ inline bool InitBase(const GluBaseTilingData& tilingData);43 __aicore__ inline bool InitBase(const GluBaseTilingData& tilingData);
44 44 
45protected:45protected:
46- __aicore__ inline void LoadOneTensor(const __local_mem__ void* input, MicroAPI::RegTensor<float>& dst,46+ __aicore__ inline void LoadOneTensor(const __ubuf__ void* input, MicroAPI::RegTensor<float>& dst,
47 MicroAPI::MaskReg& preg, uint32_t offset);47 MicroAPI::MaskReg& preg, uint32_t offset);
48- __aicore__ inline void StoreOneTensor(const __local_mem__ void* output, MicroAPI::RegTensor<float>& src,48+ __aicore__ inline void StoreOneTensor(const __ubuf__ void* output, MicroAPI::RegTensor<float>& src,
49 MicroAPI::MaskReg& preg, uint32_t offset);49 MicroAPI::MaskReg& preg, uint32_t offset);
50 __aicore__ inline void Compute(LocalTensor<T>& xATensor, LocalTensor<T>& xBTensor, LocalTensor<T>& gradTensor,50 __aicore__ inline void Compute(LocalTensor<T>& xATensor, LocalTensor<T>& xBTensor, LocalTensor<T>& gradTensor,
51 LocalTensor<T>& outATensor, LocalTensor<T>& outBTensor, int64_t dataCount);51 LocalTensor<T>& outATensor, LocalTensor<T>& outBTensor, int64_t dataCount);
@@ -114,39 +114,39 @@ __aicore__ inline bool SwiGluGradBaseKernel<T>::InitBase(const GluBaseTilingData
114}114}
115 115 
116template <typename T>116template <typename T>
117-__aicore__ inline void SwiGluGradBaseKernel<T>::LoadOneTensor(const __local_mem__ void* input,117+__aicore__ inline void SwiGluGradBaseKernel<T>::LoadOneTensor(const __ubuf__ void* input,
118 MicroAPI::RegTensor<float>& dst, MicroAPI::MaskReg& preg,118 MicroAPI::RegTensor<float>& dst, MicroAPI::MaskReg& preg,
119 uint32_t offset)119 uint32_t offset)
120{120{
121 if constexpr (IsSameType<T, half>::value) {121 if constexpr (IsSameType<T, half>::value) {
122 MicroAPI::RegTensor<half> xFp16;122 MicroAPI::RegTensor<half> xFp16;
123- DataCopy<half, MicroAPI::LoadDist::DIST_UNPACK_B16>(xFp16, (__local_mem__ half*)(input) + offset);123+ Reg::LoadAlign<half, MicroAPI::LoadDist::DIST_UNPACK_B16>(xFp16, (__ubuf__ half*)(input) + offset);
124 Cast<float, half, castTraitB162B32>(dst, xFp16, preg);124 Cast<float, half, castTraitB162B32>(dst, xFp16, preg);
125 } else if constexpr (IsSameType<T, bfloat16_t>::value) {125 } else if constexpr (IsSameType<T, bfloat16_t>::value) {
126 MicroAPI::RegTensor<bfloat16_t> xBf16;126 MicroAPI::RegTensor<bfloat16_t> xBf16;
127- DataCopy<bfloat16_t, MicroAPI::LoadDist::DIST_UNPACK_B16>(xBf16, (__local_mem__ bfloat16_t*)(input) + offset);127+ Reg::LoadAlign<bfloat16_t, MicroAPI::LoadDist::DIST_UNPACK_B16>(xBf16, (__ubuf__ bfloat16_t*)(input) + offset);
128 Cast<float, bfloat16_t, castTraitB162B32>(dst, xBf16, preg);128 Cast<float, bfloat16_t, castTraitB162B32>(dst, xBf16, preg);
129 } else {129 } else {
130- DataCopy(dst, (__local_mem__ float*)(input) + offset);130+ Reg::LoadAlign(dst, (__ubuf__ float*)(input) + offset);
131 }131 }
132}132}
133 133 
134template <typename T>134template <typename T>
135-__aicore__ inline void SwiGluGradBaseKernel<T>::StoreOneTensor(const __local_mem__ void* output,135+__aicore__ inline void SwiGluGradBaseKernel<T>::StoreOneTensor(const __ubuf__ void* output,
136 MicroAPI::RegTensor<float>& src, MicroAPI::MaskReg& preg,136 MicroAPI::RegTensor<float>& src, MicroAPI::MaskReg& preg,
137 uint32_t offset)137 uint32_t offset)
138{138{
139 if constexpr (IsSameType<T, half>::value) {139 if constexpr (IsSameType<T, half>::value) {
140 MicroAPI::RegTensor<half> xFp16;140 MicroAPI::RegTensor<half> xFp16;
141 Cast<half, float, castTraitB322B16>(xFp16, src, preg);141 Cast<half, float, castTraitB322B16>(xFp16, src, preg);
142- DataCopy<half, MicroAPI::StoreDist::DIST_PACK_B32>((__local_mem__ half*)(output) + offset, xFp16, preg);142+ Reg::StoreAlign<half, MicroAPI::StoreDist::DIST_PACK_B32>((__ubuf__ half*)(output) + offset, xFp16, preg);
143 } else if constexpr (IsSameType<T, bfloat16_t>::value) {143 } else if constexpr (IsSameType<T, bfloat16_t>::value) {
144 MicroAPI::RegTensor<bfloat16_t> xBf16;144 MicroAPI::RegTensor<bfloat16_t> xBf16;
145 Cast<bfloat16_t, float, castTraitB322B16>(xBf16, src, preg);145 Cast<bfloat16_t, float, castTraitB322B16>(xBf16, src, preg);
146- DataCopy<bfloat16_t, MicroAPI::StoreDist::DIST_PACK_B32>((__local_mem__ bfloat16_t*)(output) + offset, xBf16,146+ Reg::StoreAlign<bfloat16_t, MicroAPI::StoreDist::DIST_PACK_B32>((__ubuf__ bfloat16_t*)(output) + offset, xBf16,
147- preg);147+ preg);
148 } else {148 } else {
149- DataCopy((__local_mem__ float*)(output) + offset, src, preg);149+ Reg::StoreAlign((__ubuf__ float*)(output) + offset, src, preg);
150 }150 }
151}151}
152 152 
@@ -155,11 +155,11 @@ __aicore__ inline void SwiGluGradBaseKernel<T>::Compute(LocalTensor<T>& xATensor
155 LocalTensor<T>& gradTensor, LocalTensor<T>& outATensor,155 LocalTensor<T>& gradTensor, LocalTensor<T>& outATensor,
156 LocalTensor<T>& outBTensor, int64_t dataCount)156 LocalTensor<T>& outBTensor, int64_t dataCount)
157{157{
158- __local_mem__ T* ubSrcAddrA = (__local_mem__ T*)xATensor.GetPhyAddr();158+ __ubuf__ T* ubSrcAddrA = (__ubuf__ T*)xATensor.GetPhyAddr();
159- __local_mem__ T* ubSrcAddrB = (__local_mem__ T*)xBTensor.GetPhyAddr();159+ __ubuf__ T* ubSrcAddrB = (__ubuf__ T*)xBTensor.GetPhyAddr();
160- __local_mem__ T* ubGradAddr = (__local_mem__ T*)gradTensor.GetPhyAddr();160+ __ubuf__ T* ubGradAddr = (__ubuf__ T*)gradTensor.GetPhyAddr();
161- __local_mem__ T* ubDstAddrA = (__local_mem__ T*)outATensor.GetPhyAddr();161+ __ubuf__ T* ubDstAddrA = (__ubuf__ T*)outATensor.GetPhyAddr();
162- __local_mem__ T* ubDstAddrB = (__local_mem__ T*)outBTensor.GetPhyAddr();162+ __ubuf__ T* ubDstAddrB = (__ubuf__ T*)outBTensor.GetPhyAddr();
163 163 
164 uint16_t repeatTimes = (dataCount + vfFp32Block_ - 1) / vfFp32Block_;164 uint16_t repeatTimes = (dataCount + vfFp32Block_ - 1) / vfFp32Block_;
165 165 
@@ -211,4 +211,4 @@ __aicore__ inline void SwiGluGradBaseKernel<T>::Compute(LocalTensor<T>& xATensor
211 211 
212} // namespace SwiGluGrad212} // namespace SwiGluGrad
213 213 
214-#endif214+#endif
Mactivation/swi_glu_grad/op_kernel/arch35/swi_glu_grad_ub_rearrange.h+64-64
@@ -198,10 +198,10 @@ __aicore__ inline void SwiGluGradUbRearrangeKernel<T, U>::RearrangeUbIn(uint32_t
198 {198 {
199 MicroAPI::RegTensor<T> vregA;199 MicroAPI::RegTensor<T> vregA;
200 MicroAPI::RegTensor<T> vregB;200 MicroAPI::RegTensor<T> vregB;
201- MicroAPI::UnalignReg uSrcA;201+ MicroAPI::UnalignRegForLoad uSrcA;
202- MicroAPI::UnalignReg uSrcB;202+ MicroAPI::UnalignRegForLoad uSrcB;
203- MicroAPI::UnalignReg uDstA;203+ MicroAPI::UnalignRegForStore uDstA;
204- MicroAPI::UnalignReg uDstB;204+ MicroAPI::UnalignRegForStore uDstB;
205 205 
206 for (uint16_t row = 0; row < size0; ++row) {206 for (uint16_t row = 0; row < size0; ++row) {
207 auto curASrcAddr = ubSrcAddrA + row * rowStride;207 auto curASrcAddr = ubSrcAddrA + row * rowStride;
@@ -209,23 +209,23 @@ __aicore__ inline void SwiGluGradUbRearrangeKernel<T, U>::RearrangeUbIn(uint32_t
209 auto curADstAddr = ubDstAddrA + row * colLen;209 auto curADstAddr = ubDstAddrA + row * colLen;
210 auto curBDstAddr = ubDstAddrB + row * colLen;210 auto curBDstAddr = ubDstAddrB + row * colLen;
211 211 
212- MicroAPI::DataCopyUnAlignPre(uSrcA, curASrcAddr);212+ MicroAPI::LoadUnAlignPre(uSrcA, curASrcAddr);
213- MicroAPI::DataCopyUnAlignPre(uSrcB, curBSrcAddr);213+ MicroAPI::LoadUnAlignPre(uSrcB, curBSrcAddr);
214 214 
215 for (uint16_t i = 0; i < size1; ++i) {215 for (uint16_t i = 0; i < size1; ++i) {
216- MicroAPI::DataCopyUnAlign(vregA, uSrcA, curASrcAddr, main);216+ MicroAPI::LoadUnAlign(vregA, uSrcA, curASrcAddr, main);
217- MicroAPI::DataCopyUnAlign(vregB, uSrcB, curBSrcAddr, main);217+ MicroAPI::LoadUnAlign(vregB, uSrcB, curBSrcAddr, main);
218- MicroAPI::DataCopyUnAlign(curADstAddr, vregA, uDstA, main);218+ MicroAPI::StoreUnAlign(curADstAddr, vregA, uDstA, main);
219- MicroAPI::DataCopyUnAlign(curBDstAddr, vregB, uDstB, main);219+ MicroAPI::StoreUnAlign(curBDstAddr, vregB, uDstB, main);
220 }220 }
221 221 
222- MicroAPI::DataCopyUnAlign(vregA, uSrcA, curASrcAddr, tail);222+ MicroAPI::LoadUnAlign(vregA, uSrcA, curASrcAddr, tail);
223- MicroAPI::DataCopyUnAlign(curADstAddr, vregA, uDstA, tail);223+ MicroAPI::StoreUnAlign(curADstAddr, vregA, uDstA, tail);
224- MicroAPI::DataCopyUnAlignPost(curADstAddr, uDstA, 0);224+ MicroAPI::StoreUnAlignPost(curADstAddr, uDstA, 0);
225 225 
226- MicroAPI::DataCopyUnAlign(vregB, uSrcB, curBSrcAddr, tail);226+ MicroAPI::LoadUnAlign(vregB, uSrcB, curBSrcAddr, tail);
227- MicroAPI::DataCopyUnAlign(curBDstAddr, vregB, uDstB, tail);227+ MicroAPI::StoreUnAlign(curBDstAddr, vregB, uDstB, tail);
228- MicroAPI::DataCopyUnAlignPost(curBDstAddr, uDstB, 0);228+ MicroAPI::StoreUnAlignPost(curBDstAddr, uDstB, 0);
229 }229 }
230 }230 }
231 231 
@@ -253,10 +253,10 @@ __aicore__ inline void SwiGluGradUbRearrangeKernel<T, U>::RearrangeUbOut(uint32_
253 {253 {
254 MicroAPI::RegTensor<T> vregA;254 MicroAPI::RegTensor<T> vregA;
255 MicroAPI::RegTensor<T> vregB;255 MicroAPI::RegTensor<T> vregB;
256- MicroAPI::UnalignReg uSrcA;256+ MicroAPI::UnalignRegForLoad uSrcA;
257- MicroAPI::UnalignReg uSrcB;257+ MicroAPI::UnalignRegForLoad uSrcB;
258- MicroAPI::UnalignReg uDstA;258+ MicroAPI::UnalignRegForStore uDstA;
259- MicroAPI::UnalignReg uDstB;259+ MicroAPI::UnalignRegForStore uDstB;
260 260 
261 for (uint16_t row = 0; row < size0; ++row) {261 for (uint16_t row = 0; row < size0; ++row) {
262 auto curASrcAddr = ubSrcAddrA + row * colLen;262 auto curASrcAddr = ubSrcAddrA + row * colLen;
@@ -264,23 +264,23 @@ __aicore__ inline void SwiGluGradUbRearrangeKernel<T, U>::RearrangeUbOut(uint32_
264 auto curADstAddr = ubDstAddrA + row * dstPerRowLen;264 auto curADstAddr = ubDstAddrA + row * dstPerRowLen;
265 auto curBDstAddr = ubDstAddrB + row * dstPerRowLen;265 auto curBDstAddr = ubDstAddrB + row * dstPerRowLen;
266 266 
267- MicroAPI::DataCopyUnAlignPre(uSrcA, curASrcAddr);267+ MicroAPI::LoadUnAlignPre(uSrcA, curASrcAddr);
268- MicroAPI::DataCopyUnAlignPre(uSrcB, curBSrcAddr);268+ MicroAPI::LoadUnAlignPre(uSrcB, curBSrcAddr);
269 269 
270 for (uint16_t i = 0; i < size1; ++i) {270 for (uint16_t i = 0; i < size1; ++i) {
271- MicroAPI::DataCopyUnAlign<T, MicroAPI::PostLiteral::POST_MODE_UPDATE>(vregA, uSrcA, curASrcAddr, main);271+ MicroAPI::LoadUnAlign<T, MicroAPI::PostLiteral::POST_MODE_UPDATE>(vregA, uSrcA, curASrcAddr, main);
272- MicroAPI::DataCopyUnAlign<T, MicroAPI::PostLiteral::POST_MODE_UPDATE>(vregB, uSrcB, curBSrcAddr, main);272+ MicroAPI::LoadUnAlign<T, MicroAPI::PostLiteral::POST_MODE_UPDATE>(vregB, uSrcB, curBSrcAddr, main);
273- MicroAPI::DataCopyUnAlign<T, MicroAPI::PostLiteral::POST_MODE_UPDATE>(curADstAddr, vregA, uDstA, main);273+ MicroAPI::StoreUnAlign<T, MicroAPI::PostLiteral::POST_MODE_UPDATE>(curADstAddr, vregA, uDstA, main);
274- MicroAPI::DataCopyUnAlign<T, MicroAPI::PostLiteral::POST_MODE_UPDATE>(curBDstAddr, vregB, uDstB, main);274+ MicroAPI::StoreUnAlign<T, MicroAPI::PostLiteral::POST_MODE_UPDATE>(curBDstAddr, vregB, uDstB, main);
275 }275 }
276 276 
277- MicroAPI::DataCopyUnAlign<T, MicroAPI::PostLiteral::POST_MODE_UPDATE>(vregA, uSrcA, curASrcAddr, tail);277+ MicroAPI::LoadUnAlign<T, MicroAPI::PostLiteral::POST_MODE_UPDATE>(vregA, uSrcA, curASrcAddr, tail);
278- MicroAPI::DataCopyUnAlign<T, MicroAPI::PostLiteral::POST_MODE_UPDATE>(curADstAddr, vregA, uDstA, tail);278+ MicroAPI::StoreUnAlign<T, MicroAPI::PostLiteral::POST_MODE_UPDATE>(curADstAddr, vregA, uDstA, tail);
279- MicroAPI::DataCopyUnAlignPost(curADstAddr, uDstA, 0);279+ MicroAPI::StoreUnAlignPost(curADstAddr, uDstA, 0);
280 280 
281- MicroAPI::DataCopyUnAlign<T, MicroAPI::PostLiteral::POST_MODE_UPDATE>(vregB, uSrcB, curBSrcAddr, tail);281+ MicroAPI::LoadUnAlign<T, MicroAPI::PostLiteral::POST_MODE_UPDATE>(vregB, uSrcB, curBSrcAddr, tail);
282- MicroAPI::DataCopyUnAlign<T, MicroAPI::PostLiteral::POST_MODE_UPDATE>(curBDstAddr, vregB, uDstB, tail);282+ MicroAPI::StoreUnAlign<T, MicroAPI::PostLiteral::POST_MODE_UPDATE>(curBDstAddr, vregB, uDstB, tail);
283- MicroAPI::DataCopyUnAlignPost(curBDstAddr, uDstB, 0);283+ MicroAPI::StoreUnAlignPost(curBDstAddr, uDstB, 0);
284 }284 }
285 }285 }
286 286 
@@ -324,7 +324,7 @@ __aicore__ inline void SwiGluGradUbRearrangeKernel<T, U>::GenerateGatherIndex(ui
324 324 
325 Add(indexCast, v1, v2, pIndex); // e = c + d325 Add(indexCast, v1, v2, pIndex); // e = c + d
326 326 
327- MicroAPI::DataCopy(indexAddr, indexCast, pIndex);327+ MicroAPI::StoreAlign(indexAddr, indexCast, pIndex);
328 }328 }
329}329}
330 330 
@@ -353,13 +353,13 @@ __aicore__ inline void SwiGluGradUbRearrangeKernel<T, U>::RearrangeUbInByGather(
353 MicroAPI::RegTensor<U> index;353 MicroAPI::RegTensor<U> index;
354 MicroAPI::RegTensor<U> indexUpd;354 MicroAPI::RegTensor<U> indexUpd;
355 355 
356- MicroAPI::UnalignReg uDstA;356+ MicroAPI::UnalignRegForStore uDstA;
357- MicroAPI::UnalignReg uDstB;357+ MicroAPI::UnalignRegForStore uDstB;
358 358 
359 MicroAPI::RegTensor<T> vregA;359 MicroAPI::RegTensor<T> vregA;
360 MicroAPI::RegTensor<T> vregB;360 MicroAPI::RegTensor<T> vregB;
361 361 
362- MicroAPI::DataCopy(index, indexAddr);362+ MicroAPI::LoadAlign(index, indexAddr);
363 363 
364 MicroAPI::MaskReg pMain = MicroAPI::UpdateMask<T>(main);364 MicroAPI::MaskReg pMain = MicroAPI::UpdateMask<T>(main);
365 MicroAPI::MaskReg pTail = MicroAPI::UpdateMask<T>(tail);365 MicroAPI::MaskReg pTail = MicroAPI::UpdateMask<T>(tail);
@@ -367,25 +367,25 @@ __aicore__ inline void SwiGluGradUbRearrangeKernel<T, U>::RearrangeUbInByGather(
367 for (uint16_t i = 0; i < size0; ++i) {367 for (uint16_t i = 0; i < size0; ++i) {
368 MicroAPI::Adds(indexUpd, index, (U)(i * stride * SPLIT_HALF), pMain);368 MicroAPI::Adds(indexUpd, index, (U)(i * stride * SPLIT_HALF), pMain);
369 369 
370- MicroAPI::DataCopyGather(vregA, (__local_mem__ T*)(ubSrcAddrA), indexUpd, pMain);370+ MicroAPI::Gather(vregA, (__ubuf__ T*)(ubSrcAddrA), indexUpd, pMain);
371- MicroAPI::DataCopyGather(vregB, (__local_mem__ T*)(ubSrcAddrB), indexUpd, pMain);371+ MicroAPI::Gather(vregB, (__ubuf__ T*)(ubSrcAddrB), indexUpd, pMain);
372 372 
373- MicroAPI::DataCopyUnAlign<T, MicroAPI::PostLiteral::POST_MODE_UPDATE>(ubDstAddrA, vregA, uDstA, stride);373+ MicroAPI::StoreUnAlign<T, MicroAPI::PostLiteral::POST_MODE_UPDATE>(ubDstAddrA, vregA, uDstA, stride);
374- MicroAPI::DataCopyUnAlign<T, MicroAPI::PostLiteral::POST_MODE_UPDATE>(ubDstAddrB, vregB, uDstB, stride);374+ MicroAPI::StoreUnAlign<T, MicroAPI::PostLiteral::POST_MODE_UPDATE>(ubDstAddrB, vregB, uDstB, stride);
375 }375 }
376 376 
377- MicroAPI::DataCopyUnAlignPost(ubDstAddrA, uDstA, 0);377+ MicroAPI::StoreUnAlignPost(ubDstAddrA, uDstA, 0);
378- MicroAPI::DataCopyUnAlignPost(ubDstAddrB, uDstB, 0);378+ MicroAPI::StoreUnAlignPost(ubDstAddrB, uDstB, 0);
379 379 
380 MicroAPI::Adds(indexUpd, index, (U)(size0 * stride * SPLIT_HALF), pTail);380 MicroAPI::Adds(indexUpd, index, (U)(size0 * stride * SPLIT_HALF), pTail);
381 381 
382- MicroAPI::DataCopyGather(vregA, (__local_mem__ T*)(ubSrcAddrA), indexUpd, pTail);382+ MicroAPI::Gather(vregA, (__ubuf__ T*)(ubSrcAddrA), indexUpd, pTail);
383- MicroAPI::DataCopyUnAlign<T, MicroAPI::PostLiteral::POST_MODE_UPDATE>(ubDstAddrA, vregA, uDstA, strideTail);383+ MicroAPI::StoreUnAlign<T, MicroAPI::PostLiteral::POST_MODE_UPDATE>(ubDstAddrA, vregA, uDstA, strideTail);
384- MicroAPI::DataCopyUnAlignPost(ubDstAddrA, uDstA, 0);384+ MicroAPI::StoreUnAlignPost(ubDstAddrA, uDstA, 0);
385 385 
386- MicroAPI::DataCopyGather(vregB, (__local_mem__ T*)(ubSrcAddrB), indexUpd, pTail);386+ MicroAPI::Gather(vregB, (__ubuf__ T*)(ubSrcAddrB), indexUpd, pTail);
387- MicroAPI::DataCopyUnAlign<T, MicroAPI::PostLiteral::POST_MODE_UPDATE>(ubDstAddrB, vregB, uDstB, strideTail);387+ MicroAPI::StoreUnAlign<T, MicroAPI::PostLiteral::POST_MODE_UPDATE>(ubDstAddrB, vregB, uDstB, strideTail);
388- MicroAPI::DataCopyUnAlignPost(ubDstAddrB, uDstB, 0);388+ MicroAPI::StoreUnAlignPost(ubDstAddrB, uDstB, 0);
389 }389 }
390 390 
391 inQueX_.FreeTensor(xLocal);391 inQueX_.FreeTensor(xLocal);
@@ -420,36 +420,36 @@ __aicore__ inline void SwiGluGradUbRearrangeKernel<T, U>::RearrangeUbOutByScatte
420 MicroAPI::RegTensor<T> vregA;420 MicroAPI::RegTensor<T> vregA;
421 MicroAPI::RegTensor<T> vregB;421 MicroAPI::RegTensor<T> vregB;
422 422 
423- MicroAPI::UnalignReg uSrcA;423+ MicroAPI::UnalignRegForLoad uSrcA;
424- MicroAPI::UnalignReg uSrcB;424+ MicroAPI::UnalignRegForLoad uSrcB;
425 425 
426- MicroAPI::DataCopy(index, indexAddr);426+ MicroAPI::LoadAlign(index, indexAddr);
427 427 
428 MicroAPI::MaskReg pMain = MicroAPI::UpdateMask<T>(main);428 MicroAPI::MaskReg pMain = MicroAPI::UpdateMask<T>(main);
429 MicroAPI::MaskReg pTail = MicroAPI::UpdateMask<T>(tail);429 MicroAPI::MaskReg pTail = MicroAPI::UpdateMask<T>(tail);
430 430 
431- MicroAPI::DataCopyUnAlignPre(uSrcA, ubSrcAddrA);431+ MicroAPI::LoadUnAlignPre(uSrcA, ubSrcAddrA);
432- MicroAPI::DataCopyUnAlignPre(uSrcB, ubSrcAddrB);432+ MicroAPI::LoadUnAlignPre(uSrcB, ubSrcAddrB);
433 for (uint16_t i = 0; i < size0; ++i) {433 for (uint16_t i = 0; i < size0; ++i) {
434 MicroAPI::Adds(indexUpd, index, (U)(i * stride * SPLIT_HALF), pMain);434 MicroAPI::Adds(indexUpd, index, (U)(i * stride * SPLIT_HALF), pMain);
435 435 
436- MicroAPI::DataCopyUnAlign<T, MicroAPI::PostLiteral::POST_MODE_UPDATE>(vregA, uSrcA, ubSrcAddrA, stride);436+ MicroAPI::LoadUnAlign<T, MicroAPI::PostLiteral::POST_MODE_UPDATE>(vregA, uSrcA, ubSrcAddrA, stride);
437- MicroAPI::DataCopyUnAlign<T, MicroAPI::PostLiteral::POST_MODE_UPDATE>(vregB, uSrcB, ubSrcAddrB, stride);437+ MicroAPI::LoadUnAlign<T, MicroAPI::PostLiteral::POST_MODE_UPDATE>(vregB, uSrcB, ubSrcAddrB, stride);
438 438 
439- MicroAPI::DataCopyScatter(ubDstAddrA, vregA, indexUpd, pMain);439+ MicroAPI::Scatter(ubDstAddrA, vregA, indexUpd, pMain);
440- MicroAPI::DataCopyScatter(ubDstAddrB, vregB, indexUpd, pMain);440+ MicroAPI::Scatter(ubDstAddrB, vregB, indexUpd, pMain);
441 }441 }
442 442 
443 MicroAPI::Adds(indexUpd, index, (U)(size0 * stride * SPLIT_HALF), pTail);443 MicroAPI::Adds(indexUpd, index, (U)(size0 * stride * SPLIT_HALF), pTail);
444 444 
445- MicroAPI::DataCopyUnAlignPre(uSrcA, ubSrcAddrA);445+ MicroAPI::LoadUnAlignPre(uSrcA, ubSrcAddrA);
446- MicroAPI::DataCopyUnAlignPre(uSrcB, ubSrcAddrB);446+ MicroAPI::LoadUnAlignPre(uSrcB, ubSrcAddrB);
447 447 
448- MicroAPI::DataCopyUnAlign<T, MicroAPI::PostLiteral::POST_MODE_UPDATE>(vregA, uSrcA, ubSrcAddrA, strideTail);448+ MicroAPI::LoadUnAlign<T, MicroAPI::PostLiteral::POST_MODE_UPDATE>(vregA, uSrcA, ubSrcAddrA, strideTail);
449- MicroAPI::DataCopyUnAlign<T, MicroAPI::PostLiteral::POST_MODE_UPDATE>(vregB, uSrcB, ubSrcAddrB, strideTail);449+ MicroAPI::LoadUnAlign<T, MicroAPI::PostLiteral::POST_MODE_UPDATE>(vregB, uSrcB, ubSrcAddrB, strideTail);
450 450 
451- MicroAPI::DataCopyScatter((__local_mem__ T*)(ubDstAddrA), vregA, indexUpd, pTail);451+ MicroAPI::Scatter((__ubuf__ T*)(ubDstAddrA), vregA, indexUpd, pTail);
452- MicroAPI::DataCopyScatter((__local_mem__ T*)(ubDstAddrB), vregB, indexUpd, pTail);452+ MicroAPI::Scatter((__ubuf__ T*)(ubDstAddrB), vregB, indexUpd, pTail);
453 }453 }
454 454 
455 outQueXGrad_.EnQue<T>(outLocal);455 outQueXGrad_.EnQue<T>(outLocal);
@@ -531,4 +531,4 @@ __aicore__ inline void SwiGluGradUbRearrangeKernel<T, U>::Process()
531 531 
532} // namespace SwiGluGrad532} // namespace SwiGluGrad
533 533 
534-#endif534+#endif
Mactivation/swish/op_kernel/arch35/swish_bf16_attr_comb_0.h+4-4
@@ -89,11 +89,11 @@ private:
89 uint32_t size = i0Extent;89 uint32_t size = i0Extent;
90 uint16_t vfLoopNum = (i0Extent + (AscendC::VECTOR_REG_WIDTH / sizeof(float)) - 1) /90 uint16_t vfLoopNum = (i0Extent + (AscendC::VECTOR_REG_WIDTH / sizeof(float)) - 1) /
91 (AscendC::VECTOR_REG_WIDTH / sizeof(float));91 (AscendC::VECTOR_REG_WIDTH / sizeof(float));
92- __local_mem__ bfloat16_t* bufferIn0Addr = (__local_mem__ bfloat16_t*)bufferIn0_.GetPhyAddr();92+ __ubuf__ bfloat16_t* bufferIn0Addr = (__ubuf__ bfloat16_t*)bufferIn0_.GetPhyAddr();
93- __local_mem__ bfloat16_t* bufferOut0Addr = (__local_mem__ bfloat16_t*)bufferOut0_.GetPhyAddr();93+ __ubuf__ bfloat16_t* bufferOut0Addr = (__ubuf__ bfloat16_t*)bufferOut0_.GetPhyAddr();
94 for (uint16_t i = 0; i < vfLoopNum; i++) {94 for (uint16_t i = 0; i < vfLoopNum; i++) {
95 preg0 = AscendC::MicroAPI::UpdateMask<float>(size);95 preg0 = AscendC::MicroAPI::UpdateMask<float>(size);
96- AscendC::MicroAPI::DataCopy<bfloat16_t, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(96+ AscendC::MicroAPI::LoadAlign<bfloat16_t, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(
97 vreg0, bufferIn0Addr + i * (AscendC::VECTOR_REG_WIDTH / sizeof(float)));97 vreg0, bufferIn0Addr + i * (AscendC::VECTOR_REG_WIDTH / sizeof(float)));
98 AscendC::MicroAPI::Cast<float, bfloat16_t, castTrait0>(vreg1, vreg0, preg0);98 AscendC::MicroAPI::Cast<float, bfloat16_t, castTrait0>(vreg1, vreg0, preg0);
99 AscendC::MicroAPI::Muls<float, float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(99 AscendC::MicroAPI::Muls<float, float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(
@@ -103,7 +103,7 @@ private:
103 vreg4, vreg3, static_cast<float>(1.0), preg0);103 vreg4, vreg3, static_cast<float>(1.0), preg0);
104 AscendC::MicroAPI::Div<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg5, vreg1, vreg4, preg0);104 AscendC::MicroAPI::Div<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg5, vreg1, vreg4, preg0);
105 AscendC::MicroAPI::Cast<bfloat16_t, float, castTrait1>(vreg6, vreg5, preg0);105 AscendC::MicroAPI::Cast<bfloat16_t, float, castTrait1>(vreg6, vreg5, preg0);
106- AscendC::MicroAPI::DataCopy<bfloat16_t, AscendC::MicroAPI::StoreDist::DIST_PACK_B32>(106+ AscendC::MicroAPI::StoreAlign<bfloat16_t, AscendC::MicroAPI::StoreDist::DIST_PACK_B32>(
107 bufferOut0Addr + i * (AscendC::VECTOR_REG_WIDTH / sizeof(float)), vreg6, preg0);107 bufferOut0Addr + i * (AscendC::VECTOR_REG_WIDTH / sizeof(float)), vreg6, preg0);
108 }108 }
109 }109 }
Mactivation/swish/op_kernel/arch35/swish_f16_attr_comb_0.h+4-4
@@ -90,11 +90,11 @@ private:
90 uint32_t size = i0Extent;90 uint32_t size = i0Extent;
91 uint16_t vfLoopNum = (i0Extent + (AscendC::VECTOR_REG_WIDTH / sizeof(float)) - 1) /91 uint16_t vfLoopNum = (i0Extent + (AscendC::VECTOR_REG_WIDTH / sizeof(float)) - 1) /
92 (AscendC::VECTOR_REG_WIDTH / sizeof(float));92 (AscendC::VECTOR_REG_WIDTH / sizeof(float));
93- __local_mem__ half* bufferIn0Addr = (__local_mem__ half*)bufferIn0_.GetPhyAddr();93+ __ubuf__ half* bufferIn0Addr = (__ubuf__ half*)bufferIn0_.GetPhyAddr();
94- __local_mem__ half* bufferOut0Addr = (__local_mem__ half*)bufferOut0_.GetPhyAddr();94+ __ubuf__ half* bufferOut0Addr = (__ubuf__ half*)bufferOut0_.GetPhyAddr();
95 for (uint16_t i = 0; i < vfLoopNum; i++) {95 for (uint16_t i = 0; i < vfLoopNum; i++) {
96 preg0 = AscendC::MicroAPI::UpdateMask<float>(size);96 preg0 = AscendC::MicroAPI::UpdateMask<float>(size);
97- AscendC::MicroAPI::DataCopy<half, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(97+ AscendC::MicroAPI::LoadAlign<half, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(
98 vreg0, bufferIn0Addr + i * (AscendC::VECTOR_REG_WIDTH / sizeof(float)));98 vreg0, bufferIn0Addr + i * (AscendC::VECTOR_REG_WIDTH / sizeof(float)));
99 AscendC::MicroAPI::Cast<float, half, castTrait0>(vreg1, vreg0, preg0);99 AscendC::MicroAPI::Cast<float, half, castTrait0>(vreg1, vreg0, preg0);
100 AscendC::MicroAPI::Muls<float, float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(100 AscendC::MicroAPI::Muls<float, float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(
@@ -104,7 +104,7 @@ private:
104 vreg4, vreg3, static_cast<float>(1.0), preg0);104 vreg4, vreg3, static_cast<float>(1.0), preg0);
105 AscendC::MicroAPI::Div<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg5, vreg1, vreg4, preg0);105 AscendC::MicroAPI::Div<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg5, vreg1, vreg4, preg0);
106 AscendC::MicroAPI::Cast<half, float, castTrait1>(vreg6, vreg5, preg0);106 AscendC::MicroAPI::Cast<half, float, castTrait1>(vreg6, vreg5, preg0);
107- AscendC::MicroAPI::DataCopy<half, AscendC::MicroAPI::StoreDist::DIST_PACK_B32>(107+ AscendC::MicroAPI::StoreAlign<half, AscendC::MicroAPI::StoreDist::DIST_PACK_B32>(
108 bufferOut0Addr + i * (AscendC::VECTOR_REG_WIDTH / sizeof(float)), vreg6, preg0);108 bufferOut0Addr + i * (AscendC::VECTOR_REG_WIDTH / sizeof(float)), vreg6, preg0);
109 }109 }
110 }110 }
Mactivation/swish/op_kernel/arch35/swish_f32_attr_comb_0.h+4-4
@@ -87,11 +87,11 @@ private:
87 uint32_t size = i0Extent;87 uint32_t size = i0Extent;
88 uint16_t vfLoopNum = (i0Extent + (AscendC::VECTOR_REG_WIDTH / sizeof(float)) - 1) /88 uint16_t vfLoopNum = (i0Extent + (AscendC::VECTOR_REG_WIDTH / sizeof(float)) - 1) /
89 (AscendC::VECTOR_REG_WIDTH / sizeof(float));89 (AscendC::VECTOR_REG_WIDTH / sizeof(float));
90- __local_mem__ float* bufferIn0Addr = (__local_mem__ float*)bufferIn0_.GetPhyAddr();90+ __ubuf__ float* bufferIn0Addr = (__ubuf__ float*)bufferIn0_.GetPhyAddr();
91- __local_mem__ float* bufferOut0Addr = (__local_mem__ float*)bufferOut0_.GetPhyAddr();91+ __ubuf__ float* bufferOut0Addr = (__ubuf__ float*)bufferOut0_.GetPhyAddr();
92 for (uint16_t i = 0; i < vfLoopNum; i++) {92 for (uint16_t i = 0; i < vfLoopNum; i++) {
93 preg0 = AscendC::MicroAPI::UpdateMask<float>(size);93 preg0 = AscendC::MicroAPI::UpdateMask<float>(size);
94- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(94+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(
95 vreg0, bufferIn0Addr + i * (AscendC::VECTOR_REG_WIDTH / sizeof(float)));95 vreg0, bufferIn0Addr + i * (AscendC::VECTOR_REG_WIDTH / sizeof(float)));
96 AscendC::MicroAPI::Muls<float, float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(96 AscendC::MicroAPI::Muls<float, float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(
97 vreg1, vreg0, static_cast<float>(-1.0) * tilingDataPtr_->scale, preg0);97 vreg1, vreg0, static_cast<float>(-1.0) * tilingDataPtr_->scale, preg0);
@@ -99,7 +99,7 @@ private:
99 AscendC::MicroAPI::Adds<float, float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(99 AscendC::MicroAPI::Adds<float, float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(
100 vreg3, vreg2, static_cast<float>(1.0), preg0);100 vreg3, vreg2, static_cast<float>(1.0), preg0);
101 AscendC::MicroAPI::Div<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg4, vreg0, vreg3, preg0);101 AscendC::MicroAPI::Div<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg4, vreg0, vreg3, preg0);
102- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::StoreDist::DIST_NORM_B32>(102+ AscendC::MicroAPI::StoreAlign<float, AscendC::MicroAPI::StoreDist::DIST_NORM_B32>(
103 bufferOut0Addr + i * (AscendC::VECTOR_REG_WIDTH / sizeof(float)), vreg4, preg0);103 bufferOut0Addr + i * (AscendC::VECTOR_REG_WIDTH / sizeof(float)), vreg4, preg0);
104 }104 }
105 }105 }
Mactivation/swish_grad/op_kernel/arch35/swish_grad_dag.h+11-11
@@ -63,8 +63,8 @@ struct SwishGradCustom : public Ops::Base::Vec::ElemwiseTernaryOP<T, T, T, float
63 for (uint16_t loopIdx = 0; loopIdx < loopNum; loopIdx++) {63 for (uint16_t loopIdx = 0; loopIdx < loopNum; loopIdx++) {
64 mask = MicroAPI::UpdateMask<T, MicroAPI::RegTraitNumOne>(count);64 mask = MicroAPI::UpdateMask<T, MicroAPI::RegTraitNumOne>(count);
65 // OpCopyIn65 // OpCopyIn
66- MicroAPI::DataCopy(vregInput, (__ubuf__ T*)(srcAddr + loopIdx * vlSize));66+ MicroAPI::LoadAlign(vregInput, (__ubuf__ T*)(srcAddr + loopIdx * vlSize));
67- MicroAPI::DataCopy(vregInput2, (__ubuf__ T*)(src2Addr + loopIdx * vlSize));67+ MicroAPI::LoadAlign(vregInput2, (__ubuf__ T*)(src2Addr + loopIdx * vlSize));
68 MicroAPI::Muls(vregInput, vregInput, scale1, mask);68 MicroAPI::Muls(vregInput, vregInput, scale1, mask);
69 MicroAPI::Neg(vregInputMid, vregInput, mask);69 MicroAPI::Neg(vregInputMid, vregInput, mask);
70 MicroAPI::Exp(vregInputMid, vregInputMid, mask);70 MicroAPI::Exp(vregInputMid, vregInputMid, mask);
@@ -73,12 +73,12 @@ struct SwishGradCustom : public Ops::Base::Vec::ElemwiseTernaryOP<T, T, T, float
73 73 
74 MicroAPI::Sub(vregOutput, vregValue1, vregInputMid, mask);74 MicroAPI::Sub(vregOutput, vregValue1, vregInputMid, mask);
75 MicroAPI::Mul(vregOutput, vregOutput, vregInput, mask);75 MicroAPI::Mul(vregOutput, vregOutput, vregInput, mask);
76- MicroAPI::FusedMulDstAdd(vregOutput, vregInputMid, vregInputMid, mask);76+ MicroAPI::MulDstAdd(vregOutput, vregInputMid, vregInputMid, mask);
77 MicroAPI::Mul(vregOutput, vregOutput, vregInput2, mask);77 MicroAPI::Mul(vregOutput, vregOutput, vregInput2, mask);
78 78 
79 // OpCopyOut79 // OpCopyOut
80- MicroAPI::DataCopy((__ubuf__ T*)(dstAddr + loopIdx * vlSize), (MicroAPI::RegTensor<T>&)vregOutput,80+ MicroAPI::StoreAlign((__ubuf__ T*)(dstAddr + loopIdx * vlSize), (MicroAPI::RegTensor<T>&)vregOutput,
81- mask);81+ mask);
82 }82 }
83 }83 }
84 } else {84 } else {
@@ -91,9 +91,9 @@ struct SwishGradCustom : public Ops::Base::Vec::ElemwiseTernaryOP<T, T, T, float
91 for (uint16_t loopIdx = 0; loopIdx < loopNum; loopIdx++) {91 for (uint16_t loopIdx = 0; loopIdx < loopNum; loopIdx++) {
92 mask = MicroAPI::UpdateMask<float, MicroAPI::RegTraitNumOne>(count);92 mask = MicroAPI::UpdateMask<float, MicroAPI::RegTraitNumOne>(count);
93 // OpCopyIn93 // OpCopyIn
94- MicroAPI::DataCopy<T, MicroAPI::LoadDist::DIST_UNPACK_B16>(94+ MicroAPI::LoadAlign<T, MicroAPI::LoadDist::DIST_UNPACK_B16>(
95 vregInput16, (__ubuf__ T*)(srcAddr + loopIdx * vlSize));95 vregInput16, (__ubuf__ T*)(srcAddr + loopIdx * vlSize));
96- MicroAPI::DataCopy<T, MicroAPI::LoadDist::DIST_UNPACK_B16>(96+ MicroAPI::LoadAlign<T, MicroAPI::LoadDist::DIST_UNPACK_B16>(
97 vregInput162, (__ubuf__ T*)(src2Addr + loopIdx * vlSize));97 vregInput162, (__ubuf__ T*)(src2Addr + loopIdx * vlSize));
98 MicroAPI::Cast<float, T, castTrait0>(vregInput, vregInput16, mask);98 MicroAPI::Cast<float, T, castTrait0>(vregInput, vregInput16, mask);
99 MicroAPI::Cast<float, T, castTrait0>(vregInput2, vregInput162, mask);99 MicroAPI::Cast<float, T, castTrait0>(vregInput2, vregInput162, mask);
@@ -106,13 +106,13 @@ struct SwishGradCustom : public Ops::Base::Vec::ElemwiseTernaryOP<T, T, T, float
106 106 
107 MicroAPI::Sub(vregOutput, vregValue1, vregInputMid, mask);107 MicroAPI::Sub(vregOutput, vregValue1, vregInputMid, mask);
108 MicroAPI::Mul(vregOutput, vregOutput, vregInput, mask);108 MicroAPI::Mul(vregOutput, vregOutput, vregInput, mask);
109- MicroAPI::FusedMulDstAdd(vregOutput, vregInputMid, vregInputMid, mask);109+ MicroAPI::MulDstAdd(vregOutput, vregInputMid, vregInputMid, mask);
110 MicroAPI::Mul(vregOutput, vregOutput, vregInput2, mask);110 MicroAPI::Mul(vregOutput, vregOutput, vregInput2, mask);
111 111 
112 MicroAPI::Cast<T, float, castTrait1>(vregOutput16, vregOutput, mask);112 MicroAPI::Cast<T, float, castTrait1>(vregOutput16, vregOutput, mask);
113 // OpCopyOut113 // OpCopyOut
114- MicroAPI::DataCopy<T, MicroAPI::StoreDist::DIST_PACK_B32>((__ubuf__ T*)(dstAddr + loopIdx * vlSize),114+ MicroAPI::StoreAlign<T, MicroAPI::StoreDist::DIST_PACK_B32>(
115- vregOutput16, mask);115+ (__ubuf__ T*)(dstAddr + loopIdx * vlSize), vregOutput16, mask);
116 }116 }
117 }117 }
118 }118 }
@@ -134,4 +134,4 @@ struct SwishGradDAG {
134 using OpDag = DAGSch<Outputs, void, MemCfg>;134 using OpDag = DAGSch<Outputs, void, MemCfg>;
135};135};
136} // namespace SwishGradOp136} // namespace SwishGradOp
137-#endif // CANN_CUSTOM_OPS_SWISHGRAD_DAG_H137+#endif // CANN_CUSTOM_OPS_SWISHGRAD_DAG_H
Mindex/map_index/op_kernel/arch35/map_index.h+5-5
@@ -218,12 +218,12 @@ __aicore__ inline void MapIndex::ComputeOneRowMask(LocalTensor<int32_t>& xLocalR
218 uint32_t sreg0 = calCount;218 uint32_t sreg0 = calCount;
219 for (uint16_t i = 0; i < loopNum; i++) { // 256B219 for (uint16_t i = 0; i < loopNum; i++) { // 256B
220 preg0 = AscendC::MicroAPI::UpdateMask<int32_t>(sreg0);220 preg0 = AscendC::MicroAPI::UpdateMask<int32_t>(sreg0);
221- AscendC::MicroAPI::DataCopy(xReg, xAddr + i * vl);221+ AscendC::MicroAPI::LoadAlign(xReg, xAddr + i * vl);
222- AscendC::MicroAPI::DataCopy(dataSeqReg, dataSeqAddr + i * vl);222+ AscendC::MicroAPI::LoadAlign(dataSeqReg, dataSeqAddr + i * vl);
223 AscendC::MicroAPI::Compare<int32_t, CMPMODE::EQ>(preg2, xReg, dataSeqReg, preg0);223 AscendC::MicroAPI::Compare<int32_t, CMPMODE::EQ>(preg2, xReg, dataSeqReg, preg0);
224- AscendC::MicroAPI::MaskAnd(pregResultMask, pregResultMask, preg2, preg0);224+ AscendC::MicroAPI::And(pregResultMask, pregResultMask, preg2, preg0);
225 }225 }
226- AscendC::MicroAPI::DataCopy(maskAddr, pregResultMask);226+ AscendC::MicroAPI::StoreAlign(maskAddr, pregResultMask);
227 }227 }
228}228}
229 229 
@@ -237,4 +237,4 @@ __aicore__ inline void MapIndex::CopyOut()
237}237}
238 238 
239} // namespace MapIndexOp239} // namespace MapIndexOp
240-#endif240+#endif
Mindex/quant_update_scatter/op_kernel/arch35/quant_update_scatter_large_batch_large_quant_regbase.h+28-28
@@ -179,10 +179,10 @@ public:
179 179 
180 LocalTensor<VarType> outLocal = outQueueVar_.AllocTensor<VarType>();180 LocalTensor<VarType> outLocal = outQueueVar_.AllocTensor<VarType>();
181 181 
182- __local_mem__ UpdatesType* updateLocalAddr = (__local_mem__ UpdatesType*)updateLocal.GetPhyAddr();182+ __ubuf__ UpdatesType* updateLocalAddr = (__ubuf__ UpdatesType*)updateLocal.GetPhyAddr();
183- __local_mem__ ScalesType* scaleLocalAddr = (__local_mem__ ScalesType*)scaleLocal.GetPhyAddr();183+ __ubuf__ ScalesType* scaleLocalAddr = (__ubuf__ ScalesType*)scaleLocal.GetPhyAddr();
184- __local_mem__ OffsetsType* offsetLocalAddr = (__local_mem__ OffsetsType*)offsetLocal.GetPhyAddr();184+ __ubuf__ OffsetsType* offsetLocalAddr = (__ubuf__ OffsetsType*)offsetLocal.GetPhyAddr();
185- __local_mem__ VarType* outLocalAddr = (__local_mem__ VarType*)outLocal.GetPhyAddr();185+ __ubuf__ VarType* outLocalAddr = (__ubuf__ VarType*)outLocal.GetPhyAddr();
186 186 
187 uint16_t VL = AscendC::VECTOR_REG_WIDTH / sizeof(float);187 uint16_t VL = AscendC::VECTOR_REG_WIDTH / sizeof(float);
188 uint32_t count = static_cast<uint32_t>(dataCount);188 uint32_t count = static_cast<uint32_t>(dataCount);
@@ -217,12 +217,12 @@ public:
217 // ld and cast for update217 // ld and cast for update
218 if constexpr (IsSameType<UpdatesType, half>::value) {218 if constexpr (IsSameType<UpdatesType, half>::value) {
219 // fp16219 // fp16
220- AscendC::MicroAPI::DataCopy<half, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(220+ AscendC::MicroAPI::LoadAlign<half, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(
221 vregX, updateLocalAddr + i * VL);221 vregX, updateLocalAddr + i * VL);
222 AscendC::MicroAPI::Cast<float, half, Base::CAST_TRAIT_HALF_TO_FP32>(vregFloatX, vregX, mask);222 AscendC::MicroAPI::Cast<float, half, Base::CAST_TRAIT_HALF_TO_FP32>(vregFloatX, vregX, mask);
223 } else if constexpr (IsSameType<UpdatesType, bfloat16_t>::value) {223 } else if constexpr (IsSameType<UpdatesType, bfloat16_t>::value) {
224 // bf16224 // bf16
225- AscendC::MicroAPI::DataCopy<UpdatesType, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(225+ AscendC::MicroAPI::LoadAlign<UpdatesType, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(
226 vregX, updateLocalAddr + i * VL);226 vregX, updateLocalAddr + i * VL);
227 AscendC::MicroAPI::Cast<float, UpdatesType, Base::CAST_TRAIT_BF16_TO_FP32>(vregFloatX, vregX, mask);227 AscendC::MicroAPI::Cast<float, UpdatesType, Base::CAST_TRAIT_BF16_TO_FP32>(vregFloatX, vregX, mask);
228 }228 }
@@ -230,24 +230,24 @@ public:
230 // ld and cast for scale230 // ld and cast for scale
231 if constexpr (IsSameType<ScalesType, float>::value) {231 if constexpr (IsSameType<ScalesType, float>::value) {
232 // fp32232 // fp32
233- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(vregFloatS,233+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(
234- scaleLocalAddr + i * VL);234+ vregFloatS, scaleLocalAddr + i * VL);
235 } else if constexpr (IsSameType<ScalesType, bfloat16_t>::value) {235 } else if constexpr (IsSameType<ScalesType, bfloat16_t>::value) {
236 // bf16236 // bf16
237- AscendC::MicroAPI::DataCopy<ScalesType, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(237+ AscendC::MicroAPI::LoadAlign<ScalesType, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(
238 vregS, scaleLocalAddr + i * VL);238 vregS, scaleLocalAddr + i * VL);
239 AscendC::MicroAPI::Cast<float, ScalesType, Base::CAST_TRAIT_BF16_TO_FP32>(vregFloatS, vregS, mask);239 AscendC::MicroAPI::Cast<float, ScalesType, Base::CAST_TRAIT_BF16_TO_FP32>(vregFloatS, vregS, mask);
240 }240 }
241 // ld and cast for offset241 // ld and cast for offset
242 if constexpr (IsSameType<OffsetsType, int32_t>::value) {242 if constexpr (IsSameType<OffsetsType, int32_t>::value) {
243 // int32243 // int32
244- AscendC::MicroAPI::DataCopy<OffsetsType, AscendC::MicroAPI::LoadDist::DIST_NORM>(244+ AscendC::MicroAPI::LoadAlign<OffsetsType, AscendC::MicroAPI::LoadDist::DIST_NORM>(
245 vregO, offsetLocalAddr + i * VL);245 vregO, offsetLocalAddr + i * VL);
246 AscendC::MicroAPI::Cast<float, OffsetsType, Base::CAST_TRAIT_INT32_TO_FP32>(vregFloatO, vregO,246 AscendC::MicroAPI::Cast<float, OffsetsType, Base::CAST_TRAIT_INT32_TO_FP32>(vregFloatO, vregO,
247 mask);247 mask);
248 } else if constexpr (IsSameType<OffsetsType, bfloat16_t>::value) {248 } else if constexpr (IsSameType<OffsetsType, bfloat16_t>::value) {
249 // bf16249 // bf16
250- AscendC::MicroAPI::DataCopy<OffsetsType, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(250+ AscendC::MicroAPI::LoadAlign<OffsetsType, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(
251 vregO, offsetLocalAddr + i * VL);251 vregO, offsetLocalAddr + i * VL);
252 AscendC::MicroAPI::Cast<float, OffsetsType, Base::CAST_TRAIT_BF16_TO_FP32>(vregFloatO, vregO, mask);252 AscendC::MicroAPI::Cast<float, OffsetsType, Base::CAST_TRAIT_BF16_TO_FP32>(vregFloatO, vregO, mask);
253 }253 }
@@ -267,17 +267,17 @@ public:
267 if constexpr (IsSameType<VarType, hifloat8_t>::value) {267 if constexpr (IsSameType<VarType, hifloat8_t>::value) {
268 // hifp8268 // hifp8
269 AscendC::MicroAPI::Cast<VarType, float, Base::CAST_TRAIT_FP32_TO_HIFP8>(vregY, vregFloatY, mask);269 AscendC::MicroAPI::Cast<VarType, float, Base::CAST_TRAIT_FP32_TO_HIFP8>(vregY, vregFloatY, mask);
270- AscendC::MicroAPI::DataCopy<VarType, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(270+ AscendC::MicroAPI::StoreAlign<VarType, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(
271 outLocalAddr + i * VL, vregY, mask);271 outLocalAddr + i * VL, vregY, mask);
272 } else if constexpr (IsSameType<VarType, fp8_e5m2_t>::value) {272 } else if constexpr (IsSameType<VarType, fp8_e5m2_t>::value) {
273 // fp8_e5m2273 // fp8_e5m2
274 AscendC::MicroAPI::Cast<VarType, float, Base::CAST_TRAIT_FP32_TO_FP8E5M2>(vregY, vregFloatY, mask);274 AscendC::MicroAPI::Cast<VarType, float, Base::CAST_TRAIT_FP32_TO_FP8E5M2>(vregY, vregFloatY, mask);
275- AscendC::MicroAPI::DataCopy<VarType, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(275+ AscendC::MicroAPI::StoreAlign<VarType, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(
276 outLocalAddr + i * VL, vregY, mask);276 outLocalAddr + i * VL, vregY, mask);
277 } else if constexpr (IsSameType<VarType, fp8_e4m3fn_t>::value) {277 } else if constexpr (IsSameType<VarType, fp8_e4m3fn_t>::value) {
278 // fp8_e4m3278 // fp8_e4m3
279 AscendC::MicroAPI::Cast<VarType, float, Base::CAST_TRAIT_FP32_TO_FP8E4M3>(vregY, vregFloatY, mask);279 AscendC::MicroAPI::Cast<VarType, float, Base::CAST_TRAIT_FP32_TO_FP8E4M3>(vregY, vregFloatY, mask);
280- AscendC::MicroAPI::DataCopy<VarType, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(280+ AscendC::MicroAPI::StoreAlign<VarType, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(
281 outLocalAddr + i * VL, vregY, mask);281 outLocalAddr + i * VL, vregY, mask);
282 } else if constexpr (IsSameType<VarType, int8_t>::value) {282 } else if constexpr (IsSameType<VarType, int8_t>::value) {
283 // int8283 // int8
@@ -285,7 +285,7 @@ public:
285 mask);285 mask);
286 AscendC::MicroAPI::Cast<half, int16_t, Base::CAST_TRAIT_INT16_TO_HALF>(vregHalfY, vregInt16Y, mask);286 AscendC::MicroAPI::Cast<half, int16_t, Base::CAST_TRAIT_INT16_TO_HALF>(vregHalfY, vregInt16Y, mask);
287 AscendC::MicroAPI::Cast<int8_t, half, Base::CAST_TRAIT_HALF_TO_INT8>(vregY, vregHalfY, mask);287 AscendC::MicroAPI::Cast<int8_t, half, Base::CAST_TRAIT_HALF_TO_INT8>(vregY, vregHalfY, mask);
288- AscendC::MicroAPI::DataCopy<VarType, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(288+ AscendC::MicroAPI::StoreAlign<VarType, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(
289 outLocalAddr + i * VL, vregY, mask);289 outLocalAddr + i * VL, vregY, mask);
290 }290 }
291 }291 }
@@ -303,9 +303,9 @@ public:
303 303 
304 LocalTensor<VarType> outLocal = outQueueVar_.AllocTensor<VarType>();304 LocalTensor<VarType> outLocal = outQueueVar_.AllocTensor<VarType>();
305 305 
306- __local_mem__ UpdatesType* updateLocalAddr = (__local_mem__ UpdatesType*)updateLocal.GetPhyAddr();306+ __ubuf__ UpdatesType* updateLocalAddr = (__ubuf__ UpdatesType*)updateLocal.GetPhyAddr();
307- __local_mem__ ScalesType* scaleLocalAddr = (__local_mem__ ScalesType*)scaleLocal.GetPhyAddr();307+ __ubuf__ ScalesType* scaleLocalAddr = (__ubuf__ ScalesType*)scaleLocal.GetPhyAddr();
308- __local_mem__ VarType* outLocalAddr = (__local_mem__ VarType*)outLocal.GetPhyAddr();308+ __ubuf__ VarType* outLocalAddr = (__ubuf__ VarType*)outLocal.GetPhyAddr();
309 309 
310 uint16_t VL = AscendC::VECTOR_REG_WIDTH / sizeof(float);310 uint16_t VL = AscendC::VECTOR_REG_WIDTH / sizeof(float);
311 uint32_t count = static_cast<uint32_t>(dataCount);311 uint32_t count = static_cast<uint32_t>(dataCount);
@@ -336,12 +336,12 @@ public:
336 // ld and cast for update336 // ld and cast for update
337 if constexpr (IsSameType<UpdatesType, half>::value) {337 if constexpr (IsSameType<UpdatesType, half>::value) {
338 // fp16338 // fp16
339- AscendC::MicroAPI::DataCopy<half, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(339+ AscendC::MicroAPI::LoadAlign<half, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(
340 vregX, updateLocalAddr + i * VL);340 vregX, updateLocalAddr + i * VL);
341 AscendC::MicroAPI::Cast<float, half, Base::CAST_TRAIT_HALF_TO_FP32>(vregFloatX, vregX, mask);341 AscendC::MicroAPI::Cast<float, half, Base::CAST_TRAIT_HALF_TO_FP32>(vregFloatX, vregX, mask);
342 } else if constexpr (IsSameType<UpdatesType, bfloat16_t>::value) {342 } else if constexpr (IsSameType<UpdatesType, bfloat16_t>::value) {
343 // bf16343 // bf16
344- AscendC::MicroAPI::DataCopy<UpdatesType, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(344+ AscendC::MicroAPI::LoadAlign<UpdatesType, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(
345 vregX, updateLocalAddr + i * VL);345 vregX, updateLocalAddr + i * VL);
346 AscendC::MicroAPI::Cast<float, UpdatesType, Base::CAST_TRAIT_BF16_TO_FP32>(vregFloatX, vregX, mask);346 AscendC::MicroAPI::Cast<float, UpdatesType, Base::CAST_TRAIT_BF16_TO_FP32>(vregFloatX, vregX, mask);
347 }347 }
@@ -349,11 +349,11 @@ public:
349 // ld and cast for scale349 // ld and cast for scale
350 if constexpr (IsSameType<ScalesType, float>::value) {350 if constexpr (IsSameType<ScalesType, float>::value) {
351 // fp32351 // fp32
352- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(vregFloatS,352+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(
353- scaleLocalAddr + i * VL);353+ vregFloatS, scaleLocalAddr + i * VL);
354 } else if constexpr (IsSameType<ScalesType, bfloat16_t>::value) {354 } else if constexpr (IsSameType<ScalesType, bfloat16_t>::value) {
355 // bf16355 // bf16
356- AscendC::MicroAPI::DataCopy<ScalesType, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(356+ AscendC::MicroAPI::LoadAlign<ScalesType, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(
357 vregS, scaleLocalAddr + i * VL);357 vregS, scaleLocalAddr + i * VL);
358 AscendC::MicroAPI::Cast<float, ScalesType, Base::CAST_TRAIT_BF16_TO_FP32>(vregFloatS, vregS, mask);358 AscendC::MicroAPI::Cast<float, ScalesType, Base::CAST_TRAIT_BF16_TO_FP32>(vregFloatS, vregS, mask);
359 }359 }
@@ -370,17 +370,17 @@ public:
370 if constexpr (IsSameType<VarType, hifloat8_t>::value) {370 if constexpr (IsSameType<VarType, hifloat8_t>::value) {
371 // hifp8371 // hifp8
372 AscendC::MicroAPI::Cast<VarType, float, Base::CAST_TRAIT_FP32_TO_HIFP8>(vregY, vregFloatY, mask);372 AscendC::MicroAPI::Cast<VarType, float, Base::CAST_TRAIT_FP32_TO_HIFP8>(vregY, vregFloatY, mask);
373- AscendC::MicroAPI::DataCopy<VarType, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(373+ AscendC::MicroAPI::StoreAlign<VarType, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(
374 outLocalAddr + i * VL, vregY, mask);374 outLocalAddr + i * VL, vregY, mask);
375 } else if constexpr (IsSameType<VarType, fp8_e5m2_t>::value) {375 } else if constexpr (IsSameType<VarType, fp8_e5m2_t>::value) {
376 // fp8_e5m2376 // fp8_e5m2
377 AscendC::MicroAPI::Cast<VarType, float, Base::CAST_TRAIT_FP32_TO_FP8E5M2>(vregY, vregFloatY, mask);377 AscendC::MicroAPI::Cast<VarType, float, Base::CAST_TRAIT_FP32_TO_FP8E5M2>(vregY, vregFloatY, mask);
378- AscendC::MicroAPI::DataCopy<VarType, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(378+ AscendC::MicroAPI::StoreAlign<VarType, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(
379 outLocalAddr + i * VL, vregY, mask);379 outLocalAddr + i * VL, vregY, mask);
380 } else if constexpr (IsSameType<VarType, fp8_e4m3fn_t>::value) {380 } else if constexpr (IsSameType<VarType, fp8_e4m3fn_t>::value) {
381 // fp8_e4m3381 // fp8_e4m3
382 AscendC::MicroAPI::Cast<VarType, float, Base::CAST_TRAIT_FP32_TO_FP8E4M3>(vregY, vregFloatY, mask);382 AscendC::MicroAPI::Cast<VarType, float, Base::CAST_TRAIT_FP32_TO_FP8E4M3>(vregY, vregFloatY, mask);
383- AscendC::MicroAPI::DataCopy<VarType, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(383+ AscendC::MicroAPI::StoreAlign<VarType, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(
384 outLocalAddr + i * VL, vregY, mask);384 outLocalAddr + i * VL, vregY, mask);
385 } else if constexpr (IsSameType<VarType, int8_t>::value) {385 } else if constexpr (IsSameType<VarType, int8_t>::value) {
386 // int8386 // int8
@@ -388,7 +388,7 @@ public:
388 mask);388 mask);
389 AscendC::MicroAPI::Cast<half, int16_t, Base::CAST_TRAIT_INT16_TO_HALF>(vregHalfY, vregInt16Y, mask);389 AscendC::MicroAPI::Cast<half, int16_t, Base::CAST_TRAIT_INT16_TO_HALF>(vregHalfY, vregInt16Y, mask);
390 AscendC::MicroAPI::Cast<int8_t, half, Base::CAST_TRAIT_HALF_TO_INT8>(vregY, vregHalfY, mask);390 AscendC::MicroAPI::Cast<int8_t, half, Base::CAST_TRAIT_HALF_TO_INT8>(vregY, vregHalfY, mask);
391- AscendC::MicroAPI::DataCopy<VarType, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(391+ AscendC::MicroAPI::StoreAlign<VarType, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(
392 outLocalAddr + i * VL, vregY, mask);392 outLocalAddr + i * VL, vregY, mask);
393 }393 }
394 }394 }
@@ -452,4 +452,4 @@ public:
452 }452 }
453};453};
454} // namespace QuantUpdateScatter454} // namespace QuantUpdateScatter
455-#endif // QUANT_UPDATE_SCATTER_LARGE_BATCH_LARGE_QUANT_REGBASE_H_455+#endif // QUANT_UPDATE_SCATTER_LARGE_BATCH_LARGE_QUANT_REGBASE_H_
Mindex/quant_update_scatter/op_kernel/arch35/quant_update_scatter_large_batch_little_quant_regbase.h+26-26
@@ -187,10 +187,10 @@ public:
187 LocalTensor<UpdatesType> updateLocal = inQueueUpdates_.DeQue<UpdatesType>();187 LocalTensor<UpdatesType> updateLocal = inQueueUpdates_.DeQue<UpdatesType>();
188 LocalTensor<VarType> outLocal = outQueueVar_.AllocTensor<VarType>();188 LocalTensor<VarType> outLocal = outQueueVar_.AllocTensor<VarType>();
189 189 
190- __local_mem__ UpdatesType* updateLocalAddr = (__local_mem__ UpdatesType*)updateLocal.GetPhyAddr();190+ __ubuf__ UpdatesType* updateLocalAddr = (__ubuf__ UpdatesType*)updateLocal.GetPhyAddr();
191- __local_mem__ ScalesType* scaleLocalAddr = (__local_mem__ ScalesType*)scaleLocal.GetPhyAddr();191+ __ubuf__ ScalesType* scaleLocalAddr = (__ubuf__ ScalesType*)scaleLocal.GetPhyAddr();
192- __local_mem__ OffsetsType* offsetLocalAddr = (__local_mem__ OffsetsType*)offsetLocal.GetPhyAddr();192+ __ubuf__ OffsetsType* offsetLocalAddr = (__ubuf__ OffsetsType*)offsetLocal.GetPhyAddr();
193- __local_mem__ VarType* outLocalAddr = (__local_mem__ VarType*)outLocal.GetPhyAddr();193+ __ubuf__ VarType* outLocalAddr = (__ubuf__ VarType*)outLocal.GetPhyAddr();
194 194 
195 uint16_t VL = AscendC::VECTOR_REG_WIDTH / sizeof(float);195 uint16_t VL = AscendC::VECTOR_REG_WIDTH / sizeof(float);
196 uint32_t xLocalOffset = static_cast<uint32_t>(tilingData_.updateOriLastDimAlign);196 uint32_t xLocalOffset = static_cast<uint32_t>(tilingData_.updateOriLastDimAlign);
@@ -228,12 +228,12 @@ public:
228 // ld and cast for update228 // ld and cast for update
229 if constexpr (IsSameType<UpdatesType, half>::value) {229 if constexpr (IsSameType<UpdatesType, half>::value) {
230 // fp16230 // fp16
231- AscendC::MicroAPI::DataCopy<half, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(231+ AscendC::MicroAPI::LoadAlign<half, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(
232 vregX, updateLocalAddr + i * VL + j * xLocalOffset);232 vregX, updateLocalAddr + i * VL + j * xLocalOffset);
233 AscendC::MicroAPI::Cast<float, half, Base::CAST_TRAIT_HALF_TO_FP32>(vregFloatX, vregX, mask);233 AscendC::MicroAPI::Cast<float, half, Base::CAST_TRAIT_HALF_TO_FP32>(vregFloatX, vregX, mask);
234 } else if constexpr (IsSameType<UpdatesType, bfloat16_t>::value) {234 } else if constexpr (IsSameType<UpdatesType, bfloat16_t>::value) {
235 // bf16235 // bf16
236- AscendC::MicroAPI::DataCopy<UpdatesType, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(236+ AscendC::MicroAPI::LoadAlign<UpdatesType, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(
237 vregX, updateLocalAddr + i * VL + j * xLocalOffset);237 vregX, updateLocalAddr + i * VL + j * xLocalOffset);
238 AscendC::MicroAPI::Cast<float, UpdatesType, Base::CAST_TRAIT_BF16_TO_FP32>(vregFloatX, vregX,238 AscendC::MicroAPI::Cast<float, UpdatesType, Base::CAST_TRAIT_BF16_TO_FP32>(vregFloatX, vregX,
239 mask);239 mask);
@@ -242,11 +242,11 @@ public:
242 // ld and cast for scale242 // ld and cast for scale
243 if constexpr (IsSameType<ScalesType, float>::value) {243 if constexpr (IsSameType<ScalesType, float>::value) {
244 // fp32244 // fp32
245- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(245+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(
246 vregFloatS, scaleLocalAddr + i * VL);246 vregFloatS, scaleLocalAddr + i * VL);
247 } else if constexpr (IsSameType<ScalesType, bfloat16_t>::value) {247 } else if constexpr (IsSameType<ScalesType, bfloat16_t>::value) {
248 // bf16248 // bf16
249- AscendC::MicroAPI::DataCopy<ScalesType, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(249+ AscendC::MicroAPI::LoadAlign<ScalesType, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(
250 vregS, scaleLocalAddr + i * VL);250 vregS, scaleLocalAddr + i * VL);
251 AscendC::MicroAPI::Cast<float, ScalesType, Base::CAST_TRAIT_BF16_TO_FP32>(vregFloatS, vregS,251 AscendC::MicroAPI::Cast<float, ScalesType, Base::CAST_TRAIT_BF16_TO_FP32>(vregFloatS, vregS,
252 mask);252 mask);
@@ -254,13 +254,13 @@ public:
254 // ld and cast for offset254 // ld and cast for offset
255 if constexpr (IsSameType<OffsetsType, int32_t>::value) {255 if constexpr (IsSameType<OffsetsType, int32_t>::value) {
256 // int32256 // int32
257- AscendC::MicroAPI::DataCopy<OffsetsType, AscendC::MicroAPI::LoadDist::DIST_NORM>(257+ AscendC::MicroAPI::LoadAlign<OffsetsType, AscendC::MicroAPI::LoadDist::DIST_NORM>(
258 vregO, offsetLocalAddr + i * VL);258 vregO, offsetLocalAddr + i * VL);
259 AscendC::MicroAPI::Cast<float, OffsetsType, Base::CAST_TRAIT_INT32_TO_FP32>(vregFloatO, vregO,259 AscendC::MicroAPI::Cast<float, OffsetsType, Base::CAST_TRAIT_INT32_TO_FP32>(vregFloatO, vregO,
260 mask);260 mask);
261 } else if constexpr (IsSameType<OffsetsType, bfloat16_t>::value) {261 } else if constexpr (IsSameType<OffsetsType, bfloat16_t>::value) {
262 // bf16262 // bf16
263- AscendC::MicroAPI::DataCopy<OffsetsType, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(263+ AscendC::MicroAPI::LoadAlign<OffsetsType, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(
264 vregO, offsetLocalAddr + i * VL);264 vregO, offsetLocalAddr + i * VL);
265 AscendC::MicroAPI::Cast<float, OffsetsType, Base::CAST_TRAIT_BF16_TO_FP32>(vregFloatO, vregO,265 AscendC::MicroAPI::Cast<float, OffsetsType, Base::CAST_TRAIT_BF16_TO_FP32>(vregFloatO, vregO,
266 mask);266 mask);
@@ -282,19 +282,19 @@ public:
282 // hifp8282 // hifp8
283 AscendC::MicroAPI::Cast<VarType, float, Base::CAST_TRAIT_FP32_TO_HIFP8>(vregY, vregFloatY,283 AscendC::MicroAPI::Cast<VarType, float, Base::CAST_TRAIT_FP32_TO_HIFP8>(vregY, vregFloatY,
284 mask);284 mask);
285- AscendC::MicroAPI::DataCopy<VarType, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(285+ AscendC::MicroAPI::StoreAlign<VarType, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(
286 outLocalAddr + i * VL + j * xLocalOffset, vregY, mask);286 outLocalAddr + i * VL + j * xLocalOffset, vregY, mask);
287 } else if constexpr (IsSameType<VarType, fp8_e5m2_t>::value) {287 } else if constexpr (IsSameType<VarType, fp8_e5m2_t>::value) {
288 // fp8_e5m2288 // fp8_e5m2
289 AscendC::MicroAPI::Cast<VarType, float, Base::CAST_TRAIT_FP32_TO_FP8E5M2>(vregY, vregFloatY,289 AscendC::MicroAPI::Cast<VarType, float, Base::CAST_TRAIT_FP32_TO_FP8E5M2>(vregY, vregFloatY,
290 mask);290 mask);
291- AscendC::MicroAPI::DataCopy<VarType, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(291+ AscendC::MicroAPI::StoreAlign<VarType, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(
292 outLocalAddr + i * VL + j * xLocalOffset, vregY, mask);292 outLocalAddr + i * VL + j * xLocalOffset, vregY, mask);
293 } else if constexpr (IsSameType<VarType, fp8_e4m3fn_t>::value) {293 } else if constexpr (IsSameType<VarType, fp8_e4m3fn_t>::value) {
294 // fp8_e4m3294 // fp8_e4m3
295 AscendC::MicroAPI::Cast<VarType, float, Base::CAST_TRAIT_FP32_TO_FP8E4M3>(vregY, vregFloatY,295 AscendC::MicroAPI::Cast<VarType, float, Base::CAST_TRAIT_FP32_TO_FP8E4M3>(vregY, vregFloatY,
296 mask);296 mask);
297- AscendC::MicroAPI::DataCopy<VarType, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(297+ AscendC::MicroAPI::StoreAlign<VarType, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(
298 outLocalAddr + i * VL + j * xLocalOffset, vregY, mask);298 outLocalAddr + i * VL + j * xLocalOffset, vregY, mask);
299 } else if constexpr (IsSameType<VarType, int8_t>::value) {299 } else if constexpr (IsSameType<VarType, int8_t>::value) {
300 // int8300 // int8
@@ -303,7 +303,7 @@ public:
303 AscendC::MicroAPI::Cast<half, int16_t, Base::CAST_TRAIT_INT16_TO_HALF>(vregHalfY, vregInt16Y,303 AscendC::MicroAPI::Cast<half, int16_t, Base::CAST_TRAIT_INT16_TO_HALF>(vregHalfY, vregInt16Y,
304 mask);304 mask);
305 AscendC::MicroAPI::Cast<int8_t, half, Base::CAST_TRAIT_HALF_TO_INT8>(vregY, vregHalfY, mask);305 AscendC::MicroAPI::Cast<int8_t, half, Base::CAST_TRAIT_HALF_TO_INT8>(vregY, vregHalfY, mask);
306- AscendC::MicroAPI::DataCopy<VarType, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(306+ AscendC::MicroAPI::StoreAlign<VarType, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(
307 outLocalAddr + i * VL + j * xLocalOffset, vregY, mask);307 outLocalAddr + i * VL + j * xLocalOffset, vregY, mask);
308 }308 }
309 }309 }
@@ -318,9 +318,9 @@ public:
318 LocalTensor<UpdatesType> updateLocal = inQueueUpdates_.DeQue<UpdatesType>();318 LocalTensor<UpdatesType> updateLocal = inQueueUpdates_.DeQue<UpdatesType>();
319 LocalTensor<VarType> outLocal = outQueueVar_.AllocTensor<VarType>();319 LocalTensor<VarType> outLocal = outQueueVar_.AllocTensor<VarType>();
320 320 
321- __local_mem__ UpdatesType* updateLocalAddr = (__local_mem__ UpdatesType*)updateLocal.GetPhyAddr();321+ __ubuf__ UpdatesType* updateLocalAddr = (__ubuf__ UpdatesType*)updateLocal.GetPhyAddr();
322- __local_mem__ ScalesType* scaleLocalAddr = (__local_mem__ ScalesType*)scaleLocal.GetPhyAddr();322+ __ubuf__ ScalesType* scaleLocalAddr = (__ubuf__ ScalesType*)scaleLocal.GetPhyAddr();
323- __local_mem__ VarType* outLocalAddr = (__local_mem__ VarType*)outLocal.GetPhyAddr();323+ __ubuf__ VarType* outLocalAddr = (__ubuf__ VarType*)outLocal.GetPhyAddr();
324 324 
325 uint16_t VL = AscendC::VECTOR_REG_WIDTH / sizeof(float);325 uint16_t VL = AscendC::VECTOR_REG_WIDTH / sizeof(float);
326 uint32_t xLocalOffset = static_cast<uint32_t>(tilingData_.updateOriLastDimAlign);326 uint32_t xLocalOffset = static_cast<uint32_t>(tilingData_.updateOriLastDimAlign);
@@ -354,12 +354,12 @@ public:
354 // ld and cast for update354 // ld and cast for update
355 if constexpr (IsSameType<UpdatesType, half>::value) {355 if constexpr (IsSameType<UpdatesType, half>::value) {
356 // fp16356 // fp16
357- AscendC::MicroAPI::DataCopy<half, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(357+ AscendC::MicroAPI::LoadAlign<half, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(
358 vregX, updateLocalAddr + i * VL + j * xLocalOffset);358 vregX, updateLocalAddr + i * VL + j * xLocalOffset);
359 AscendC::MicroAPI::Cast<float, half, Base::CAST_TRAIT_HALF_TO_FP32>(vregFloatX, vregX, mask);359 AscendC::MicroAPI::Cast<float, half, Base::CAST_TRAIT_HALF_TO_FP32>(vregFloatX, vregX, mask);
360 } else if constexpr (IsSameType<UpdatesType, bfloat16_t>::value) {360 } else if constexpr (IsSameType<UpdatesType, bfloat16_t>::value) {
361 // bf16361 // bf16
362- AscendC::MicroAPI::DataCopy<UpdatesType, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(362+ AscendC::MicroAPI::LoadAlign<UpdatesType, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(
363 vregX, updateLocalAddr + i * VL + j * xLocalOffset);363 vregX, updateLocalAddr + i * VL + j * xLocalOffset);
364 AscendC::MicroAPI::Cast<float, UpdatesType, Base::CAST_TRAIT_BF16_TO_FP32>(vregFloatX, vregX,364 AscendC::MicroAPI::Cast<float, UpdatesType, Base::CAST_TRAIT_BF16_TO_FP32>(vregFloatX, vregX,
365 mask);365 mask);
@@ -368,11 +368,11 @@ public:
368 // ld and cast for scale368 // ld and cast for scale
369 if constexpr (IsSameType<ScalesType, float>::value) {369 if constexpr (IsSameType<ScalesType, float>::value) {
370 // fp32370 // fp32
371- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(371+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(
372 vregFloatS, scaleLocalAddr + i * VL);372 vregFloatS, scaleLocalAddr + i * VL);
373 } else if constexpr (IsSameType<ScalesType, bfloat16_t>::value) {373 } else if constexpr (IsSameType<ScalesType, bfloat16_t>::value) {
374 // bf16374 // bf16
375- AscendC::MicroAPI::DataCopy<ScalesType, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(375+ AscendC::MicroAPI::LoadAlign<ScalesType, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(
376 vregS, scaleLocalAddr + i * VL);376 vregS, scaleLocalAddr + i * VL);
377 AscendC::MicroAPI::Cast<float, ScalesType, Base::CAST_TRAIT_BF16_TO_FP32>(vregFloatS, vregS,377 AscendC::MicroAPI::Cast<float, ScalesType, Base::CAST_TRAIT_BF16_TO_FP32>(vregFloatS, vregS,
378 mask);378 mask);
@@ -391,19 +391,19 @@ public:
391 // hifp8391 // hifp8
392 AscendC::MicroAPI::Cast<VarType, float, Base::CAST_TRAIT_FP32_TO_HIFP8>(vregY, vregFloatY,392 AscendC::MicroAPI::Cast<VarType, float, Base::CAST_TRAIT_FP32_TO_HIFP8>(vregY, vregFloatY,
393 mask);393 mask);
394- AscendC::MicroAPI::DataCopy<VarType, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(394+ AscendC::MicroAPI::StoreAlign<VarType, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(
395 outLocalAddr + i * VL + j * xLocalOffset, vregY, mask);395 outLocalAddr + i * VL + j * xLocalOffset, vregY, mask);
396 } else if constexpr (IsSameType<VarType, fp8_e5m2_t>::value) {396 } else if constexpr (IsSameType<VarType, fp8_e5m2_t>::value) {
397 // fp8_e5m2397 // fp8_e5m2
398 AscendC::MicroAPI::Cast<VarType, float, Base::CAST_TRAIT_FP32_TO_FP8E5M2>(vregY, vregFloatY,398 AscendC::MicroAPI::Cast<VarType, float, Base::CAST_TRAIT_FP32_TO_FP8E5M2>(vregY, vregFloatY,
399 mask);399 mask);
400- AscendC::MicroAPI::DataCopy<VarType, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(400+ AscendC::MicroAPI::StoreAlign<VarType, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(
401 outLocalAddr + i * VL + j * xLocalOffset, vregY, mask);401 outLocalAddr + i * VL + j * xLocalOffset, vregY, mask);
402 } else if constexpr (IsSameType<VarType, fp8_e4m3fn_t>::value) {402 } else if constexpr (IsSameType<VarType, fp8_e4m3fn_t>::value) {
403 // fp8_e4m3403 // fp8_e4m3
404 AscendC::MicroAPI::Cast<VarType, float, Base::CAST_TRAIT_FP32_TO_FP8E4M3>(vregY, vregFloatY,404 AscendC::MicroAPI::Cast<VarType, float, Base::CAST_TRAIT_FP32_TO_FP8E4M3>(vregY, vregFloatY,
405 mask);405 mask);
406- AscendC::MicroAPI::DataCopy<VarType, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(406+ AscendC::MicroAPI::StoreAlign<VarType, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(
407 outLocalAddr + i * VL + j * xLocalOffset, vregY, mask);407 outLocalAddr + i * VL + j * xLocalOffset, vregY, mask);
408 } else if constexpr (IsSameType<VarType, int8_t>::value) {408 } else if constexpr (IsSameType<VarType, int8_t>::value) {
409 // int8409 // int8
@@ -412,7 +412,7 @@ public:
412 AscendC::MicroAPI::Cast<half, int16_t, Base::CAST_TRAIT_INT16_TO_HALF>(vregHalfY, vregInt16Y,412 AscendC::MicroAPI::Cast<half, int16_t, Base::CAST_TRAIT_INT16_TO_HALF>(vregHalfY, vregInt16Y,
413 mask);413 mask);
414 AscendC::MicroAPI::Cast<int8_t, half, Base::CAST_TRAIT_HALF_TO_INT8>(vregY, vregHalfY, mask);414 AscendC::MicroAPI::Cast<int8_t, half, Base::CAST_TRAIT_HALF_TO_INT8>(vregY, vregHalfY, mask);
415- AscendC::MicroAPI::DataCopy<VarType, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(415+ AscendC::MicroAPI::StoreAlign<VarType, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(
416 outLocalAddr + i * VL + j * xLocalOffset, vregY, mask);416 outLocalAddr + i * VL + j * xLocalOffset, vregY, mask);
417 }417 }
418 }418 }
@@ -476,4 +476,4 @@ public:
476 }476 }
477};477};
478} // namespace QuantUpdateScatter478} // namespace QuantUpdateScatter
479-#endif // QUANT_UPDATE_SCATTER_LARGE_BATCH_LITTLE_QUANT_REGBASE_H_479+#endif // QUANT_UPDATE_SCATTER_LARGE_BATCH_LITTLE_QUANT_REGBASE_H_
Mindex/quant_update_scatter/op_kernel/arch35/quant_update_scatter_large_batch_regbase.h+26-26
@@ -176,10 +176,10 @@ public:
176 LocalTensor<UpdatesType> updateLocal = inQueueUpdates_.DeQue<UpdatesType>();176 LocalTensor<UpdatesType> updateLocal = inQueueUpdates_.DeQue<UpdatesType>();
177 LocalTensor<VarType> outLocal = outQueueVar_.AllocTensor<VarType>();177 LocalTensor<VarType> outLocal = outQueueVar_.AllocTensor<VarType>();
178 178 
179- __local_mem__ UpdatesType* updateLocalAddr = (__local_mem__ UpdatesType*)updateLocal.GetPhyAddr();179+ __ubuf__ UpdatesType* updateLocalAddr = (__ubuf__ UpdatesType*)updateLocal.GetPhyAddr();
180- __local_mem__ ScalesType* scaleLocalAddr = (__local_mem__ ScalesType*)scaleLocal.GetPhyAddr();180+ __ubuf__ ScalesType* scaleLocalAddr = (__ubuf__ ScalesType*)scaleLocal.GetPhyAddr();
181- __local_mem__ OffsetsType* offsetLocalAddr = (__local_mem__ OffsetsType*)offsetLocal.GetPhyAddr();181+ __ubuf__ OffsetsType* offsetLocalAddr = (__ubuf__ OffsetsType*)offsetLocal.GetPhyAddr();
182- __local_mem__ VarType* outLocalAddr = (__local_mem__ VarType*)outLocal.GetPhyAddr();182+ __ubuf__ VarType* outLocalAddr = (__ubuf__ VarType*)outLocal.GetPhyAddr();
183 183 
184 uint16_t VL = AscendC::VECTOR_REG_WIDTH / sizeof(float);184 uint16_t VL = AscendC::VECTOR_REG_WIDTH / sizeof(float);
185 uint32_t xLocalOffset = tilingData_.updateOriLastDimAlign;185 uint32_t xLocalOffset = tilingData_.updateOriLastDimAlign;
@@ -213,12 +213,12 @@ public:
213 // ld and cast for update213 // ld and cast for update
214 if constexpr (IsSameType<UpdatesType, half>::value) {214 if constexpr (IsSameType<UpdatesType, half>::value) {
215 // fp16215 // fp16
216- AscendC::MicroAPI::DataCopy<half, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(216+ AscendC::MicroAPI::LoadAlign<half, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(
217 vregX, updateLocalAddr + i * VL + j * xLocalOffset);217 vregX, updateLocalAddr + i * VL + j * xLocalOffset);
218 AscendC::MicroAPI::Cast<float, half, Base::CAST_TRAIT_HALF_TO_FP32>(vregFloatX, vregX, mask);218 AscendC::MicroAPI::Cast<float, half, Base::CAST_TRAIT_HALF_TO_FP32>(vregFloatX, vregX, mask);
219 } else if constexpr (IsSameType<UpdatesType, bfloat16_t>::value) {219 } else if constexpr (IsSameType<UpdatesType, bfloat16_t>::value) {
220 // bf16220 // bf16
221- AscendC::MicroAPI::DataCopy<UpdatesType, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(221+ AscendC::MicroAPI::LoadAlign<UpdatesType, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(
222 vregX, updateLocalAddr + i * VL + j * xLocalOffset);222 vregX, updateLocalAddr + i * VL + j * xLocalOffset);
223 AscendC::MicroAPI::Cast<float, UpdatesType, Base::CAST_TRAIT_BF16_TO_FP32>(vregFloatX, vregX,223 AscendC::MicroAPI::Cast<float, UpdatesType, Base::CAST_TRAIT_BF16_TO_FP32>(vregFloatX, vregX,
224 mask);224 mask);
@@ -227,11 +227,11 @@ public:
227 // ld and cast for scale227 // ld and cast for scale
228 if constexpr (IsSameType<ScalesType, float>::value) {228 if constexpr (IsSameType<ScalesType, float>::value) {
229 // fp32229 // fp32
230- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(230+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(
231 vregFloatS, scaleLocalAddr + i * VL);231 vregFloatS, scaleLocalAddr + i * VL);
232 } else if constexpr (IsSameType<ScalesType, bfloat16_t>::value) {232 } else if constexpr (IsSameType<ScalesType, bfloat16_t>::value) {
233 // bf16233 // bf16
234- AscendC::MicroAPI::DataCopy<ScalesType, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(234+ AscendC::MicroAPI::LoadAlign<ScalesType, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(
235 vregS, scaleLocalAddr + i * VL);235 vregS, scaleLocalAddr + i * VL);
236 AscendC::MicroAPI::Cast<float, ScalesType, Base::CAST_TRAIT_BF16_TO_FP32>(vregFloatS, vregS,236 AscendC::MicroAPI::Cast<float, ScalesType, Base::CAST_TRAIT_BF16_TO_FP32>(vregFloatS, vregS,
237 mask);237 mask);
@@ -239,13 +239,13 @@ public:
239 // ld and cast for offset239 // ld and cast for offset
240 if constexpr (IsSameType<OffsetsType, int32_t>::value) {240 if constexpr (IsSameType<OffsetsType, int32_t>::value) {
241 // int32241 // int32
242- AscendC::MicroAPI::DataCopy<OffsetsType, AscendC::MicroAPI::LoadDist::DIST_NORM>(242+ AscendC::MicroAPI::LoadAlign<OffsetsType, AscendC::MicroAPI::LoadDist::DIST_NORM>(
243 vregO, offsetLocalAddr + i * VL);243 vregO, offsetLocalAddr + i * VL);
244 AscendC::MicroAPI::Cast<float, OffsetsType, Base::CAST_TRAIT_INT32_TO_FP32>(vregFloatO, vregO,244 AscendC::MicroAPI::Cast<float, OffsetsType, Base::CAST_TRAIT_INT32_TO_FP32>(vregFloatO, vregO,
245 mask);245 mask);
246 } else if constexpr (IsSameType<OffsetsType, bfloat16_t>::value) {246 } else if constexpr (IsSameType<OffsetsType, bfloat16_t>::value) {
247 // bf16247 // bf16
248- AscendC::MicroAPI::DataCopy<OffsetsType, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(248+ AscendC::MicroAPI::LoadAlign<OffsetsType, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(
249 vregO, offsetLocalAddr + i * VL);249 vregO, offsetLocalAddr + i * VL);
250 AscendC::MicroAPI::Cast<float, OffsetsType, Base::CAST_TRAIT_BF16_TO_FP32>(vregFloatO, vregO,250 AscendC::MicroAPI::Cast<float, OffsetsType, Base::CAST_TRAIT_BF16_TO_FP32>(vregFloatO, vregO,
251 mask);251 mask);
@@ -267,19 +267,19 @@ public:
267 // hifp8267 // hifp8
268 AscendC::MicroAPI::Cast<VarType, float, Base::CAST_TRAIT_FP32_TO_HIFP8>(vregY, vregFloatY,268 AscendC::MicroAPI::Cast<VarType, float, Base::CAST_TRAIT_FP32_TO_HIFP8>(vregY, vregFloatY,
269 mask);269 mask);
270- AscendC::MicroAPI::DataCopy<VarType, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(270+ AscendC::MicroAPI::StoreAlign<VarType, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(
271 outLocalAddr + i * VL + j * xLocalOffset, vregY, mask);271 outLocalAddr + i * VL + j * xLocalOffset, vregY, mask);
272 } else if constexpr (IsSameType<VarType, fp8_e5m2_t>::value) {272 } else if constexpr (IsSameType<VarType, fp8_e5m2_t>::value) {
273 // fp8_e5m2273 // fp8_e5m2
274 AscendC::MicroAPI::Cast<VarType, float, Base::CAST_TRAIT_FP32_TO_FP8E5M2>(vregY, vregFloatY,274 AscendC::MicroAPI::Cast<VarType, float, Base::CAST_TRAIT_FP32_TO_FP8E5M2>(vregY, vregFloatY,
275 mask);275 mask);
276- AscendC::MicroAPI::DataCopy<VarType, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(276+ AscendC::MicroAPI::StoreAlign<VarType, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(
277 outLocalAddr + i * VL + j * xLocalOffset, vregY, mask);277 outLocalAddr + i * VL + j * xLocalOffset, vregY, mask);
278 } else if constexpr (IsSameType<VarType, fp8_e4m3fn_t>::value) {278 } else if constexpr (IsSameType<VarType, fp8_e4m3fn_t>::value) {
279 // fp8_e4m3279 // fp8_e4m3
280 AscendC::MicroAPI::Cast<VarType, float, Base::CAST_TRAIT_FP32_TO_FP8E4M3>(vregY, vregFloatY,280 AscendC::MicroAPI::Cast<VarType, float, Base::CAST_TRAIT_FP32_TO_FP8E4M3>(vregY, vregFloatY,
281 mask);281 mask);
282- AscendC::MicroAPI::DataCopy<VarType, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(282+ AscendC::MicroAPI::StoreAlign<VarType, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(
283 outLocalAddr + i * VL + j * xLocalOffset, vregY, mask);283 outLocalAddr + i * VL + j * xLocalOffset, vregY, mask);
284 } else if constexpr (IsSameType<VarType, int8_t>::value) {284 } else if constexpr (IsSameType<VarType, int8_t>::value) {
285 // int8285 // int8
@@ -288,7 +288,7 @@ public:
288 AscendC::MicroAPI::Cast<half, int16_t, Base::CAST_TRAIT_INT16_TO_HALF>(vregHalfY, vregInt16Y,288 AscendC::MicroAPI::Cast<half, int16_t, Base::CAST_TRAIT_INT16_TO_HALF>(vregHalfY, vregInt16Y,
289 mask);289 mask);
290 AscendC::MicroAPI::Cast<int8_t, half, Base::CAST_TRAIT_HALF_TO_INT8>(vregY, vregHalfY, mask);290 AscendC::MicroAPI::Cast<int8_t, half, Base::CAST_TRAIT_HALF_TO_INT8>(vregY, vregHalfY, mask);
291- AscendC::MicroAPI::DataCopy<VarType, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(291+ AscendC::MicroAPI::StoreAlign<VarType, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(
292 outLocalAddr + i * VL + j * xLocalOffset, vregY, mask);292 outLocalAddr + i * VL + j * xLocalOffset, vregY, mask);
293 }293 }
294 }294 }
@@ -303,9 +303,9 @@ public:
303 LocalTensor<UpdatesType> updateLocal = inQueueUpdates_.DeQue<UpdatesType>();303 LocalTensor<UpdatesType> updateLocal = inQueueUpdates_.DeQue<UpdatesType>();
304 LocalTensor<VarType> outLocal = outQueueVar_.AllocTensor<VarType>();304 LocalTensor<VarType> outLocal = outQueueVar_.AllocTensor<VarType>();
305 305 
306- __local_mem__ UpdatesType* updateLocalAddr = (__local_mem__ UpdatesType*)updateLocal.GetPhyAddr();306+ __ubuf__ UpdatesType* updateLocalAddr = (__ubuf__ UpdatesType*)updateLocal.GetPhyAddr();
307- __local_mem__ ScalesType* scaleLocalAddr = (__local_mem__ ScalesType*)scaleLocal.GetPhyAddr();307+ __ubuf__ ScalesType* scaleLocalAddr = (__ubuf__ ScalesType*)scaleLocal.GetPhyAddr();
308- __local_mem__ VarType* outLocalAddr = (__local_mem__ VarType*)outLocal.GetPhyAddr();308+ __ubuf__ VarType* outLocalAddr = (__ubuf__ VarType*)outLocal.GetPhyAddr();
309 309 
310 uint16_t VL = AscendC::VECTOR_REG_WIDTH / sizeof(float);310 uint16_t VL = AscendC::VECTOR_REG_WIDTH / sizeof(float);
311 uint32_t xLocalOffset = tilingData_.updateOriLastDimAlign;311 uint32_t xLocalOffset = tilingData_.updateOriLastDimAlign;
@@ -335,12 +335,12 @@ public:
335 // ld and cast for update335 // ld and cast for update
336 if constexpr (IsSameType<UpdatesType, half>::value) {336 if constexpr (IsSameType<UpdatesType, half>::value) {
337 // fp16337 // fp16
338- AscendC::MicroAPI::DataCopy<half, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(338+ AscendC::MicroAPI::LoadAlign<half, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(
339 vregX, updateLocalAddr + i * VL + j * xLocalOffset);339 vregX, updateLocalAddr + i * VL + j * xLocalOffset);
340 AscendC::MicroAPI::Cast<float, half, Base::CAST_TRAIT_HALF_TO_FP32>(vregFloatX, vregX, mask);340 AscendC::MicroAPI::Cast<float, half, Base::CAST_TRAIT_HALF_TO_FP32>(vregFloatX, vregX, mask);
341 } else if constexpr (IsSameType<UpdatesType, bfloat16_t>::value) {341 } else if constexpr (IsSameType<UpdatesType, bfloat16_t>::value) {
342 // bf16342 // bf16
343- AscendC::MicroAPI::DataCopy<UpdatesType, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(343+ AscendC::MicroAPI::LoadAlign<UpdatesType, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(
344 vregX, updateLocalAddr + i * VL + j * xLocalOffset);344 vregX, updateLocalAddr + i * VL + j * xLocalOffset);
345 AscendC::MicroAPI::Cast<float, UpdatesType, Base::CAST_TRAIT_BF16_TO_FP32>(vregFloatX, vregX,345 AscendC::MicroAPI::Cast<float, UpdatesType, Base::CAST_TRAIT_BF16_TO_FP32>(vregFloatX, vregX,
346 mask);346 mask);
@@ -349,11 +349,11 @@ public:
349 // ld and cast for scale349 // ld and cast for scale
350 if constexpr (IsSameType<ScalesType, float>::value) {350 if constexpr (IsSameType<ScalesType, float>::value) {
351 // fp32351 // fp32
352- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(352+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(
353 vregFloatS, scaleLocalAddr + i * VL);353 vregFloatS, scaleLocalAddr + i * VL);
354 } else if constexpr (IsSameType<ScalesType, bfloat16_t>::value) {354 } else if constexpr (IsSameType<ScalesType, bfloat16_t>::value) {
355 // bf16355 // bf16
356- AscendC::MicroAPI::DataCopy<ScalesType, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(356+ AscendC::MicroAPI::LoadAlign<ScalesType, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(
357 vregS, scaleLocalAddr + i * VL);357 vregS, scaleLocalAddr + i * VL);
358 AscendC::MicroAPI::Cast<float, ScalesType, Base::CAST_TRAIT_BF16_TO_FP32>(vregFloatS, vregS,358 AscendC::MicroAPI::Cast<float, ScalesType, Base::CAST_TRAIT_BF16_TO_FP32>(vregFloatS, vregS,
359 mask);359 mask);
@@ -372,19 +372,19 @@ public:
372 // hifp8372 // hifp8
373 AscendC::MicroAPI::Cast<VarType, float, Base::CAST_TRAIT_FP32_TO_HIFP8>(vregY, vregFloatY,373 AscendC::MicroAPI::Cast<VarType, float, Base::CAST_TRAIT_FP32_TO_HIFP8>(vregY, vregFloatY,
374 mask);374 mask);
375- AscendC::MicroAPI::DataCopy<VarType, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(375+ AscendC::MicroAPI::StoreAlign<VarType, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(
376 outLocalAddr + i * VL + j * xLocalOffset, vregY, mask);376 outLocalAddr + i * VL + j * xLocalOffset, vregY, mask);
377 } else if constexpr (IsSameType<VarType, fp8_e5m2_t>::value) {377 } else if constexpr (IsSameType<VarType, fp8_e5m2_t>::value) {
378 // fp8_e5m2378 // fp8_e5m2
379 AscendC::MicroAPI::Cast<VarType, float, Base::CAST_TRAIT_FP32_TO_FP8E5M2>(vregY, vregFloatY,379 AscendC::MicroAPI::Cast<VarType, float, Base::CAST_TRAIT_FP32_TO_FP8E5M2>(vregY, vregFloatY,
380 mask);380 mask);
381- AscendC::MicroAPI::DataCopy<VarType, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(381+ AscendC::MicroAPI::StoreAlign<VarType, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(
382 outLocalAddr + i * VL + j * xLocalOffset, vregY, mask);382 outLocalAddr + i * VL + j * xLocalOffset, vregY, mask);
383 } else if constexpr (IsSameType<VarType, fp8_e4m3fn_t>::value) {383 } else if constexpr (IsSameType<VarType, fp8_e4m3fn_t>::value) {
384 // fp8_e4m3384 // fp8_e4m3
385 AscendC::MicroAPI::Cast<VarType, float, Base::CAST_TRAIT_FP32_TO_FP8E4M3>(vregY, vregFloatY,385 AscendC::MicroAPI::Cast<VarType, float, Base::CAST_TRAIT_FP32_TO_FP8E4M3>(vregY, vregFloatY,
386 mask);386 mask);
387- AscendC::MicroAPI::DataCopy<VarType, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(387+ AscendC::MicroAPI::StoreAlign<VarType, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(
388 outLocalAddr + i * VL + j * xLocalOffset, vregY, mask);388 outLocalAddr + i * VL + j * xLocalOffset, vregY, mask);
389 } else if constexpr (IsSameType<VarType, int8_t>::value) {389 } else if constexpr (IsSameType<VarType, int8_t>::value) {
390 // int8390 // int8
@@ -393,7 +393,7 @@ public:
393 AscendC::MicroAPI::Cast<half, int16_t, Base::CAST_TRAIT_INT16_TO_HALF>(vregHalfY, vregInt16Y,393 AscendC::MicroAPI::Cast<half, int16_t, Base::CAST_TRAIT_INT16_TO_HALF>(vregHalfY, vregInt16Y,
394 mask);394 mask);
395 AscendC::MicroAPI::Cast<int8_t, half, Base::CAST_TRAIT_HALF_TO_INT8>(vregY, vregHalfY, mask);395 AscendC::MicroAPI::Cast<int8_t, half, Base::CAST_TRAIT_HALF_TO_INT8>(vregY, vregHalfY, mask);
396- AscendC::MicroAPI::DataCopy<VarType, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(396+ AscendC::MicroAPI::StoreAlign<VarType, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(
397 outLocalAddr + i * VL + j * xLocalOffset, vregY, mask);397 outLocalAddr + i * VL + j * xLocalOffset, vregY, mask);
398 }398 }
399 }399 }
@@ -453,4 +453,4 @@ public:
453 }453 }
454};454};
455} // namespace QuantUpdateScatter455} // namespace QuantUpdateScatter
456-#endif // QuantUpdateScatterLargeBatchRegbase456+#endif // QuantUpdateScatterLargeBatchRegbase
Mindex/quant_update_scatter/op_kernel/arch35/quant_update_scatter_large_ele_large_quant_regbase.h+28-28
@@ -187,10 +187,10 @@ public:
187 187 
188 LocalTensor<VarType> outLocal = outQueueVar_.AllocTensor<VarType>();188 LocalTensor<VarType> outLocal = outQueueVar_.AllocTensor<VarType>();
189 189 
190- __local_mem__ UpdatesType* updateLocalAddr = (__local_mem__ UpdatesType*)updateLocal.GetPhyAddr();190+ __ubuf__ UpdatesType* updateLocalAddr = (__ubuf__ UpdatesType*)updateLocal.GetPhyAddr();
191- __local_mem__ ScalesType* scaleLocalAddr = (__local_mem__ ScalesType*)scaleLocal.GetPhyAddr();191+ __ubuf__ ScalesType* scaleLocalAddr = (__ubuf__ ScalesType*)scaleLocal.GetPhyAddr();
192- __local_mem__ OffsetsType* offsetLocalAddr = (__local_mem__ OffsetsType*)offsetLocal.GetPhyAddr();192+ __ubuf__ OffsetsType* offsetLocalAddr = (__ubuf__ OffsetsType*)offsetLocal.GetPhyAddr();
193- __local_mem__ VarType* outLocalAddr = (__local_mem__ VarType*)outLocal.GetPhyAddr();193+ __ubuf__ VarType* outLocalAddr = (__ubuf__ VarType*)outLocal.GetPhyAddr();
194 194 
195 uint16_t VL = AscendC::VECTOR_REG_WIDTH / sizeof(float);195 uint16_t VL = AscendC::VECTOR_REG_WIDTH / sizeof(float);
196 uint32_t count = static_cast<uint32_t>(dataCount);196 uint32_t count = static_cast<uint32_t>(dataCount);
@@ -225,12 +225,12 @@ public:
225 // ld and cast for update225 // ld and cast for update
226 if constexpr (IsSameType<UpdatesType, half>::value) {226 if constexpr (IsSameType<UpdatesType, half>::value) {
227 // fp16227 // fp16
228- AscendC::MicroAPI::DataCopy<half, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(228+ AscendC::MicroAPI::LoadAlign<half, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(
229 vregX, updateLocalAddr + i * VL);229 vregX, updateLocalAddr + i * VL);
230 AscendC::MicroAPI::Cast<float, half, Base::CAST_TRAIT_HALF_TO_FP32>(vregFloatX, vregX, mask);230 AscendC::MicroAPI::Cast<float, half, Base::CAST_TRAIT_HALF_TO_FP32>(vregFloatX, vregX, mask);
231 } else if constexpr (IsSameType<UpdatesType, bfloat16_t>::value) {231 } else if constexpr (IsSameType<UpdatesType, bfloat16_t>::value) {
232 // bf16232 // bf16
233- AscendC::MicroAPI::DataCopy<UpdatesType, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(233+ AscendC::MicroAPI::LoadAlign<UpdatesType, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(
234 vregX, updateLocalAddr + i * VL);234 vregX, updateLocalAddr + i * VL);
235 AscendC::MicroAPI::Cast<float, UpdatesType, Base::CAST_TRAIT_BF16_TO_FP32>(vregFloatX, vregX, mask);235 AscendC::MicroAPI::Cast<float, UpdatesType, Base::CAST_TRAIT_BF16_TO_FP32>(vregFloatX, vregX, mask);
236 }236 }
@@ -238,24 +238,24 @@ public:
238 // ld and cast for scale238 // ld and cast for scale
239 if constexpr (IsSameType<ScalesType, float>::value) {239 if constexpr (IsSameType<ScalesType, float>::value) {
240 // fp32240 // fp32
241- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(vregFloatS,241+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(
242- scaleLocalAddr + i * VL);242+ vregFloatS, scaleLocalAddr + i * VL);
243 } else if constexpr (IsSameType<ScalesType, bfloat16_t>::value) {243 } else if constexpr (IsSameType<ScalesType, bfloat16_t>::value) {
244 // bf16244 // bf16
245- AscendC::MicroAPI::DataCopy<ScalesType, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(245+ AscendC::MicroAPI::LoadAlign<ScalesType, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(
246 vregS, scaleLocalAddr + i * VL);246 vregS, scaleLocalAddr + i * VL);
247 AscendC::MicroAPI::Cast<float, ScalesType, Base::CAST_TRAIT_BF16_TO_FP32>(vregFloatS, vregS, mask);247 AscendC::MicroAPI::Cast<float, ScalesType, Base::CAST_TRAIT_BF16_TO_FP32>(vregFloatS, vregS, mask);
248 }248 }
249 // ld and cast for offset249 // ld and cast for offset
250 if constexpr (IsSameType<OffsetsType, int32_t>::value) {250 if constexpr (IsSameType<OffsetsType, int32_t>::value) {
251 // int32251 // int32
252- AscendC::MicroAPI::DataCopy<OffsetsType, AscendC::MicroAPI::LoadDist::DIST_NORM>(252+ AscendC::MicroAPI::LoadAlign<OffsetsType, AscendC::MicroAPI::LoadDist::DIST_NORM>(
253 vregO, offsetLocalAddr + i * VL);253 vregO, offsetLocalAddr + i * VL);
254 AscendC::MicroAPI::Cast<float, OffsetsType, Base::CAST_TRAIT_INT32_TO_FP32>(vregFloatO, vregO,254 AscendC::MicroAPI::Cast<float, OffsetsType, Base::CAST_TRAIT_INT32_TO_FP32>(vregFloatO, vregO,
255 mask);255 mask);
256 } else if constexpr (IsSameType<OffsetsType, bfloat16_t>::value) {256 } else if constexpr (IsSameType<OffsetsType, bfloat16_t>::value) {
257 // bf16257 // bf16
258- AscendC::MicroAPI::DataCopy<OffsetsType, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(258+ AscendC::MicroAPI::LoadAlign<OffsetsType, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(
259 vregO, offsetLocalAddr + i * VL);259 vregO, offsetLocalAddr + i * VL);
260 AscendC::MicroAPI::Cast<float, OffsetsType, Base::CAST_TRAIT_BF16_TO_FP32>(vregFloatO, vregO, mask);260 AscendC::MicroAPI::Cast<float, OffsetsType, Base::CAST_TRAIT_BF16_TO_FP32>(vregFloatO, vregO, mask);
261 }261 }
@@ -275,17 +275,17 @@ public:
275 if constexpr (IsSameType<VarType, hifloat8_t>::value) {275 if constexpr (IsSameType<VarType, hifloat8_t>::value) {
276 // hifp8276 // hifp8
277 AscendC::MicroAPI::Cast<VarType, float, Base::CAST_TRAIT_FP32_TO_HIFP8>(vregY, vregFloatY, mask);277 AscendC::MicroAPI::Cast<VarType, float, Base::CAST_TRAIT_FP32_TO_HIFP8>(vregY, vregFloatY, mask);
278- AscendC::MicroAPI::DataCopy<VarType, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(278+ AscendC::MicroAPI::StoreAlign<VarType, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(
279 outLocalAddr + i * VL, vregY, mask);279 outLocalAddr + i * VL, vregY, mask);
280 } else if constexpr (IsSameType<VarType, fp8_e5m2_t>::value) {280 } else if constexpr (IsSameType<VarType, fp8_e5m2_t>::value) {
281 // fp8_e5m2281 // fp8_e5m2
282 AscendC::MicroAPI::Cast<VarType, float, Base::CAST_TRAIT_FP32_TO_FP8E5M2>(vregY, vregFloatY, mask);282 AscendC::MicroAPI::Cast<VarType, float, Base::CAST_TRAIT_FP32_TO_FP8E5M2>(vregY, vregFloatY, mask);
283- AscendC::MicroAPI::DataCopy<VarType, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(283+ AscendC::MicroAPI::StoreAlign<VarType, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(
284 outLocalAddr + i * VL, vregY, mask);284 outLocalAddr + i * VL, vregY, mask);
285 } else if constexpr (IsSameType<VarType, fp8_e4m3fn_t>::value) {285 } else if constexpr (IsSameType<VarType, fp8_e4m3fn_t>::value) {
286 // fp8_e4m3286 // fp8_e4m3
287 AscendC::MicroAPI::Cast<VarType, float, Base::CAST_TRAIT_FP32_TO_FP8E4M3>(vregY, vregFloatY, mask);287 AscendC::MicroAPI::Cast<VarType, float, Base::CAST_TRAIT_FP32_TO_FP8E4M3>(vregY, vregFloatY, mask);
288- AscendC::MicroAPI::DataCopy<VarType, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(288+ AscendC::MicroAPI::StoreAlign<VarType, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(
289 outLocalAddr + i * VL, vregY, mask);289 outLocalAddr + i * VL, vregY, mask);
290 } else if constexpr (IsSameType<VarType, int8_t>::value) {290 } else if constexpr (IsSameType<VarType, int8_t>::value) {
291 // int8291 // int8
@@ -293,7 +293,7 @@ public:
293 mask);293 mask);
294 AscendC::MicroAPI::Cast<half, int16_t, Base::CAST_TRAIT_INT16_TO_HALF>(vregHalfY, vregInt16Y, mask);294 AscendC::MicroAPI::Cast<half, int16_t, Base::CAST_TRAIT_INT16_TO_HALF>(vregHalfY, vregInt16Y, mask);
295 AscendC::MicroAPI::Cast<int8_t, half, Base::CAST_TRAIT_HALF_TO_INT8>(vregY, vregHalfY, mask);295 AscendC::MicroAPI::Cast<int8_t, half, Base::CAST_TRAIT_HALF_TO_INT8>(vregY, vregHalfY, mask);
296- AscendC::MicroAPI::DataCopy<VarType, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(296+ AscendC::MicroAPI::StoreAlign<VarType, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(
297 outLocalAddr + i * VL, vregY, mask);297 outLocalAddr + i * VL, vregY, mask);
298 }298 }
299 }299 }
@@ -311,9 +311,9 @@ public:
311 311 
312 LocalTensor<VarType> outLocal = outQueueVar_.AllocTensor<VarType>();312 LocalTensor<VarType> outLocal = outQueueVar_.AllocTensor<VarType>();
313 313 
314- __local_mem__ UpdatesType* updateLocalAddr = (__local_mem__ UpdatesType*)updateLocal.GetPhyAddr();314+ __ubuf__ UpdatesType* updateLocalAddr = (__ubuf__ UpdatesType*)updateLocal.GetPhyAddr();
315- __local_mem__ ScalesType* scaleLocalAddr = (__local_mem__ ScalesType*)scaleLocal.GetPhyAddr();315+ __ubuf__ ScalesType* scaleLocalAddr = (__ubuf__ ScalesType*)scaleLocal.GetPhyAddr();
316- __local_mem__ VarType* outLocalAddr = (__local_mem__ VarType*)outLocal.GetPhyAddr();316+ __ubuf__ VarType* outLocalAddr = (__ubuf__ VarType*)outLocal.GetPhyAddr();
317 317 
318 uint16_t VL = AscendC::VECTOR_REG_WIDTH / sizeof(float);318 uint16_t VL = AscendC::VECTOR_REG_WIDTH / sizeof(float);
319 uint32_t count = static_cast<uint32_t>(dataCount);319 uint32_t count = static_cast<uint32_t>(dataCount);
@@ -344,12 +344,12 @@ public:
344 // ld and cast for update344 // ld and cast for update
345 if constexpr (IsSameType<UpdatesType, half>::value) {345 if constexpr (IsSameType<UpdatesType, half>::value) {
346 // fp16346 // fp16
347- AscendC::MicroAPI::DataCopy<half, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(347+ AscendC::MicroAPI::LoadAlign<half, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(
348 vregX, updateLocalAddr + i * VL);348 vregX, updateLocalAddr + i * VL);
349 AscendC::MicroAPI::Cast<float, half, Base::CAST_TRAIT_HALF_TO_FP32>(vregFloatX, vregX, mask);349 AscendC::MicroAPI::Cast<float, half, Base::CAST_TRAIT_HALF_TO_FP32>(vregFloatX, vregX, mask);
350 } else if constexpr (IsSameType<UpdatesType, bfloat16_t>::value) {350 } else if constexpr (IsSameType<UpdatesType, bfloat16_t>::value) {
351 // bf16351 // bf16
352- AscendC::MicroAPI::DataCopy<UpdatesType, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(352+ AscendC::MicroAPI::LoadAlign<UpdatesType, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(
353 vregX, updateLocalAddr + i * VL);353 vregX, updateLocalAddr + i * VL);
354 AscendC::MicroAPI::Cast<float, UpdatesType, Base::CAST_TRAIT_BF16_TO_FP32>(vregFloatX, vregX, mask);354 AscendC::MicroAPI::Cast<float, UpdatesType, Base::CAST_TRAIT_BF16_TO_FP32>(vregFloatX, vregX, mask);
355 }355 }
@@ -357,11 +357,11 @@ public:
357 // ld and cast for scale357 // ld and cast for scale
358 if constexpr (IsSameType<ScalesType, float>::value) {358 if constexpr (IsSameType<ScalesType, float>::value) {
359 // fp32359 // fp32
360- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(vregFloatS,360+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(
361- scaleLocalAddr + i * VL);361+ vregFloatS, scaleLocalAddr + i * VL);
362 } else if constexpr (IsSameType<ScalesType, bfloat16_t>::value) {362 } else if constexpr (IsSameType<ScalesType, bfloat16_t>::value) {
363 // bf16363 // bf16
364- AscendC::MicroAPI::DataCopy<ScalesType, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(364+ AscendC::MicroAPI::LoadAlign<ScalesType, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(
365 vregS, scaleLocalAddr + i * VL);365 vregS, scaleLocalAddr + i * VL);
366 AscendC::MicroAPI::Cast<float, ScalesType, Base::CAST_TRAIT_BF16_TO_FP32>(vregFloatS, vregS, mask);366 AscendC::MicroAPI::Cast<float, ScalesType, Base::CAST_TRAIT_BF16_TO_FP32>(vregFloatS, vregS, mask);
367 }367 }
@@ -378,17 +378,17 @@ public:
378 if constexpr (IsSameType<VarType, hifloat8_t>::value) {378 if constexpr (IsSameType<VarType, hifloat8_t>::value) {
379 // hifp8379 // hifp8
380 AscendC::MicroAPI::Cast<VarType, float, Base::CAST_TRAIT_FP32_TO_HIFP8>(vregY, vregFloatY, mask);380 AscendC::MicroAPI::Cast<VarType, float, Base::CAST_TRAIT_FP32_TO_HIFP8>(vregY, vregFloatY, mask);
381- AscendC::MicroAPI::DataCopy<VarType, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(381+ AscendC::MicroAPI::StoreAlign<VarType, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(
382 outLocalAddr + i * VL, vregY, mask);382 outLocalAddr + i * VL, vregY, mask);
383 } else if constexpr (IsSameType<VarType, fp8_e5m2_t>::value) {383 } else if constexpr (IsSameType<VarType, fp8_e5m2_t>::value) {
384 // fp8_e5m2384 // fp8_e5m2
385 AscendC::MicroAPI::Cast<VarType, float, Base::CAST_TRAIT_FP32_TO_FP8E5M2>(vregY, vregFloatY, mask);385 AscendC::MicroAPI::Cast<VarType, float, Base::CAST_TRAIT_FP32_TO_FP8E5M2>(vregY, vregFloatY, mask);
386- AscendC::MicroAPI::DataCopy<VarType, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(386+ AscendC::MicroAPI::StoreAlign<VarType, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(
387 outLocalAddr + i * VL, vregY, mask);387 outLocalAddr + i * VL, vregY, mask);
388 } else if constexpr (IsSameType<VarType, fp8_e4m3fn_t>::value) {388 } else if constexpr (IsSameType<VarType, fp8_e4m3fn_t>::value) {
389 // fp8_e4m3389 // fp8_e4m3
390 AscendC::MicroAPI::Cast<VarType, float, Base::CAST_TRAIT_FP32_TO_FP8E4M3>(vregY, vregFloatY, mask);390 AscendC::MicroAPI::Cast<VarType, float, Base::CAST_TRAIT_FP32_TO_FP8E4M3>(vregY, vregFloatY, mask);
391- AscendC::MicroAPI::DataCopy<VarType, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(391+ AscendC::MicroAPI::StoreAlign<VarType, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(
392 outLocalAddr + i * VL, vregY, mask);392 outLocalAddr + i * VL, vregY, mask);
393 } else if constexpr (IsSameType<VarType, int8_t>::value) {393 } else if constexpr (IsSameType<VarType, int8_t>::value) {
394 // int8394 // int8
@@ -396,7 +396,7 @@ public:
396 mask);396 mask);
397 AscendC::MicroAPI::Cast<half, int16_t, Base::CAST_TRAIT_INT16_TO_HALF>(vregHalfY, vregInt16Y, mask);397 AscendC::MicroAPI::Cast<half, int16_t, Base::CAST_TRAIT_INT16_TO_HALF>(vregHalfY, vregInt16Y, mask);
398 AscendC::MicroAPI::Cast<int8_t, half, Base::CAST_TRAIT_HALF_TO_INT8>(vregY, vregHalfY, mask);398 AscendC::MicroAPI::Cast<int8_t, half, Base::CAST_TRAIT_HALF_TO_INT8>(vregY, vregHalfY, mask);
399- AscendC::MicroAPI::DataCopy<VarType, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(399+ AscendC::MicroAPI::StoreAlign<VarType, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(
400 outLocalAddr + i * VL, vregY, mask);400 outLocalAddr + i * VL, vregY, mask);
401 }401 }
402 }402 }
@@ -457,4 +457,4 @@ public:
457 }457 }
458};458};
459} // namespace QuantUpdateScatter459} // namespace QuantUpdateScatter
460-#endif // QUANT_UPDATE_SCATTER_LARGE_ELE_LARGE_QUANT_REGBASE_H_460+#endif // QUANT_UPDATE_SCATTER_LARGE_ELE_LARGE_QUANT_REGBASE_H_
Mindex/quant_update_scatter/op_kernel/arch35/quant_update_scatter_large_ele_little_quant_regbase.h+26-26
@@ -204,10 +204,10 @@ public:
204 LocalTensor<UpdatesType> updateLocal = inQueueUpdates_.DeQue<UpdatesType>();204 LocalTensor<UpdatesType> updateLocal = inQueueUpdates_.DeQue<UpdatesType>();
205 LocalTensor<VarType> outLocal = outQueueVar_.AllocTensor<VarType>();205 LocalTensor<VarType> outLocal = outQueueVar_.AllocTensor<VarType>();
206 206 
207- __local_mem__ UpdatesType* updateLocalAddr = (__local_mem__ UpdatesType*)updateLocal.GetPhyAddr();207+ __ubuf__ UpdatesType* updateLocalAddr = (__ubuf__ UpdatesType*)updateLocal.GetPhyAddr();
208- __local_mem__ ScalesType* scaleLocalAddr = (__local_mem__ ScalesType*)scaleLocal.GetPhyAddr();208+ __ubuf__ ScalesType* scaleLocalAddr = (__ubuf__ ScalesType*)scaleLocal.GetPhyAddr();
209- __local_mem__ OffsetsType* offsetLocalAddr = (__local_mem__ OffsetsType*)offsetLocal.GetPhyAddr();209+ __ubuf__ OffsetsType* offsetLocalAddr = (__ubuf__ OffsetsType*)offsetLocal.GetPhyAddr();
210- __local_mem__ VarType* outLocalAddr = (__local_mem__ VarType*)outLocal.GetPhyAddr();210+ __ubuf__ VarType* outLocalAddr = (__ubuf__ VarType*)outLocal.GetPhyAddr();
211 211 
212 uint16_t VL = AscendC::VECTOR_REG_WIDTH / sizeof(float);212 uint16_t VL = AscendC::VECTOR_REG_WIDTH / sizeof(float);
213 uint32_t xLocalOffset = static_cast<uint32_t>(tilingData_.updateOriLastDimAlign);213 uint32_t xLocalOffset = static_cast<uint32_t>(tilingData_.updateOriLastDimAlign);
@@ -245,12 +245,12 @@ public:
245 // ld and cast for update245 // ld and cast for update
246 if constexpr (IsSameType<UpdatesType, half>::value) {246 if constexpr (IsSameType<UpdatesType, half>::value) {
247 // fp16247 // fp16
248- AscendC::MicroAPI::DataCopy<half, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(248+ AscendC::MicroAPI::LoadAlign<half, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(
249 vregX, updateLocalAddr + i * VL + j * xLocalOffset);249 vregX, updateLocalAddr + i * VL + j * xLocalOffset);
250 AscendC::MicroAPI::Cast<float, half, Base::CAST_TRAIT_HALF_TO_FP32>(vregFloatX, vregX, mask);250 AscendC::MicroAPI::Cast<float, half, Base::CAST_TRAIT_HALF_TO_FP32>(vregFloatX, vregX, mask);
251 } else if constexpr (IsSameType<UpdatesType, bfloat16_t>::value) {251 } else if constexpr (IsSameType<UpdatesType, bfloat16_t>::value) {
252 // bf16252 // bf16
253- AscendC::MicroAPI::DataCopy<UpdatesType, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(253+ AscendC::MicroAPI::LoadAlign<UpdatesType, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(
254 vregX, updateLocalAddr + i * VL + j * xLocalOffset);254 vregX, updateLocalAddr + i * VL + j * xLocalOffset);
255 AscendC::MicroAPI::Cast<float, UpdatesType, Base::CAST_TRAIT_BF16_TO_FP32>(vregFloatX, vregX,255 AscendC::MicroAPI::Cast<float, UpdatesType, Base::CAST_TRAIT_BF16_TO_FP32>(vregFloatX, vregX,
256 mask);256 mask);
@@ -259,11 +259,11 @@ public:
259 // ld and cast for scale259 // ld and cast for scale
260 if constexpr (IsSameType<ScalesType, float>::value) {260 if constexpr (IsSameType<ScalesType, float>::value) {
261 // fp32261 // fp32
262- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(262+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(
263 vregFloatS, scaleLocalAddr + i * VL);263 vregFloatS, scaleLocalAddr + i * VL);
264 } else if constexpr (IsSameType<ScalesType, bfloat16_t>::value) {264 } else if constexpr (IsSameType<ScalesType, bfloat16_t>::value) {
265 // bf16265 // bf16
266- AscendC::MicroAPI::DataCopy<ScalesType, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(266+ AscendC::MicroAPI::LoadAlign<ScalesType, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(
267 vregS, scaleLocalAddr + i * VL);267 vregS, scaleLocalAddr + i * VL);
268 AscendC::MicroAPI::Cast<float, ScalesType, Base::CAST_TRAIT_BF16_TO_FP32>(vregFloatS, vregS,268 AscendC::MicroAPI::Cast<float, ScalesType, Base::CAST_TRAIT_BF16_TO_FP32>(vregFloatS, vregS,
269 mask);269 mask);
@@ -272,13 +272,13 @@ public:
272 // ld and cast for offset272 // ld and cast for offset
273 if constexpr (IsSameType<OffsetsType, int32_t>::value) {273 if constexpr (IsSameType<OffsetsType, int32_t>::value) {
274 // int32274 // int32
275- AscendC::MicroAPI::DataCopy<OffsetsType, AscendC::MicroAPI::LoadDist::DIST_NORM>(275+ AscendC::MicroAPI::LoadAlign<OffsetsType, AscendC::MicroAPI::LoadDist::DIST_NORM>(
276 vregO, offsetLocalAddr + i * VL);276 vregO, offsetLocalAddr + i * VL);
277 AscendC::MicroAPI::Cast<float, OffsetsType, Base::CAST_TRAIT_INT32_TO_FP32>(vregFloatO, vregO,277 AscendC::MicroAPI::Cast<float, OffsetsType, Base::CAST_TRAIT_INT32_TO_FP32>(vregFloatO, vregO,
278 mask);278 mask);
279 } else if constexpr (IsSameType<OffsetsType, bfloat16_t>::value) {279 } else if constexpr (IsSameType<OffsetsType, bfloat16_t>::value) {
280 // bf16280 // bf16
281- AscendC::MicroAPI::DataCopy<OffsetsType, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(281+ AscendC::MicroAPI::LoadAlign<OffsetsType, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(
282 vregO, offsetLocalAddr + i * VL);282 vregO, offsetLocalAddr + i * VL);
283 AscendC::MicroAPI::Cast<float, OffsetsType, Base::CAST_TRAIT_BF16_TO_FP32>(vregFloatO, vregO,283 AscendC::MicroAPI::Cast<float, OffsetsType, Base::CAST_TRAIT_BF16_TO_FP32>(vregFloatO, vregO,
284 mask);284 mask);
@@ -301,19 +301,19 @@ public:
301 // hifp8301 // hifp8
302 AscendC::MicroAPI::Cast<VarType, float, Base::CAST_TRAIT_FP32_TO_HIFP8>(vregY, vregFloatY,302 AscendC::MicroAPI::Cast<VarType, float, Base::CAST_TRAIT_FP32_TO_HIFP8>(vregY, vregFloatY,
303 mask);303 mask);
304- AscendC::MicroAPI::DataCopy<VarType, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(304+ AscendC::MicroAPI::StoreAlign<VarType, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(
305 outLocalAddr + i * VL + j * xLocalOffset, vregY, mask);305 outLocalAddr + i * VL + j * xLocalOffset, vregY, mask);
306 } else if constexpr (IsSameType<VarType, fp8_e5m2_t>::value) {306 } else if constexpr (IsSameType<VarType, fp8_e5m2_t>::value) {
307 // fp8_e5m2307 // fp8_e5m2
308 AscendC::MicroAPI::Cast<VarType, float, Base::CAST_TRAIT_FP32_TO_FP8E5M2>(vregY, vregFloatY,308 AscendC::MicroAPI::Cast<VarType, float, Base::CAST_TRAIT_FP32_TO_FP8E5M2>(vregY, vregFloatY,
309 mask);309 mask);
310- AscendC::MicroAPI::DataCopy<VarType, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(310+ AscendC::MicroAPI::StoreAlign<VarType, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(
311 outLocalAddr + i * VL + j * xLocalOffset, vregY, mask);311 outLocalAddr + i * VL + j * xLocalOffset, vregY, mask);
312 } else if constexpr (IsSameType<VarType, fp8_e4m3fn_t>::value) {312 } else if constexpr (IsSameType<VarType, fp8_e4m3fn_t>::value) {
313 // fp8_e4m3313 // fp8_e4m3
314 AscendC::MicroAPI::Cast<VarType, float, Base::CAST_TRAIT_FP32_TO_FP8E4M3>(vregY, vregFloatY,314 AscendC::MicroAPI::Cast<VarType, float, Base::CAST_TRAIT_FP32_TO_FP8E4M3>(vregY, vregFloatY,
315 mask);315 mask);
316- AscendC::MicroAPI::DataCopy<VarType, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(316+ AscendC::MicroAPI::StoreAlign<VarType, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(
317 outLocalAddr + i * VL + j * xLocalOffset, vregY, mask);317 outLocalAddr + i * VL + j * xLocalOffset, vregY, mask);
318 } else if constexpr (IsSameType<VarType, int8_t>::value) {318 } else if constexpr (IsSameType<VarType, int8_t>::value) {
319 // int8319 // int8
@@ -322,7 +322,7 @@ public:
322 AscendC::MicroAPI::Cast<half, int16_t, Base::CAST_TRAIT_INT16_TO_HALF>(vregHalfY, vregInt16Y,322 AscendC::MicroAPI::Cast<half, int16_t, Base::CAST_TRAIT_INT16_TO_HALF>(vregHalfY, vregInt16Y,
323 mask);323 mask);
324 AscendC::MicroAPI::Cast<int8_t, half, Base::CAST_TRAIT_HALF_TO_INT8>(vregY, vregHalfY, mask);324 AscendC::MicroAPI::Cast<int8_t, half, Base::CAST_TRAIT_HALF_TO_INT8>(vregY, vregHalfY, mask);
325- AscendC::MicroAPI::DataCopy<VarType, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(325+ AscendC::MicroAPI::StoreAlign<VarType, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(
326 outLocalAddr + i * VL + j * xLocalOffset, vregY, mask);326 outLocalAddr + i * VL + j * xLocalOffset, vregY, mask);
327 }327 }
328 }328 }
@@ -337,9 +337,9 @@ public:
337 LocalTensor<UpdatesType> updateLocal = inQueueUpdates_.DeQue<UpdatesType>();337 LocalTensor<UpdatesType> updateLocal = inQueueUpdates_.DeQue<UpdatesType>();
338 LocalTensor<VarType> outLocal = outQueueVar_.AllocTensor<VarType>();338 LocalTensor<VarType> outLocal = outQueueVar_.AllocTensor<VarType>();
339 339 
340- __local_mem__ UpdatesType* updateLocalAddr = (__local_mem__ UpdatesType*)updateLocal.GetPhyAddr();340+ __ubuf__ UpdatesType* updateLocalAddr = (__ubuf__ UpdatesType*)updateLocal.GetPhyAddr();
341- __local_mem__ ScalesType* scaleLocalAddr = (__local_mem__ ScalesType*)scaleLocal.GetPhyAddr();341+ __ubuf__ ScalesType* scaleLocalAddr = (__ubuf__ ScalesType*)scaleLocal.GetPhyAddr();
342- __local_mem__ VarType* outLocalAddr = (__local_mem__ VarType*)outLocal.GetPhyAddr();342+ __ubuf__ VarType* outLocalAddr = (__ubuf__ VarType*)outLocal.GetPhyAddr();
343 343 
344 uint16_t VL = AscendC::VECTOR_REG_WIDTH / sizeof(float);344 uint16_t VL = AscendC::VECTOR_REG_WIDTH / sizeof(float);
345 uint32_t xLocalOffset = static_cast<uint32_t>(tilingData_.updateOriLastDimAlign);345 uint32_t xLocalOffset = static_cast<uint32_t>(tilingData_.updateOriLastDimAlign);
@@ -373,12 +373,12 @@ public:
373 // ld and cast for update373 // ld and cast for update
374 if constexpr (IsSameType<UpdatesType, half>::value) {374 if constexpr (IsSameType<UpdatesType, half>::value) {
375 // fp16375 // fp16
376- AscendC::MicroAPI::DataCopy<half, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(376+ AscendC::MicroAPI::LoadAlign<half, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(
377 vregX, updateLocalAddr + i * VL + j * xLocalOffset);377 vregX, updateLocalAddr + i * VL + j * xLocalOffset);
378 AscendC::MicroAPI::Cast<float, half, Base::CAST_TRAIT_HALF_TO_FP32>(vregFloatX, vregX, mask);378 AscendC::MicroAPI::Cast<float, half, Base::CAST_TRAIT_HALF_TO_FP32>(vregFloatX, vregX, mask);
379 } else if constexpr (IsSameType<UpdatesType, bfloat16_t>::value) {379 } else if constexpr (IsSameType<UpdatesType, bfloat16_t>::value) {
380 // bf16380 // bf16
381- AscendC::MicroAPI::DataCopy<UpdatesType, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(381+ AscendC::MicroAPI::LoadAlign<UpdatesType, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(
382 vregX, updateLocalAddr + i * VL + j * xLocalOffset);382 vregX, updateLocalAddr + i * VL + j * xLocalOffset);
383 AscendC::MicroAPI::Cast<float, UpdatesType, Base::CAST_TRAIT_BF16_TO_FP32>(vregFloatX, vregX,383 AscendC::MicroAPI::Cast<float, UpdatesType, Base::CAST_TRAIT_BF16_TO_FP32>(vregFloatX, vregX,
384 mask);384 mask);
@@ -387,11 +387,11 @@ public:
387 // ld and cast for scale387 // ld and cast for scale
388 if constexpr (IsSameType<ScalesType, float>::value) {388 if constexpr (IsSameType<ScalesType, float>::value) {
389 // fp32389 // fp32
390- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(390+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(
391 vregFloatS, scaleLocalAddr + i * VL);391 vregFloatS, scaleLocalAddr + i * VL);
392 } else if constexpr (IsSameType<ScalesType, bfloat16_t>::value) {392 } else if constexpr (IsSameType<ScalesType, bfloat16_t>::value) {
393 // bf16393 // bf16
394- AscendC::MicroAPI::DataCopy<ScalesType, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(394+ AscendC::MicroAPI::LoadAlign<ScalesType, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(
395 vregS, scaleLocalAddr + i * VL);395 vregS, scaleLocalAddr + i * VL);
396 AscendC::MicroAPI::Cast<float, ScalesType, Base::CAST_TRAIT_BF16_TO_FP32>(vregFloatS, vregS,396 AscendC::MicroAPI::Cast<float, ScalesType, Base::CAST_TRAIT_BF16_TO_FP32>(vregFloatS, vregS,
397 mask);397 mask);
@@ -410,19 +410,19 @@ public:
410 // hifp8410 // hifp8
411 AscendC::MicroAPI::Cast<VarType, float, Base::CAST_TRAIT_FP32_TO_HIFP8>(vregY, vregFloatY,411 AscendC::MicroAPI::Cast<VarType, float, Base::CAST_TRAIT_FP32_TO_HIFP8>(vregY, vregFloatY,
412 mask);412 mask);
413- AscendC::MicroAPI::DataCopy<VarType, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(413+ AscendC::MicroAPI::StoreAlign<VarType, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(
414 outLocalAddr + i * VL + j * xLocalOffset, vregY, mask);414 outLocalAddr + i * VL + j * xLocalOffset, vregY, mask);
415 } else if constexpr (IsSameType<VarType, fp8_e5m2_t>::value) {415 } else if constexpr (IsSameType<VarType, fp8_e5m2_t>::value) {
416 // fp8_e5m2416 // fp8_e5m2
417 AscendC::MicroAPI::Cast<VarType, float, Base::CAST_TRAIT_FP32_TO_FP8E5M2>(vregY, vregFloatY,417 AscendC::MicroAPI::Cast<VarType, float, Base::CAST_TRAIT_FP32_TO_FP8E5M2>(vregY, vregFloatY,
418 mask);418 mask);
419- AscendC::MicroAPI::DataCopy<VarType, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(419+ AscendC::MicroAPI::StoreAlign<VarType, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(
420 outLocalAddr + i * VL + j * xLocalOffset, vregY, mask);420 outLocalAddr + i * VL + j * xLocalOffset, vregY, mask);
421 } else if constexpr (IsSameType<VarType, fp8_e4m3fn_t>::value) {421 } else if constexpr (IsSameType<VarType, fp8_e4m3fn_t>::value) {
422 // fp8_e4m3422 // fp8_e4m3
423 AscendC::MicroAPI::Cast<VarType, float, Base::CAST_TRAIT_FP32_TO_FP8E4M3>(vregY, vregFloatY,423 AscendC::MicroAPI::Cast<VarType, float, Base::CAST_TRAIT_FP32_TO_FP8E4M3>(vregY, vregFloatY,
424 mask);424 mask);
425- AscendC::MicroAPI::DataCopy<VarType, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(425+ AscendC::MicroAPI::StoreAlign<VarType, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(
426 outLocalAddr + i * VL + j * xLocalOffset, vregY, mask);426 outLocalAddr + i * VL + j * xLocalOffset, vregY, mask);
427 } else if constexpr (IsSameType<VarType, int8_t>::value) {427 } else if constexpr (IsSameType<VarType, int8_t>::value) {
428 // int8428 // int8
@@ -431,7 +431,7 @@ public:
431 AscendC::MicroAPI::Cast<half, int16_t, Base::CAST_TRAIT_INT16_TO_HALF>(vregHalfY, vregInt16Y,431 AscendC::MicroAPI::Cast<half, int16_t, Base::CAST_TRAIT_INT16_TO_HALF>(vregHalfY, vregInt16Y,
432 mask);432 mask);
433 AscendC::MicroAPI::Cast<int8_t, half, Base::CAST_TRAIT_HALF_TO_INT8>(vregY, vregHalfY, mask);433 AscendC::MicroAPI::Cast<int8_t, half, Base::CAST_TRAIT_HALF_TO_INT8>(vregY, vregHalfY, mask);
434- AscendC::MicroAPI::DataCopy<VarType, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(434+ AscendC::MicroAPI::StoreAlign<VarType, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(
435 outLocalAddr + i * VL + j * xLocalOffset, vregY, mask);435 outLocalAddr + i * VL + j * xLocalOffset, vregY, mask);
436 }436 }
437 }437 }
@@ -493,4 +493,4 @@ public:
493 }493 }
494};494};
495} // namespace QuantUpdateScatter495} // namespace QuantUpdateScatter
496-#endif // QUANT_UPDATE_SCATTER_LARGE_ELE_LITTLE_QUANT_REGBASE_BASE_H_496+#endif // QUANT_UPDATE_SCATTER_LARGE_ELE_LITTLE_QUANT_REGBASE_BASE_H_
Mindex/quant_update_scatter/op_kernel/arch35/quant_update_scatter_regbase.h+28-28
@@ -170,11 +170,11 @@ public:
170 {170 {
171 LocalTensor<VarType> outLocal = outQueueVar_.AllocTensor<VarType>();171 LocalTensor<VarType> outLocal = outQueueVar_.AllocTensor<VarType>();
172 172 
173- __local_mem__ UpdatesType* updateLocalAddr = (__local_mem__ UpdatesType*)updateLocal.GetPhyAddr() +173+ __ubuf__ UpdatesType* updateLocalAddr = (__ubuf__ UpdatesType*)updateLocal.GetPhyAddr() +
174- bsIdx * copyBlockCount_ * tilingData_.updateOriLastDimAlign;174+ bsIdx * copyBlockCount_ * tilingData_.updateOriLastDimAlign;
175- __local_mem__ ScalesType* scaleLocalAddr = (__local_mem__ ScalesType*)scaleLocal.GetPhyAddr();175+ __ubuf__ ScalesType* scaleLocalAddr = (__ubuf__ ScalesType*)scaleLocal.GetPhyAddr();
176- __local_mem__ OffsetsType* offsetLocalAddr = (__local_mem__ OffsetsType*)offsetLocal.GetPhyAddr();176+ __ubuf__ OffsetsType* offsetLocalAddr = (__ubuf__ OffsetsType*)offsetLocal.GetPhyAddr();
177- __local_mem__ VarType* outLocalAddr = (__local_mem__ VarType*)outLocal.GetPhyAddr();177+ __ubuf__ VarType* outLocalAddr = (__ubuf__ VarType*)outLocal.GetPhyAddr();
178 178 
179 uint16_t VL = AscendC::VECTOR_REG_WIDTH / sizeof(float);179 uint16_t VL = AscendC::VECTOR_REG_WIDTH / sizeof(float);
180 uint32_t xLocalOffset = static_cast<uint32_t>(tilingData_.updateOriLastDimAlign);180 uint32_t xLocalOffset = static_cast<uint32_t>(tilingData_.updateOriLastDimAlign);
@@ -208,12 +208,12 @@ public:
208 // ld and cast for update208 // ld and cast for update
209 if constexpr (IsSameType<UpdatesType, half>::value) {209 if constexpr (IsSameType<UpdatesType, half>::value) {
210 // fp16210 // fp16
211- AscendC::MicroAPI::DataCopy<half, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(211+ AscendC::MicroAPI::LoadAlign<half, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(
212 vregX, updateLocalAddr + i * VL + j * xLocalOffset);212 vregX, updateLocalAddr + i * VL + j * xLocalOffset);
213 AscendC::MicroAPI::Cast<float, half, Base::CAST_TRAIT_HALF_TO_FP32>(vregFloatX, vregX, mask);213 AscendC::MicroAPI::Cast<float, half, Base::CAST_TRAIT_HALF_TO_FP32>(vregFloatX, vregX, mask);
214 } else if constexpr (IsSameType<UpdatesType, bfloat16_t>::value) {214 } else if constexpr (IsSameType<UpdatesType, bfloat16_t>::value) {
215 // bf16215 // bf16
216- AscendC::MicroAPI::DataCopy<UpdatesType, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(216+ AscendC::MicroAPI::LoadAlign<UpdatesType, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(
217 vregX, updateLocalAddr + i * VL + j * xLocalOffset);217 vregX, updateLocalAddr + i * VL + j * xLocalOffset);
218 AscendC::MicroAPI::Cast<float, UpdatesType, Base::CAST_TRAIT_BF16_TO_FP32>(vregFloatX, vregX,218 AscendC::MicroAPI::Cast<float, UpdatesType, Base::CAST_TRAIT_BF16_TO_FP32>(vregFloatX, vregX,
219 mask);219 mask);
@@ -222,11 +222,11 @@ public:
222 // ld and cast for scale222 // ld and cast for scale
223 if constexpr (IsSameType<ScalesType, float>::value) {223 if constexpr (IsSameType<ScalesType, float>::value) {
224 // fp32224 // fp32
225- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(225+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(
226 vregFloatS, scaleLocalAddr + i * VL);226 vregFloatS, scaleLocalAddr + i * VL);
227 } else if constexpr (IsSameType<ScalesType, bfloat16_t>::value) {227 } else if constexpr (IsSameType<ScalesType, bfloat16_t>::value) {
228 // bf16228 // bf16
229- AscendC::MicroAPI::DataCopy<ScalesType, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(229+ AscendC::MicroAPI::LoadAlign<ScalesType, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(
230 vregS, scaleLocalAddr + i * VL);230 vregS, scaleLocalAddr + i * VL);
231 AscendC::MicroAPI::Cast<float, ScalesType, Base::CAST_TRAIT_BF16_TO_FP32>(vregFloatS, vregS,231 AscendC::MicroAPI::Cast<float, ScalesType, Base::CAST_TRAIT_BF16_TO_FP32>(vregFloatS, vregS,
232 mask);232 mask);
@@ -235,13 +235,13 @@ public:
235 // ld and cast for offset235 // ld and cast for offset
236 if constexpr (IsSameType<OffsetsType, int32_t>::value) {236 if constexpr (IsSameType<OffsetsType, int32_t>::value) {
237 // int32237 // int32
238- AscendC::MicroAPI::DataCopy<OffsetsType, AscendC::MicroAPI::LoadDist::DIST_NORM>(238+ AscendC::MicroAPI::LoadAlign<OffsetsType, AscendC::MicroAPI::LoadDist::DIST_NORM>(
239 vregO, offsetLocalAddr + i * VL);239 vregO, offsetLocalAddr + i * VL);
240 AscendC::MicroAPI::Cast<float, OffsetsType, Base::CAST_TRAIT_INT32_TO_FP32>(vregFloatO, vregO,240 AscendC::MicroAPI::Cast<float, OffsetsType, Base::CAST_TRAIT_INT32_TO_FP32>(vregFloatO, vregO,
241 mask);241 mask);
242 } else if constexpr (IsSameType<OffsetsType, bfloat16_t>::value) {242 } else if constexpr (IsSameType<OffsetsType, bfloat16_t>::value) {
243 // bf16243 // bf16
244- AscendC::MicroAPI::DataCopy<OffsetsType, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(244+ AscendC::MicroAPI::LoadAlign<OffsetsType, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(
245 vregO, offsetLocalAddr + i * VL);245 vregO, offsetLocalAddr + i * VL);
246 AscendC::MicroAPI::Cast<float, OffsetsType, Base::CAST_TRAIT_BF16_TO_FP32>(vregFloatO, vregO,246 AscendC::MicroAPI::Cast<float, OffsetsType, Base::CAST_TRAIT_BF16_TO_FP32>(vregFloatO, vregO,
247 mask);247 mask);
@@ -262,19 +262,19 @@ public:
262 // hifp8262 // hifp8
263 AscendC::MicroAPI::Cast<VarType, float, Base::CAST_TRAIT_FP32_TO_HIFP8>(vregY, vregFloatY,263 AscendC::MicroAPI::Cast<VarType, float, Base::CAST_TRAIT_FP32_TO_HIFP8>(vregY, vregFloatY,
264 mask);264 mask);
265- AscendC::MicroAPI::DataCopy<VarType, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(265+ AscendC::MicroAPI::StoreAlign<VarType, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(
266 outLocalAddr + i * VL + j * xLocalOffset, vregY, mask);266 outLocalAddr + i * VL + j * xLocalOffset, vregY, mask);
267 } else if constexpr (IsSameType<VarType, fp8_e5m2_t>::value) {267 } else if constexpr (IsSameType<VarType, fp8_e5m2_t>::value) {
268 // fp8_e5m2268 // fp8_e5m2
269 AscendC::MicroAPI::Cast<VarType, float, Base::CAST_TRAIT_FP32_TO_FP8E5M2>(vregY, vregFloatY,269 AscendC::MicroAPI::Cast<VarType, float, Base::CAST_TRAIT_FP32_TO_FP8E5M2>(vregY, vregFloatY,
270 mask);270 mask);
271- AscendC::MicroAPI::DataCopy<VarType, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(271+ AscendC::MicroAPI::StoreAlign<VarType, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(
272 outLocalAddr + i * VL + j * xLocalOffset, vregY, mask);272 outLocalAddr + i * VL + j * xLocalOffset, vregY, mask);
273 } else if constexpr (IsSameType<VarType, fp8_e4m3fn_t>::value) {273 } else if constexpr (IsSameType<VarType, fp8_e4m3fn_t>::value) {
274 // fp8_e4m3274 // fp8_e4m3
275 AscendC::MicroAPI::Cast<VarType, float, Base::CAST_TRAIT_FP32_TO_FP8E4M3>(vregY, vregFloatY,275 AscendC::MicroAPI::Cast<VarType, float, Base::CAST_TRAIT_FP32_TO_FP8E4M3>(vregY, vregFloatY,
276 mask);276 mask);
277- AscendC::MicroAPI::DataCopy<VarType, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(277+ AscendC::MicroAPI::StoreAlign<VarType, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(
278 outLocalAddr + i * VL + j * xLocalOffset, vregY, mask);278 outLocalAddr + i * VL + j * xLocalOffset, vregY, mask);
279 } else if constexpr (IsSameType<VarType, int8_t>::value) {279 } else if constexpr (IsSameType<VarType, int8_t>::value) {
280 // int8280 // int8
@@ -283,7 +283,7 @@ public:
283 AscendC::MicroAPI::Cast<half, int16_t, Base::CAST_TRAIT_INT16_TO_HALF>(vregHalfY, vregInt16Y,283 AscendC::MicroAPI::Cast<half, int16_t, Base::CAST_TRAIT_INT16_TO_HALF>(vregHalfY, vregInt16Y,
284 mask);284 mask);
285 AscendC::MicroAPI::Cast<int8_t, half, Base::CAST_TRAIT_HALF_TO_INT8>(vregY, vregHalfY, mask);285 AscendC::MicroAPI::Cast<int8_t, half, Base::CAST_TRAIT_HALF_TO_INT8>(vregY, vregHalfY, mask);
286- AscendC::MicroAPI::DataCopy<VarType, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(286+ AscendC::MicroAPI::StoreAlign<VarType, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(
287 outLocalAddr + i * VL + j * xLocalOffset, vregY, mask);287 outLocalAddr + i * VL + j * xLocalOffset, vregY, mask);
288 }288 }
289 }289 }
@@ -298,10 +298,10 @@ public:
298 {298 {
299 LocalTensor<VarType> outLocal = outQueueVar_.AllocTensor<VarType>();299 LocalTensor<VarType> outLocal = outQueueVar_.AllocTensor<VarType>();
300 300 
301- __local_mem__ UpdatesType* updateLocalAddr = (__local_mem__ UpdatesType*)updateLocal.GetPhyAddr() +301+ __ubuf__ UpdatesType* updateLocalAddr = (__ubuf__ UpdatesType*)updateLocal.GetPhyAddr() +
302- bsIdx * copyBlockCount_ * tilingData_.updateOriLastDimAlign;302+ bsIdx * copyBlockCount_ * tilingData_.updateOriLastDimAlign;
303- __local_mem__ ScalesType* scaleLocalAddr = (__local_mem__ ScalesType*)scaleLocal.GetPhyAddr();303+ __ubuf__ ScalesType* scaleLocalAddr = (__ubuf__ ScalesType*)scaleLocal.GetPhyAddr();
304- __local_mem__ VarType* outLocalAddr = (__local_mem__ VarType*)outLocal.GetPhyAddr();304+ __ubuf__ VarType* outLocalAddr = (__ubuf__ VarType*)outLocal.GetPhyAddr();
305 305 
306 uint16_t VL = AscendC::VECTOR_REG_WIDTH / sizeof(float);306 uint16_t VL = AscendC::VECTOR_REG_WIDTH / sizeof(float);
307 uint32_t xLocalOffset = static_cast<uint32_t>(tilingData_.updateOriLastDimAlign);307 uint32_t xLocalOffset = static_cast<uint32_t>(tilingData_.updateOriLastDimAlign);
@@ -331,12 +331,12 @@ public:
331 // ld and cast for update331 // ld and cast for update
332 if constexpr (IsSameType<UpdatesType, half>::value) {332 if constexpr (IsSameType<UpdatesType, half>::value) {
333 // fp16333 // fp16
334- AscendC::MicroAPI::DataCopy<half, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(334+ AscendC::MicroAPI::LoadAlign<half, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(
335 vregX, updateLocalAddr + i * VL + j * xLocalOffset);335 vregX, updateLocalAddr + i * VL + j * xLocalOffset);
336 AscendC::MicroAPI::Cast<float, half, Base::CAST_TRAIT_HALF_TO_FP32>(vregFloatX, vregX, mask);336 AscendC::MicroAPI::Cast<float, half, Base::CAST_TRAIT_HALF_TO_FP32>(vregFloatX, vregX, mask);
337 } else if constexpr (IsSameType<UpdatesType, bfloat16_t>::value) {337 } else if constexpr (IsSameType<UpdatesType, bfloat16_t>::value) {
338 // bf16338 // bf16
339- AscendC::MicroAPI::DataCopy<UpdatesType, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(339+ AscendC::MicroAPI::LoadAlign<UpdatesType, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(
340 vregX, updateLocalAddr + i * VL + j * xLocalOffset);340 vregX, updateLocalAddr + i * VL + j * xLocalOffset);
341 AscendC::MicroAPI::Cast<float, UpdatesType, Base::CAST_TRAIT_BF16_TO_FP32>(vregFloatX, vregX,341 AscendC::MicroAPI::Cast<float, UpdatesType, Base::CAST_TRAIT_BF16_TO_FP32>(vregFloatX, vregX,
342 mask);342 mask);
@@ -345,11 +345,11 @@ public:
345 // ld and cast for scale345 // ld and cast for scale
346 if constexpr (IsSameType<ScalesType, float>::value) {346 if constexpr (IsSameType<ScalesType, float>::value) {
347 // fp32347 // fp32
348- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(348+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(
349 vregFloatS, scaleLocalAddr + i * VL);349 vregFloatS, scaleLocalAddr + i * VL);
350 } else if constexpr (IsSameType<ScalesType, bfloat16_t>::value) {350 } else if constexpr (IsSameType<ScalesType, bfloat16_t>::value) {
351 // bf16351 // bf16
352- AscendC::MicroAPI::DataCopy<ScalesType, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(352+ AscendC::MicroAPI::LoadAlign<ScalesType, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(
353 vregS, scaleLocalAddr + i * VL);353 vregS, scaleLocalAddr + i * VL);
354 AscendC::MicroAPI::Cast<float, ScalesType, Base::CAST_TRAIT_BF16_TO_FP32>(vregFloatS, vregS,354 AscendC::MicroAPI::Cast<float, ScalesType, Base::CAST_TRAIT_BF16_TO_FP32>(vregFloatS, vregS,
355 mask);355 mask);
@@ -367,19 +367,19 @@ public:
367 // hifp8367 // hifp8
368 AscendC::MicroAPI::Cast<VarType, float, Base::CAST_TRAIT_FP32_TO_HIFP8>(vregY, vregFloatY,368 AscendC::MicroAPI::Cast<VarType, float, Base::CAST_TRAIT_FP32_TO_HIFP8>(vregY, vregFloatY,
369 mask);369 mask);
370- AscendC::MicroAPI::DataCopy<VarType, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(370+ AscendC::MicroAPI::StoreAlign<VarType, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(
371 outLocalAddr + i * VL + j * xLocalOffset, vregY, mask);371 outLocalAddr + i * VL + j * xLocalOffset, vregY, mask);
372 } else if constexpr (IsSameType<VarType, fp8_e5m2_t>::value) {372 } else if constexpr (IsSameType<VarType, fp8_e5m2_t>::value) {
373 // fp8_e5m2373 // fp8_e5m2
374 AscendC::MicroAPI::Cast<VarType, float, Base::CAST_TRAIT_FP32_TO_FP8E5M2>(vregY, vregFloatY,374 AscendC::MicroAPI::Cast<VarType, float, Base::CAST_TRAIT_FP32_TO_FP8E5M2>(vregY, vregFloatY,
375 mask);375 mask);
376- AscendC::MicroAPI::DataCopy<VarType, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(376+ AscendC::MicroAPI::StoreAlign<VarType, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(
377 outLocalAddr + i * VL + j * xLocalOffset, vregY, mask);377 outLocalAddr + i * VL + j * xLocalOffset, vregY, mask);
378 } else if constexpr (IsSameType<VarType, fp8_e4m3fn_t>::value) {378 } else if constexpr (IsSameType<VarType, fp8_e4m3fn_t>::value) {
379 // fp8_e4m3379 // fp8_e4m3
380 AscendC::MicroAPI::Cast<VarType, float, Base::CAST_TRAIT_FP32_TO_FP8E4M3>(vregY, vregFloatY,380 AscendC::MicroAPI::Cast<VarType, float, Base::CAST_TRAIT_FP32_TO_FP8E4M3>(vregY, vregFloatY,
381 mask);381 mask);
382- AscendC::MicroAPI::DataCopy<VarType, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(382+ AscendC::MicroAPI::StoreAlign<VarType, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(
383 outLocalAddr + i * VL + j * xLocalOffset, vregY, mask);383 outLocalAddr + i * VL + j * xLocalOffset, vregY, mask);
384 } else if constexpr (IsSameType<VarType, int8_t>::value) {384 } else if constexpr (IsSameType<VarType, int8_t>::value) {
385 // int8385 // int8
@@ -388,7 +388,7 @@ public:
388 AscendC::MicroAPI::Cast<half, int16_t, Base::CAST_TRAIT_INT16_TO_HALF>(vregHalfY, vregInt16Y,388 AscendC::MicroAPI::Cast<half, int16_t, Base::CAST_TRAIT_INT16_TO_HALF>(vregHalfY, vregInt16Y,
389 mask);389 mask);
390 AscendC::MicroAPI::Cast<int8_t, half, Base::CAST_TRAIT_HALF_TO_INT8>(vregY, vregHalfY, mask);390 AscendC::MicroAPI::Cast<int8_t, half, Base::CAST_TRAIT_HALF_TO_INT8>(vregY, vregHalfY, mask);
391- AscendC::MicroAPI::DataCopy<VarType, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(391+ AscendC::MicroAPI::StoreAlign<VarType, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(
392 outLocalAddr + i * VL + j * xLocalOffset, vregY, mask);392 outLocalAddr + i * VL + j * xLocalOffset, vregY, mask);
393 }393 }
394 }394 }
@@ -448,4 +448,4 @@ public:
448 }448 }
449};449};
450} // namespace QuantUpdateScatter450} // namespace QuantUpdateScatter
451-#endif // QUANT_UPDATE_SCATTER_REGBASE_451+#endif // QUANT_UPDATE_SCATTER_REGBASE_
Mindex/sparse_slice/op_kernel/arch35/sparse_slice_dimension_base.h+76-77
@@ -36,7 +36,7 @@ constexpr int64_t START_TILING_DATA_OFFSET = 64 + 24 * sizeof(int64_t);
36constexpr int64_t END_TILING_DATA_OFFSET = 64 + 48 * sizeof(int64_t);36constexpr int64_t END_TILING_DATA_OFFSET = 64 + 48 * sizeof(int64_t);
37constexpr int64_t TILING_DATA_ARRAY_SIZE = 24 * sizeof(int64_t);37constexpr int64_t TILING_DATA_ARRAY_SIZE = 24 * sizeof(int64_t);
38 38 
39-static constexpr MultiCopyConfig config = {false};39+static constexpr NdDmaConfig config = {false};
40 40 
41template <typename T>41template <typename T>
42class SparseSliceDimension : public SparseSliceBase {42class SparseSliceDimension : public SparseSliceBase {
@@ -380,12 +380,12 @@ __aicore__ inline void SparseSliceDimension<T>::CopyInIndices(int64_t offset, in
380 if constexpr (isPad) {380 if constexpr (isPad) {
381 rightPaddingNum = static_cast<uint8_t>(ops::CeilDiv<int64_t>(count, INT64_ALIGN_NUM) * INT64_ALIGN_NUM - count);381 rightPaddingNum = static_cast<uint8_t>(ops::CeilDiv<int64_t>(count, INT64_ALIGN_NUM) * INT64_ALIGN_NUM - count);
382 }382 }
383- MultiCopyLoopInfo<INDICES_TENSOR_RANK> loopInfo{383+ NdDmaLoopInfo<INDICES_TENSOR_RANK> loopInfo{
384 {(uint64_t)1, (uint64_t)rankNumbers_}, {(uint32_t)count + rightPaddingNum, (uint32_t)1},384 {(uint64_t)1, (uint64_t)rankNumbers_}, {(uint32_t)count + rightPaddingNum, (uint32_t)1},
385 {(uint32_t)rankNumbers_, (uint32_t)count}, {(uint8_t)0, (uint8_t)0},385 {(uint32_t)rankNumbers_, (uint32_t)count}, {(uint8_t)0, (uint8_t)0},
386 {(uint8_t)rightPaddingNum, (uint8_t)0},386 {(uint8_t)rightPaddingNum, (uint8_t)0},
387 };387 };
388- MultiCopyParams<int64_t, INDICES_TENSOR_RANK> params{loopInfo, NUM_ZERO};388+ NdDmaParams<int64_t, INDICES_TENSOR_RANK> params{loopInfo, NUM_ZERO};
389 DataCopy<int64_t, INDICES_TENSOR_RANK, config>(indicesUb, indicesGm_[offset * rankNumbers_], params);389 DataCopy<int64_t, INDICES_TENSOR_RANK, config>(indicesUb, indicesGm_[offset * rankNumbers_], params);
390 indicesQueue_.EnQue(indicesUb);390 indicesQueue_.EnQue(indicesUb);
391}391}
@@ -462,13 +462,13 @@ __aicore__ inline void SparseSliceDimension<T>::CompareMultiDim(const LocalTenso
462 462 
463 for (uint16_t dim = 0; dim < rankNum; dim++) {463 for (uint16_t dim = 0; dim < rankNum; dim++) {
464 uint32_t sreg0 = count;464 uint32_t sreg0 = count;
465- MicroAPI::DataCopy<uint32_t, MicroAPI::LoadDist::DIST_BRC_B32>(465+ MicroAPI::LoadAlign<uint32_t, MicroAPI::LoadDist::DIST_BRC_B32>(
466 vregLowerBoundLowerHalf, (__ubuf__ uint32_t*)(sliceStartAddr) + dim * DIGIT_TWO);466 vregLowerBoundLowerHalf, (__ubuf__ uint32_t*)(sliceStartAddr) + dim * DIGIT_TWO);
467- MicroAPI::DataCopy<int32_t, MicroAPI::LoadDist::DIST_BRC_B32>(467+ MicroAPI::LoadAlign<int32_t, MicroAPI::LoadDist::DIST_BRC_B32>(
468 vregLowerBoundHigherHalf, (__ubuf__ int32_t*)(sliceStartAddr) + dim * DIGIT_TWO + 1);468 vregLowerBoundHigherHalf, (__ubuf__ int32_t*)(sliceStartAddr) + dim * DIGIT_TWO + 1);
469- MicroAPI::DataCopy<uint32_t, MicroAPI::LoadDist::DIST_BRC_B32>(469+ MicroAPI::LoadAlign<uint32_t, MicroAPI::LoadDist::DIST_BRC_B32>(
470 vregUpperBoundLowerHalf, (__ubuf__ uint32_t*)(sliceEndAddr) + dim * DIGIT_TWO);470 vregUpperBoundLowerHalf, (__ubuf__ uint32_t*)(sliceEndAddr) + dim * DIGIT_TWO);
471- MicroAPI::DataCopy<int32_t, MicroAPI::LoadDist::DIST_BRC_B32>(471+ MicroAPI::LoadAlign<int32_t, MicroAPI::LoadDist::DIST_BRC_B32>(
472 vregUpperBoundHigherHalf, (__ubuf__ int32_t*)(sliceEndAddr) + dim * DIGIT_TWO + 1);472 vregUpperBoundHigherHalf, (__ubuf__ int32_t*)(sliceEndAddr) + dim * DIGIT_TWO + 1);
473 473 
474 AscendC::MicroAPI::LocalMemBar<AscendC::MicroAPI::MemType::VEC_STORE,474 AscendC::MicroAPI::LocalMemBar<AscendC::MicroAPI::MemType::VEC_STORE,
@@ -476,13 +476,13 @@ __aicore__ inline void SparseSliceDimension<T>::CompareMultiDim(const LocalTenso
476 for (uint16_t loopIndex = 0; loopIndex < loopNum; loopIndex++) {476 for (uint16_t loopIndex = 0; loopIndex < loopNum; loopIndex++) {
477 // use b32 updatemask since there's twice copyin of int64 from ub to register477 // use b32 updatemask since there's twice copyin of int64 from ub to register
478 pregIndicesB32 = MicroAPI::UpdateMask<int32_t>(sreg0);478 pregIndicesB32 = MicroAPI::UpdateMask<int32_t>(sreg0);
479- MicroAPI::DataCopy(vregInputOne,479+ MicroAPI::LoadAlign(vregInputOne,
480- (__ubuf__ int64_t*)(indicesUbAddr + dim * offsetPerLoop + loopIndex * vl));480+ (__ubuf__ int64_t*)(indicesUbAddr + dim * offsetPerLoop + loopIndex * vl));
481- MicroAPI::DataCopy(vregInputTwo,481+ MicroAPI::LoadAlign(vregInputTwo,
482- (__ubuf__ int64_t*)(indicesUbAddr + dim * offsetPerLoop + loopIndex * vl + vlSize));482+ (__ubuf__ int64_t*)(indicesUbAddr + dim * offsetPerLoop + loopIndex * vl + vlSize));
483 483 
484 // copyin b8 mask and unpack it to b32 mask484 // copyin b8 mask and unpack it to b32 mask
485- MicroAPI::DataCopy<int8_t, MicroAPI::LoadDist::DIST_UNPACK4_B8>(485+ MicroAPI::LoadAlign<int8_t, MicroAPI::LoadDist::DIST_UNPACK4_B8>(
486 vregInMask, (__ubuf__ int8_t*)(maskUbAddr + loopIndex * vl));486 vregInMask, (__ubuf__ int8_t*)(maskUbAddr + loopIndex * vl));
487 487 
488 // split lower part and higher part to compare488 // split lower part and higher part to compare
@@ -497,7 +497,7 @@ __aicore__ inline void SparseSliceDimension<T>::CompareMultiDim(const LocalTenso
497 pregB32All);497 pregB32All);
498 MicroAPI::Compare<int32_t, CMPMODE::EQ>(flagHigherEqual, vregInputHigherHalf, vregLowerBoundHigherHalf,498 MicroAPI::Compare<int32_t, CMPMODE::EQ>(flagHigherEqual, vregInputHigherHalf, vregLowerBoundHigherHalf,
499 pregB32All);499 pregB32All);
500- MicroAPI::MaskSel(flagResultB32, flagLowerCmp, flagHigherCmp, flagHigherEqual);500+ MicroAPI::Select(flagResultB32, flagLowerCmp, flagHigherCmp, flagHigherEqual);
501 // upper bound compare501 // upper bound compare
502 MicroAPI::Compare<uint32_t, CMPMODE::LT>(flagLowerCmp,502 MicroAPI::Compare<uint32_t, CMPMODE::LT>(flagLowerCmp,
503 (MicroAPI::RegTensor<uint32_t>&)vregInputLowerHalf,503 (MicroAPI::RegTensor<uint32_t>&)vregInputLowerHalf,
@@ -506,29 +506,30 @@ __aicore__ inline void SparseSliceDimension<T>::CompareMultiDim(const LocalTenso
506 pregB32All);506 pregB32All);
507 MicroAPI::Compare<int32_t, CMPMODE::EQ>(flagHigherEqual, vregInputHigherHalf, vregUpperBoundHigherHalf,507 MicroAPI::Compare<int32_t, CMPMODE::EQ>(flagHigherEqual, vregInputHigherHalf, vregUpperBoundHigherHalf,
508 pregB32All);508 pregB32All);
509- MicroAPI::MaskSel(flagResultB32Two, flagLowerCmp, flagHigherCmp, flagHigherEqual);509+ MicroAPI::Select(flagResultB32Two, flagLowerCmp, flagHigherCmp, flagHigherEqual);
510 510 
511- MicroAPI::MaskAnd(flagResultB32, flagResultB32, flagResultB32Two, pregIndicesB32);511+ MicroAPI::And(flagResultB32, flagResultB32, flagResultB32Two, pregIndicesB32);
512 512 
513 MicroAPI::Select((MicroAPI::RegTensor<int32_t>&)vregOutMask, (MicroAPI::RegTensor<int32_t>&)vregInMask,513 MicroAPI::Select((MicroAPI::RegTensor<int32_t>&)vregOutMask, (MicroAPI::RegTensor<int32_t>&)vregInMask,
514 vregZeros, flagResultB32);514 vregZeros, flagResultB32);
515- MicroAPI::DataCopy<int8_t, MicroAPI::StoreDist::DIST_PACK4_B32>(515+ MicroAPI::StoreAlign<int8_t, MicroAPI::StoreDist::DIST_PACK4_B32>(
516 (__ubuf__ int8_t*)(maskUbAddr + loopIndex * vl), vregOutMask, pregIndicesB32);516 (__ubuf__ int8_t*)(maskUbAddr + loopIndex * vl), vregOutMask, pregIndicesB32);
517 }517 }
518 }518 }
519 // reduce part519 // reduce part
520 AscendC::MicroAPI::LocalMemBar<AscendC::MicroAPI::MemType::VEC_STORE, AscendC::MicroAPI::MemType::VEC_LOAD>();520 AscendC::MicroAPI::LocalMemBar<AscendC::MicroAPI::MemType::VEC_STORE, AscendC::MicroAPI::MemType::VEC_LOAD>();
521 uint32_t sreg0 = count;521 uint32_t sreg0 = count;
522- MicroAPI::DataCopy<int32_t, MicroAPI::LoadDist::DIST_NORM>(vregCounterIn, (__ubuf__ int32_t*)(counterUbAddr));522+ MicroAPI::LoadAlign<int32_t, MicroAPI::LoadDist::DIST_NORM>(vregCounterIn, (__ubuf__ int32_t*)(counterUbAddr));
523 for (uint16_t loopIndex = 0; loopIndex < loopNum; loopIndex++) {523 for (uint16_t loopIndex = 0; loopIndex < loopNum; loopIndex++) {
524 pregIndicesB32 = MicroAPI::UpdateMask<int32_t>(sreg0);524 pregIndicesB32 = MicroAPI::UpdateMask<int32_t>(sreg0);
525- MicroAPI::DataCopy<int8_t, MicroAPI::LoadDist::DIST_UNPACK4_B8>(525+ MicroAPI::LoadAlign<int8_t, MicroAPI::LoadDist::DIST_UNPACK4_B8>(
526 vregInMask, (__ubuf__ int8_t*)(maskUbAddr + loopIndex * vl));526 vregInMask, (__ubuf__ int8_t*)(maskUbAddr + loopIndex * vl));
527- MicroAPI::ReduceSum<int32_t>(reduceResult, (MicroAPI::RegTensor<int32_t>&)vregInMask, pregIndicesB32);527+ MicroAPI::Reduce<ReduceType::SUM, int32_t>(reduceResult, (MicroAPI::RegTensor<int32_t>&)vregInMask,
528+ pregIndicesB32);
528 MicroAPI::Add(vregCounterIn, vregCounterIn, reduceResult, pregB32First);529 MicroAPI::Add(vregCounterIn, vregCounterIn, reduceResult, pregB32First);
529 }530 }
530- MicroAPI::DataCopy<int32_t, MicroAPI::StoreDist::DIST_NORM>((__ubuf__ int32_t*)(counterUbAddr), vregCounterIn,531+ MicroAPI::StoreAlign<int32_t, MicroAPI::StoreDist::DIST_NORM>((__ubuf__ int32_t*)(counterUbAddr), vregCounterIn,
531- pregB32First);532+ pregB32First);
532 } // vf ends533 } // vf ends
533}534}
534 535 
@@ -687,7 +688,7 @@ __aicore__ inline void SparseSliceDimension<T>::GatherIndicesMultiDim(__ubuf__ i
687 MicroAPI::MaskReg pregGatheredTwo;688 MicroAPI::MaskReg pregGatheredTwo;
688 689 
689 uint16_t rankNum = rn;690 uint16_t rankNum = rn;
690- MicroAPI::UnalignReg ureg0;691+ MicroAPI::UnalignRegForStore ureg0;
691 692 
692 MicroAPI::Duplicate(vregOnes, 1);693 MicroAPI::Duplicate(vregOnes, 1);
693 694 
@@ -695,22 +696,22 @@ __aicore__ inline void SparseSliceDimension<T>::GatherIndicesMultiDim(__ubuf__ i
695 MicroAPI::ClearSpr<SpecialPurposeReg::AR>();696 MicroAPI::ClearSpr<SpecialPurposeReg::AR>();
696 uint32_t sreg0 = count;697 uint32_t sreg0 = count;
697 __ubuf__ int64_t* yIndicesCurDimAddr = (__ubuf__ int64_t*)(yIndicesUbAddr) + dim * count;698 __ubuf__ int64_t* yIndicesCurDimAddr = (__ubuf__ int64_t*)(yIndicesUbAddr) + dim * count;
698- MicroAPI::DataCopy<uint32_t, MicroAPI::LoadDist::DIST_BRC_B32>(699+ MicroAPI::LoadAlign<uint32_t, MicroAPI::LoadDist::DIST_BRC_B32>(
699 vregLowerBoundLowerHalf, (__ubuf__ uint32_t*)(sliceStartAddr) + dim * DIGIT_TWO);700 vregLowerBoundLowerHalf, (__ubuf__ uint32_t*)(sliceStartAddr) + dim * DIGIT_TWO);
700- MicroAPI::DataCopy<int32_t, MicroAPI::LoadDist::DIST_BRC_B32>(701+ MicroAPI::LoadAlign<int32_t, MicroAPI::LoadDist::DIST_BRC_B32>(
701 vregLowerBoundHigherHalf, (__ubuf__ int32_t*)(sliceStartAddr) + dim * DIGIT_TWO + 1);702 vregLowerBoundHigherHalf, (__ubuf__ int32_t*)(sliceStartAddr) + dim * DIGIT_TWO + 1);
702 carryOut = MicroAPI::CreateMask<uint8_t>();703 carryOut = MicroAPI::CreateMask<uint8_t>();
703 for (uint16_t loopIndex = 0; loopIndex < loopNum; loopIndex++) {704 for (uint16_t loopIndex = 0; loopIndex < loopNum; loopIndex++) {
704 pregIndices = MicroAPI::UpdateMask<int32_t>(sreg0);705 pregIndices = MicroAPI::UpdateMask<int32_t>(sreg0);
705- MicroAPI::DataCopy(vregInputOne, (__ubuf__ int64_t*)(indicesUbAddr + dim * count + loopIndex * vl));706+ MicroAPI::LoadAlign(vregInputOne, (__ubuf__ int64_t*)(indicesUbAddr + dim * count + loopIndex * vl));
706- MicroAPI::DataCopy(vregInputTwo,707+ MicroAPI::LoadAlign(vregInputTwo,
707- (__ubuf__ int64_t*)(indicesUbAddr + dim * count + loopIndex * vl + vlSize));708+ (__ubuf__ int64_t*)(indicesUbAddr + dim * count + loopIndex * vl + vlSize));
708- MicroAPI::DataCopy<int8_t, MicroAPI::LoadDist::DIST_UNPACK4_B8>(709+ MicroAPI::LoadAlign<int8_t, MicroAPI::LoadDist::DIST_UNPACK4_B8>(
709 vregInMask, (__ubuf__ int8_t*)(maskUbAddr + loopIndex * vl));710 vregInMask, (__ubuf__ int8_t*)(maskUbAddr + loopIndex * vl));
710 MicroAPI::Compare<int32_t, CMPMODE::EQ>(pregSelect, (MicroAPI::RegTensor<int32_t>&)vregInMask, vregOnes,711 MicroAPI::Compare<int32_t, CMPMODE::EQ>(pregSelect, (MicroAPI::RegTensor<int32_t>&)vregInMask, vregOnes,
711 pregIndices);712 pregIndices);
712 713 
713- MicroAPI::GatherMask<int32_t, MicroAPI::GatherMaskMode::NO_STORE_REG>(714+ MicroAPI::Squeeze<int32_t, MicroAPI::GatherMaskMode::NO_STORE_REG>(
714 vregInMaskGathered, (MicroAPI::RegTensor<int32_t>&)vregInMask, pregSelect);715 vregInMaskGathered, (MicroAPI::RegTensor<int32_t>&)vregInMask, pregSelect);
715 MicroAPI::Compare<int32_t, CMPMODE::EQ>(pregGathered, vregInMaskGathered, vregOnes, pregIndices);716 MicroAPI::Compare<int32_t, CMPMODE::EQ>(pregGathered, vregInMaskGathered, vregOnes, pregIndices);
716 717 
@@ -720,16 +721,15 @@ __aicore__ inline void SparseSliceDimension<T>::GatherIndicesMultiDim(__ubuf__ i
720 (MicroAPI::RegTensor<int32_t>&)vregInputTwo);721 (MicroAPI::RegTensor<int32_t>&)vregInputTwo);
721 722 
722 // do int64 sub before gathermask723 // do int64 sub before gathermask
723- MicroAPI::SubCarryOut(carryOut, vregInputLowerHalf, vregInputLowerHalf, vregLowerBoundLowerHalf,724+ MicroAPI::Sub(carryOut, vregInputLowerHalf, vregInputLowerHalf, vregLowerBoundLowerHalf, pregSelect);
724- pregSelect);725+ MicroAPI::SubC(carryOut, vregInputHigherHalf, vregInputHigherHalf, vregLowerBoundHigherHalf, carryOut,
725- MicroAPI::SubCarryOuts(carryOut, vregInputHigherHalf, vregInputHigherHalf, vregLowerBoundHigherHalf,726+ pregSelect);
726- carryOut, pregSelect);
727 727 
728 // gather mask728 // gather mask
729- MicroAPI::GatherMask<int32_t, MicroAPI::GatherMaskMode::NO_STORE_REG>(729+ MicroAPI::Squeeze<int32_t, MicroAPI::GatherMaskMode::NO_STORE_REG>(
730 vregGatherLowerHalf, (MicroAPI::RegTensor<int32_t>&)vregInputLowerHalf, pregSelect);730 vregGatherLowerHalf, (MicroAPI::RegTensor<int32_t>&)vregInputLowerHalf, pregSelect);
731- MicroAPI::GatherMask<int32_t, MicroAPI::GatherMaskMode::NO_STORE_REG>(vregGatherHigherHalf,731+ MicroAPI::Squeeze<int32_t, MicroAPI::GatherMaskMode::NO_STORE_REG>(vregGatherHigherHalf,
732- vregInputHigherHalf, pregSelect);732+ vregInputHigherHalf, pregSelect);
733 733 
734 // vreg interleave and mask interleave734 // vreg interleave and mask interleave
735 MicroAPI::Interleave<int32_t>(vregGatherResult0, vregGatherResult1, vregGatherLowerHalf,735 MicroAPI::Interleave<int32_t>(vregGatherResult0, vregGatherResult1, vregGatherLowerHalf,
@@ -737,17 +737,17 @@ __aicore__ inline void SparseSliceDimension<T>::GatherIndicesMultiDim(__ubuf__ i
737 MicroAPI::MaskInterleave<int32_t>(pregSelectFinal0, pregSelectFinal1, pregGathered, pregGathered);737 MicroAPI::MaskInterleave<int32_t>(pregSelectFinal0, pregSelectFinal1, pregGathered, pregGathered);
738 738 
739 // additional gathermask to save real copyout data len739 // additional gathermask to save real copyout data len
740- MicroAPI::GatherMask<int32_t, MicroAPI::GatherMaskMode::STORE_REG>(vregGatherOutput0, vregGatherResult0,740+ MicroAPI::Squeeze<int32_t, MicroAPI::GatherMaskMode::STORE_REG>(vregGatherOutput0, vregGatherResult0,
741- pregSelectFinal0);741+ pregSelectFinal0);
742- MicroAPI::DataCopyUnAlign<int64_t, MicroAPI::PostLiteral::POST_MODE_UPDATE>(742+ MicroAPI::StoreUnAlign<int64_t, MicroAPI::PostLiteral::POST_MODE_UPDATE>(
743 (__ubuf__ int64_t*)(yIndicesCurDimAddr), (MicroAPI::RegTensor<int64_t>&)vregGatherResult0, ureg0);743 (__ubuf__ int64_t*)(yIndicesCurDimAddr), (MicroAPI::RegTensor<int64_t>&)vregGatherResult0, ureg0);
744 744 
745- MicroAPI::GatherMask<int32_t, MicroAPI::GatherMaskMode::STORE_REG>(vregGatherOutput1, vregGatherResult1,745+ MicroAPI::Squeeze<int32_t, MicroAPI::GatherMaskMode::STORE_REG>(vregGatherOutput1, vregGatherResult1,
746- pregSelectFinal1);746+ pregSelectFinal1);
747- MicroAPI::DataCopyUnAlign<int64_t, MicroAPI::PostLiteral::POST_MODE_UPDATE>(747+ MicroAPI::StoreUnAlign<int64_t, MicroAPI::PostLiteral::POST_MODE_UPDATE>(
748 (__ubuf__ int64_t*)(yIndicesCurDimAddr), (MicroAPI::RegTensor<int64_t>&)vregGatherResult1, ureg0);748 (__ubuf__ int64_t*)(yIndicesCurDimAddr), (MicroAPI::RegTensor<int64_t>&)vregGatherResult1, ureg0);
749 }749 }
750- AscendC::MicroAPI::DataCopyUnAlignPost(yIndicesCurDimAddr, ureg0);750+ AscendC::MicroAPI::StoreUnAlignPost(yIndicesCurDimAddr, ureg0);
751 }751 }
752 }752 }
753}753}
@@ -789,58 +789,57 @@ __aicore__ inline void SparseSliceDimension<T>::GatherValues(__ubuf__ int8_t* ma
789 789 
790 __VEC_SCOPE__790 __VEC_SCOPE__
791 {791 {
792- MicroAPI::UnalignReg ureg0;792+ MicroAPI::UnalignRegForStore ureg0;
793 AscendC::MicroAPI::ClearSpr<SpecialPurposeReg::AR>();793 AscendC::MicroAPI::ClearSpr<SpecialPurposeReg::AR>();
794 for (uint16_t loopIdx = 0; loopIdx < loopNum; loopIdx++) {794 for (uint16_t loopIdx = 0; loopIdx < loopNum; loopIdx++) {
795 pregInput = MicroAPI::UpdateMask<T1>(sreg0);795 pregInput = MicroAPI::UpdateMask<T1>(sreg0);
796- MicroAPI::DataCopy(vregInput, valuesUbAddr + loopIdx * repeatElm);796+ MicroAPI::LoadAlign(vregInput, valuesUbAddr + loopIdx * repeatElm);
797- MicroAPI::DataCopy(vregMask, maskUbAddr + loopIdx * repeatElm);797+ MicroAPI::LoadAlign(vregMask, maskUbAddr + loopIdx * repeatElm);
798 if constexpr (sizeof(T1) == sizeof(int8_t)) {798 if constexpr (sizeof(T1) == sizeof(int8_t)) {
799 pregCompare = pregInput;799 pregCompare = pregInput;
800 } else if constexpr (sizeof(T1) == sizeof(int16_t)) {800 } else if constexpr (sizeof(T1) == sizeof(int16_t)) {
801- MicroAPI::MaskPack<MicroAPI::HighLowPart::LOWEST>(pregCompare, pregInput);801+ MicroAPI::Pack<MicroAPI::HighLowPart::LOWEST>(pregCompare, pregInput);
802 } else if constexpr (sizeof(T1) == sizeof(int32_t)) {802 } else if constexpr (sizeof(T1) == sizeof(int32_t)) {
803- MicroAPI::MaskPack<MicroAPI::HighLowPart::LOWEST>(pregCompare, pregInput);803+ MicroAPI::Pack<MicroAPI::HighLowPart::LOWEST>(pregCompare, pregInput);
804- MicroAPI::MaskPack<MicroAPI::HighLowPart::LOWEST>(pregCompare, pregCompare);804+ MicroAPI::Pack<MicroAPI::HighLowPart::LOWEST>(pregCompare, pregCompare);
805 } else if constexpr (sizeof(T1) == sizeof(int64_t)) {805 } else if constexpr (sizeof(T1) == sizeof(int64_t)) {
806- MicroAPI::MaskPack<MicroAPI::HighLowPart::LOWEST>(pregCompare, pregInput);806+ MicroAPI::Pack<MicroAPI::HighLowPart::LOWEST>(pregCompare, pregInput);
807- MicroAPI::MaskPack<MicroAPI::HighLowPart::LOWEST>(pregCompare, pregCompare);807+ MicroAPI::Pack<MicroAPI::HighLowPart::LOWEST>(pregCompare, pregCompare);
808- MicroAPI::MaskPack<MicroAPI::HighLowPart::LOWEST>(pregCompare, pregCompare);808+ MicroAPI::Pack<MicroAPI::HighLowPart::LOWEST>(pregCompare, pregCompare);
809 }809 }
810- MicroAPI::CompareScalar<int8_t, CMPMODE::EQ>(pregSelectB8, vregMask, (int8_t)1, pregCompare);810+ MicroAPI::Compares<int8_t, CMPMODE::EQ>(pregSelectB8, vregMask, (int8_t)1, pregCompare);
811 if constexpr (sizeof(T1) == sizeof(int64_t)) {811 if constexpr (sizeof(T1) == sizeof(int64_t)) {
812- MicroAPI::MaskUnPack<MicroAPI::HighLowPart::LOWEST>(pregSelected, pregSelectB8);812+ MicroAPI::UnPack<MicroAPI::HighLowPart::LOWEST>(pregSelected, pregSelectB8);
813- MicroAPI::MaskUnPack<MicroAPI::HighLowPart::LOWEST>(pregSelected, pregSelected);813+ MicroAPI::UnPack<MicroAPI::HighLowPart::LOWEST>(pregSelected, pregSelected);
814 MicroAPI::DeInterleave<int32_t>(vregInputLowerHalf, vregInputHigherHalf,814 MicroAPI::DeInterleave<int32_t>(vregInputLowerHalf, vregInputHigherHalf,
815 (MicroAPI::RegTensor<int32_t>&)vregInput,815 (MicroAPI::RegTensor<int32_t>&)vregInput,
816 (MicroAPI::RegTensor<int32_t>&)vregInput);816 (MicroAPI::RegTensor<int32_t>&)vregInput);
817- MicroAPI::GatherMask<int32_t, MicroAPI::GatherMaskMode::NO_STORE_REG>(vregGatherLowerHalf,817+ MicroAPI::Squeeze<int32_t, MicroAPI::GatherMaskMode::NO_STORE_REG>(vregGatherLowerHalf,
818- vregInputLowerHalf, pregSelected);818+ vregInputLowerHalf, pregSelected);
819- MicroAPI::GatherMask<int32_t, MicroAPI::GatherMaskMode::NO_STORE_REG>(819+ MicroAPI::Squeeze<int32_t, MicroAPI::GatherMaskMode::NO_STORE_REG>(vregGatherHigherHalf,
820- vregGatherHigherHalf, vregInputHigherHalf, pregSelected);820+ vregInputHigherHalf, pregSelected);
821 MicroAPI::Interleave<int32_t>((MicroAPI::RegTensor<int32_t>&)vregGathered, vregGatherTemp,821 MicroAPI::Interleave<int32_t>((MicroAPI::RegTensor<int32_t>&)vregGathered, vregGatherTemp,
822 vregGatherLowerHalf, vregGatherHigherHalf);822 vregGatherLowerHalf, vregGatherHigherHalf);
823 } else if constexpr (sizeof(T1) == sizeof(int8_t)) {823 } else if constexpr (sizeof(T1) == sizeof(int8_t)) {
824 pregSelected = pregSelectB8;824 pregSelected = pregSelectB8;
825 } else if constexpr (sizeof(T1) == sizeof(int16_t)) {825 } else if constexpr (sizeof(T1) == sizeof(int16_t)) {
826- MicroAPI::MaskUnPack<MicroAPI::HighLowPart::LOWEST>(pregSelected, pregSelectB8);826+ MicroAPI::UnPack<MicroAPI::HighLowPart::LOWEST>(pregSelected, pregSelectB8);
827 } else if constexpr (sizeof(T1) == sizeof(int32_t)) {827 } else if constexpr (sizeof(T1) == sizeof(int32_t)) {
828- MicroAPI::MaskUnPack<MicroAPI::HighLowPart::LOWEST>(pregSelected, pregSelectB8);828+ MicroAPI::UnPack<MicroAPI::HighLowPart::LOWEST>(pregSelected, pregSelectB8);
829- MicroAPI::MaskUnPack<MicroAPI::HighLowPart::LOWEST>(pregSelected, pregSelected);829+ MicroAPI::UnPack<MicroAPI::HighLowPart::LOWEST>(pregSelected, pregSelected);
830 }830 }
831 if constexpr (sizeof(T1) == sizeof(int64_t)) {831 if constexpr (sizeof(T1) == sizeof(int64_t)) {
832 MicroAPI::MaskInterleave<int32_t>(pregSelectFinal1, pregSelectFinal2, pregSelected, pregSelected);832 MicroAPI::MaskInterleave<int32_t>(pregSelectFinal1, pregSelectFinal2, pregSelected, pregSelected);
833- MicroAPI::GatherMask<int32_t, MicroAPI::GatherMaskMode::STORE_REG>(vregGatherTemp, vregInputHigherHalf,833+ MicroAPI::Squeeze<int32_t, MicroAPI::GatherMaskMode::STORE_REG>(vregGatherTemp, vregInputHigherHalf,
834- pregSelectFinal1);834+ pregSelectFinal1);
835- MicroAPI::DataCopyUnAlign<T1, MicroAPI::PostLiteral::POST_MODE_UPDATE>(835+ MicroAPI::StoreUnAlign<T1, MicroAPI::PostLiteral::POST_MODE_UPDATE>(
836 yValuesUbAddr, (MicroAPI::RegTensor<T1>&)vregGathered, ureg0);836 yValuesUbAddr, (MicroAPI::RegTensor<T1>&)vregGathered, ureg0);
837 } else {837 } else {
838- MicroAPI::GatherMask<T1, MicroAPI::GatherMaskMode::STORE_REG>(vregGathered, vregInput, pregSelected);838+ MicroAPI::Squeeze<T1, MicroAPI::GatherMaskMode::STORE_REG>(vregGathered, vregInput, pregSelected);
839- MicroAPI::DataCopyUnAlign<T1, MicroAPI::PostLiteral::POST_MODE_UPDATE>(yValuesUbAddr, vregGathered,839+ MicroAPI::StoreUnAlign<T1, MicroAPI::PostLiteral::POST_MODE_UPDATE>(yValuesUbAddr, vregGathered, ureg0);
840- ureg0);
841 }840 }
842 }841 }
843- MicroAPI::DataCopyUnAlignPost(yValuesUbAddr, ureg0);842+ MicroAPI::StoreUnAlignPost(yValuesUbAddr, ureg0);
844 }843 }
845 specialArNum_ = AscendC::MicroAPI::GetSpr<SpecialPurposeReg::AR>() / sizeof(T1);844 specialArNum_ = AscendC::MicroAPI::GetSpr<SpecialPurposeReg::AR>() / sizeof(T1);
846}845}
@@ -901,8 +900,8 @@ __aicore__ inline void SparseSliceDimension<T>::Transpose2D(LocalTensor<int64_t>
901 900 
902 for (uint16_t loopIndex = 0; loopIndex < loopNum; loopIndex++) {901 for (uint16_t loopIndex = 0; loopIndex < loopNum; loopIndex++) {
903 preg = MicroAPI::UpdateMask<int64_t>(sreg0);902 preg = MicroAPI::UpdateMask<int64_t>(sreg0);
904- MicroAPI::MaskPack<MicroAPI::HighLowPart::LOWEST>(pregB32, preg);903+ MicroAPI::Pack<MicroAPI::HighLowPart::LOWEST>(pregB32, preg);
905- MicroAPI::DataCopy(vregInput, yIndicesTransposedAddr + loopIndex * vlSize);904+ MicroAPI::LoadAlign(vregInput, yIndicesTransposedAddr + loopIndex * vlSize);
906 MicroAPI::Arange<int32_t>(vregSrcIndex, (int32_t)(loopIndex * vlSize));905 MicroAPI::Arange<int32_t>(vregSrcIndex, (int32_t)(loopIndex * vlSize));
907 MicroAPI::Div(vreg0, vregSrcIndex, vregCol, pregB32);906 MicroAPI::Div(vreg0, vregSrcIndex, vregCol, pregB32);
908 MicroAPI::Mul(vreg1, vreg0, vregCol, pregB32);907 MicroAPI::Mul(vreg1, vreg0, vregCol, pregB32);
@@ -910,7 +909,7 @@ __aicore__ inline void SparseSliceDimension<T>::Transpose2D(LocalTensor<int64_t>
910 MicroAPI::Mul(vreg3, vreg2, vregRow, pregB32);909 MicroAPI::Mul(vreg3, vreg2, vregRow, pregB32);
911 MicroAPI::Add(vregDstIndex, vreg3, vreg0, pregB32);910 MicroAPI::Add(vregDstIndex, vreg3, vreg0, pregB32);
912 MicroAPI::UnPack<int64_t, int32_t, MicroAPI::HighLowPart::LOWEST>(vregDstIndexB64, vregDstIndex);911 MicroAPI::UnPack<int64_t, int32_t, MicroAPI::HighLowPart::LOWEST>(vregDstIndexB64, vregDstIndex);
913- MicroAPI::DataCopyScatter(yIndicesAddr, vregInput, (MicroAPI::RegTensor<uint64_t>&)vregDstIndexB64, preg);912+ MicroAPI::Scatter(yIndicesAddr, vregInput, (MicroAPI::RegTensor<uint64_t>&)vregDstIndexB64, preg);
914 }913 }
915 }914 }
916 yIndicesQueue_.EnQue(yIndicesUb);915 yIndicesQueue_.EnQue(yIndicesUb);
@@ -939,12 +938,12 @@ __aicore__ inline void SparseSliceDimension<T>::Transpose2D2Dim(LocalTensor<int6
939 {938 {
940 for (uint16_t loopIndex = 0; loopIndex < loopNum; loopIndex++) {939 for (uint16_t loopIndex = 0; loopIndex < loopNum; loopIndex++) {
941 preg = MicroAPI::UpdateMask<int64_t, MicroAPI::RegTraitNumTwo>(sreg0);940 preg = MicroAPI::UpdateMask<int64_t, MicroAPI::RegTraitNumTwo>(sreg0);
942- MicroAPI::DataCopy(vregInputDim0, yIndicesTransposedAddr + loopIndex * vlSize);941+ MicroAPI::LoadAlign(vregInputDim0, yIndicesTransposedAddr + loopIndex * vlSize);
943- MicroAPI::DataCopy(vregInputDim1, yIndicesTransposedAddr + loopIndex * vlSize + colNum);942+ MicroAPI::LoadAlign(vregInputDim1, yIndicesTransposedAddr + loopIndex * vlSize + colNum);
944 MicroAPI::Interleave(vregOutputPart0, vregOutputPart1, vregInputDim0, vregInputDim1);943 MicroAPI::Interleave(vregOutputPart0, vregOutputPart1, vregInputDim0, vregInputDim1);
945- MicroAPI::DataCopy<int64_t, MicroAPI::StoreDist::DIST_NORM>(yIndicesAddr + loopIndex * vlSize * DIGIT_TWO,944+ MicroAPI::StoreAlign<int64_t, MicroAPI::StoreDist::DIST_NORM>(yIndicesAddr + loopIndex * vlSize * DIGIT_TWO,
946- vregOutputPart0, preg);945+ vregOutputPart0, preg);
947- MicroAPI::DataCopy<int64_t, MicroAPI::StoreDist::DIST_NORM>(946+ MicroAPI::StoreAlign<int64_t, MicroAPI::StoreDist::DIST_NORM>(
948 yIndicesAddr + loopIndex * vlSize * DIGIT_TWO + vlSize, vregOutputPart1, preg);947 yIndicesAddr + loopIndex * vlSize * DIGIT_TWO + vlSize, vregOutputPart1, preg);
949 }948 }
950 }949 }
Mquant/ascend_anti_quant_v2/op_kernel/arch35/ascend_anti_quant_v2_common.h+7-7
@@ -81,24 +81,24 @@ protected:
81protected:81protected:
82 template <typename XType>82 template <typename XType>
83 __aicore__ inline void LoadCastXToFloat(AscendC::Reg::RegTensor<xCopyDtype>& vregX,83 __aicore__ inline void LoadCastXToFloat(AscendC::Reg::RegTensor<xCopyDtype>& vregX,
84- AscendC::Reg::RegTensor<float>& vregFloatX,84+ AscendC::Reg::RegTensor<float>& vregFloatX, __ubuf__ xCopyDtype* srcAddr,
85- __local_mem__ xCopyDtype* srcAddr, AscendC::Reg::MaskReg& mask)85+ AscendC::Reg::MaskReg& mask)
86 {86 {
87 if constexpr (IsSameType<XType, hifloat8_t>::value) {87 if constexpr (IsSameType<XType, hifloat8_t>::value) {
88- AscendC::Reg::DataCopy<XType, AscendC::Reg::LoadDist::DIST_UNPACK4_B8>(vregX, srcAddr);88+ AscendC::Reg::LoadAlign<XType, AscendC::Reg::LoadDist::DIST_UNPACK4_B8>(vregX, srcAddr);
89 AscendC::Reg::Cast<float, XType, AscendAntiQuantV2Base<T, T1, T2, U, SqrtMode>::CAST_TRAIT_HIFP8_TO_FP32>(89 AscendC::Reg::Cast<float, XType, AscendAntiQuantV2Base<T, T1, T2, U, SqrtMode>::CAST_TRAIT_HIFP8_TO_FP32>(
90 vregFloatX, vregX, mask);90 vregFloatX, vregX, mask);
91 } else if constexpr (IsSameType<XType, fp8_e5m2_t>::value) {91 } else if constexpr (IsSameType<XType, fp8_e5m2_t>::value) {
92- AscendC::Reg::DataCopy<XType, AscendC::Reg::LoadDist::DIST_UNPACK4_B8>(vregX, srcAddr);92+ AscendC::Reg::LoadAlign<XType, AscendC::Reg::LoadDist::DIST_UNPACK4_B8>(vregX, srcAddr);
93 AscendC::Reg::Cast<float, XType, AscendAntiQuantV2Base<T, T1, T2, U, SqrtMode>::CAST_TRAIT_FP8E5M2_TO_FP32>(93 AscendC::Reg::Cast<float, XType, AscendAntiQuantV2Base<T, T1, T2, U, SqrtMode>::CAST_TRAIT_FP8E5M2_TO_FP32>(
94 vregFloatX, vregX, mask);94 vregFloatX, vregX, mask);
95 } else if constexpr (IsSameType<XType, fp8_e4m3fn_t>::value) {95 } else if constexpr (IsSameType<XType, fp8_e4m3fn_t>::value) {
96- AscendC::Reg::DataCopy<XType, AscendC::Reg::LoadDist::DIST_UNPACK4_B8>(vregX, srcAddr);96+ AscendC::Reg::LoadAlign<XType, AscendC::Reg::LoadDist::DIST_UNPACK4_B8>(vregX, srcAddr);
97 AscendC::Reg::Cast<float, XType, AscendAntiQuantV2Base<T, T1, T2, U, SqrtMode>::CAST_TRAIT_FP8E4M3_TO_FP32>(97 AscendC::Reg::Cast<float, XType, AscendAntiQuantV2Base<T, T1, T2, U, SqrtMode>::CAST_TRAIT_FP8E4M3_TO_FP32>(
98 vregFloatX, vregX, mask);98 vregFloatX, vregX, mask);
99 } else if constexpr (IsSameType<XType, int8_t>::value) {99 } else if constexpr (IsSameType<XType, int8_t>::value) {
100 AscendC::Reg::RegTensor<half> vregHalfX;100 AscendC::Reg::RegTensor<half> vregHalfX;
101- AscendC::Reg::DataCopy<XType, AscendC::Reg::LoadDist::DIST_UNPACK4_B8>(vregX, srcAddr);101+ AscendC::Reg::LoadAlign<XType, AscendC::Reg::LoadDist::DIST_UNPACK4_B8>(vregX, srcAddr);
102 AscendC::Reg::Cast<half, XType, AscendAntiQuantV2Base<T, T1, T2, U, SqrtMode>::CAST_TRAIT_INT8_TO_HALF>(102 AscendC::Reg::Cast<half, XType, AscendAntiQuantV2Base<T, T1, T2, U, SqrtMode>::CAST_TRAIT_INT8_TO_HALF>(
103 vregHalfX, vregX, mask);103 vregHalfX, vregX, mask);
104 AscendC::Reg::Cast<float, half, AscendAntiQuantV2Base<T, T1, T2, U, SqrtMode>::CAST_TRAIT_HALF_TO_FP32>(104 AscendC::Reg::Cast<float, half, AscendAntiQuantV2Base<T, T1, T2, U, SqrtMode>::CAST_TRAIT_HALF_TO_FP32>(
@@ -106,7 +106,7 @@ protected:
106 } else if constexpr (IsSameType<XType, int4b_t>::value) {106 } else if constexpr (IsSameType<XType, int4b_t>::value) {
107 AscendC::Reg::RegTensor<half> vregTmpX;107 AscendC::Reg::RegTensor<half> vregTmpX;
108 AscendC::Reg::RegTensor<uint16_t> vregTmpU16X;108 AscendC::Reg::RegTensor<uint16_t> vregTmpU16X;
109- AscendC::Reg::DataCopy<xCopyDtype, AscendC::Reg::LoadDist::DIST_UNPACK4_B8>(vregX, srcAddr);109+ AscendC::Reg::LoadAlign<xCopyDtype, AscendC::Reg::LoadDist::DIST_UNPACK4_B8>(vregX, srcAddr);
110 AscendC::Reg::Cast<half, int4x2_t, AscendAntiQuantV2Base<T, T1, T2, U, SqrtMode>::CAST_TRAIT_INT8_TO_HALF>(110 AscendC::Reg::Cast<half, int4x2_t, AscendAntiQuantV2Base<T, T1, T2, U, SqrtMode>::CAST_TRAIT_INT8_TO_HALF>(
111 (AscendC::Reg::RegTensor<half>&)vregTmpU16X, (AscendC::Reg::RegTensor<int4x2_t>&)vregX, mask);111 (AscendC::Reg::RegTensor<half>&)vregTmpU16X, (AscendC::Reg::RegTensor<int4x2_t>&)vregX, mask);
112 AscendC::Reg::UnPack((AscendC::Reg::RegTensor<uint32_t>&)vregTmpX, vregTmpU16X);112 AscendC::Reg::UnPack((AscendC::Reg::RegTensor<uint32_t>&)vregTmpX, vregTmpU16X);
Mquant/ascend_anti_quant_v2/op_kernel/arch35/ascend_anti_quant_v2_per_channel_nddma_regbase.h+21-22
@@ -169,8 +169,8 @@ __aicore__ inline void AscendAntiQuantV2PerChannelNddmaRegbase<T, T1, T2, U, Sqr
169 int64_t sLen, int64_t sInOffset)169 int64_t sLen, int64_t sInOffset)
170{170{
171 LocalTensor<T1> sLocal = inQueueScale_.AllocTensor<T1>();171 LocalTensor<T1> sLocal = inQueueScale_.AllocTensor<T1>();
172- static constexpr AscendC::MultiCopyConfig copyConfig = {false, 0, 0, false};172+ static constexpr AscendC::NdDmaConfig copyConfig = {false, 0, 0, false};
173- MultiCopyLoopInfo<AscendAntiQuantV2Base<T, T1, T2, U, SqrtMode>::MULTI_COPY_DIM> copyLoopInfo;173+ NdDmaLoopInfo<AscendAntiQuantV2Base<T, T1, T2, U, SqrtMode>::MULTI_COPY_DIM> copyLoopInfo;
174 copyLoopInfo.loopSrcStride[0] = 1;174 copyLoopInfo.loopSrcStride[0] = 1;
175 copyLoopInfo.loopSrcStride[1] = 0;175 copyLoopInfo.loopSrcStride[1] = 0;
176 copyLoopInfo.loopDstStride[0] = 1;176 copyLoopInfo.loopDstStride[0] = 1;
@@ -179,8 +179,8 @@ __aicore__ inline void AscendAntiQuantV2PerChannelNddmaRegbase<T, T1, T2, U, Sqr
179 copyLoopInfo.loopSize[1] = tilingData_->baseN;179 copyLoopInfo.loopSize[1] = tilingData_->baseN;
180 180 
181 T1 constValue = 0;181 T1 constValue = 0;
182- AscendC::MultiCopyParams<T1, AscendAntiQuantV2Base<T, T1, T2, U, SqrtMode>::MULTI_COPY_DIM> copyParams = {182+ AscendC::NdDmaParams<T1, AscendAntiQuantV2Base<T, T1, T2, U, SqrtMode>::MULTI_COPY_DIM> copyParams = {copyLoopInfo,
183- copyLoopInfo, constValue};183+ constValue};
184 AscendC::DataCopy<T1, AscendAntiQuantV2Base<T, T1, T2, U, SqrtMode>::MULTI_COPY_DIM, copyConfig>(184 AscendC::DataCopy<T1, AscendAntiQuantV2Base<T, T1, T2, U, SqrtMode>::MULTI_COPY_DIM, copyConfig>(
185 sLocal, scaleGm_[sInOffset], copyParams);185 sLocal, scaleGm_[sInOffset], copyParams);
186 inQueueScale_.EnQue(sLocal);186 inQueueScale_.EnQue(sLocal);
@@ -191,8 +191,8 @@ __aicore__ inline void AscendAntiQuantV2PerChannelNddmaRegbase<T, T1, T2, U, Sqr
191 int64_t sLen, int64_t sInOffset)191 int64_t sLen, int64_t sInOffset)
192{192{
193 LocalTensor<T2> oLocal = inQueueOffset_.AllocTensor<T2>();193 LocalTensor<T2> oLocal = inQueueOffset_.AllocTensor<T2>();
194- static constexpr AscendC::MultiCopyConfig copyConfig = {false, 0, 0, false};194+ static constexpr AscendC::NdDmaConfig copyConfig = {false, 0, 0, false};
195- MultiCopyLoopInfo<AscendAntiQuantV2Base<T, T1, T2, U, SqrtMode>::MULTI_COPY_DIM> copyLoopInfo;195+ NdDmaLoopInfo<AscendAntiQuantV2Base<T, T1, T2, U, SqrtMode>::MULTI_COPY_DIM> copyLoopInfo;
196 copyLoopInfo.loopSrcStride[0] = 1;196 copyLoopInfo.loopSrcStride[0] = 1;
197 copyLoopInfo.loopSrcStride[1] = 0;197 copyLoopInfo.loopSrcStride[1] = 0;
198 copyLoopInfo.loopDstStride[0] = 1;198 copyLoopInfo.loopDstStride[0] = 1;
@@ -201,8 +201,8 @@ __aicore__ inline void AscendAntiQuantV2PerChannelNddmaRegbase<T, T1, T2, U, Sqr
201 copyLoopInfo.loopSize[1] = tilingData_->baseN;201 copyLoopInfo.loopSize[1] = tilingData_->baseN;
202 202 
203 T2 constValue = 0;203 T2 constValue = 0;
204- AscendC::MultiCopyParams<T2, AscendAntiQuantV2Base<T, T1, T2, U, SqrtMode>::MULTI_COPY_DIM> copyParams = {204+ AscendC::NdDmaParams<T2, AscendAntiQuantV2Base<T, T1, T2, U, SqrtMode>::MULTI_COPY_DIM> copyParams = {copyLoopInfo,
205- copyLoopInfo, constValue};205+ constValue};
206 AscendC::DataCopy<T2, AscendAntiQuantV2Base<T, T1, T2, U, SqrtMode>::MULTI_COPY_DIM, copyConfig>(206 AscendC::DataCopy<T2, AscendAntiQuantV2Base<T, T1, T2, U, SqrtMode>::MULTI_COPY_DIM, copyConfig>(
207 oLocal, offsetGm_[sInOffset], copyParams);207 oLocal, offsetGm_[sInOffset], copyParams);
208 inQueueOffset_.EnQue(oLocal);208 inQueueOffset_.EnQue(oLocal);
@@ -257,9 +257,9 @@ __aicore__ inline void AscendAntiQuantV2PerChannelNddmaRegbase<T, T1, T2, U, Sqr
257 LocalTensor<xCopyDtype> xLocal = inQueueX_.DeQue<xCopyDtype>();257 LocalTensor<xCopyDtype> xLocal = inQueueX_.DeQue<xCopyDtype>();
258 LocalTensor<U> outLocal = outQueueY_.AllocTensor<U>();258 LocalTensor<U> outLocal = outQueueY_.AllocTensor<U>();
259 259 
260- __local_mem__ xCopyDtype* xLocalAddr = (__local_mem__ xCopyDtype*)xLocal.GetPhyAddr();260+ __ubuf__ xCopyDtype* xLocalAddr = (__ubuf__ xCopyDtype*)xLocal.GetPhyAddr();
261- __local_mem__ T1* scaleLocalAddr = (__local_mem__ T1*)sLocal.GetPhyAddr();261+ __ubuf__ T1* scaleLocalAddr = (__ubuf__ T1*)sLocal.GetPhyAddr();
262- __local_mem__ U* outLocalAddr = (__local_mem__ U*)outLocal.GetPhyAddr();262+ __ubuf__ U* outLocalAddr = (__ubuf__ U*)outLocal.GetPhyAddr();
263 263 
264 uint16_t VL = AscendC::VECTOR_REG_WIDTH / sizeof(float);264 uint16_t VL = AscendC::VECTOR_REG_WIDTH / sizeof(float);
265 uint16_t HalfVL = VL / 2;265 uint16_t HalfVL = VL / 2;
@@ -288,17 +288,16 @@ __aicore__ inline void AscendAntiQuantV2PerChannelNddmaRegbase<T, T1, T2, U, Sqr
288 for (uint16_t i = 0; i < vfLoopNum; i++) {288 for (uint16_t i = 0; i < vfLoopNum; i++) {
289 mask = AscendC::Reg::UpdateMask<float>(count);289 mask = AscendC::Reg::UpdateMask<float>(count);
290 // ld and cast for x290 // ld and cast for x
291- __local_mem__ xCopyDtype* xSrc = IsSameType<T, int4b_t>::value ? xLocalAddr + i * HalfVL :291+ __ubuf__ xCopyDtype* xSrc = IsSameType<T, int4b_t>::value ? xLocalAddr + i * HalfVL : xLocalAddr + i * VL;
292- xLocalAddr + i * VL;
293 this->template LoadCastXToFloat<T>(vregX, vregFloatX, xSrc, mask);292 this->template LoadCastXToFloat<T>(vregX, vregFloatX, xSrc, mask);
294 293 
295 // ld and cast for scale294 // ld and cast for scale
296 if constexpr (IsSameType<T1, float>::value) {295 if constexpr (IsSameType<T1, float>::value) {
297 // fp32296 // fp32
298- AscendC::Reg::DataCopy<float, AscendC::Reg::LoadDist::DIST_NORM>(vregFloatS, scaleLocalAddr + i * VL);297+ AscendC::Reg::LoadAlign<float, AscendC::Reg::LoadDist::DIST_NORM>(vregFloatS, scaleLocalAddr + i * VL);
299 } else if constexpr (IsSameType<T1, bfloat16_t>::value) {298 } else if constexpr (IsSameType<T1, bfloat16_t>::value) {
300 // bf16299 // bf16
301- AscendC::Reg::DataCopy<T1, AscendC::Reg::LoadDist::DIST_UNPACK_B16>(vregS, scaleLocalAddr + i * VL);300+ AscendC::Reg::LoadAlign<T1, AscendC::Reg::LoadDist::DIST_UNPACK_B16>(vregS, scaleLocalAddr + i * VL);
302 AscendC::Reg::Cast<float, T1, AscendAntiQuantV2Base<T, T1, T2, U, SqrtMode>::CAST_TRAIT_BF16_TO_FP32>(301 AscendC::Reg::Cast<float, T1, AscendAntiQuantV2Base<T, T1, T2, U, SqrtMode>::CAST_TRAIT_BF16_TO_FP32>(
303 vregFloatS, vregS, mask);302 vregFloatS, vregS, mask);
304 }303 }
@@ -306,13 +305,13 @@ __aicore__ inline void AscendAntiQuantV2PerChannelNddmaRegbase<T, T1, T2, U, Sqr
306 // compute305 // compute
307 if constexpr (HasOffset) {306 if constexpr (HasOffset) {
308 // ld and cast for offset307 // ld and cast for offset
309- __local_mem__ T2* offsetLocalAddr = (__local_mem__ T2*)oLocal.GetPhyAddr();308+ __ubuf__ T2* offsetLocalAddr = (__ubuf__ T2*)oLocal.GetPhyAddr();
310 if constexpr (IsSameType<T2, float>::value) {309 if constexpr (IsSameType<T2, float>::value) {
311- AscendC::Reg::DataCopy<float, AscendC::Reg::LoadDist::DIST_NORM>(vregFloatO,310+ AscendC::Reg::LoadAlign<float, AscendC::Reg::LoadDist::DIST_NORM>(vregFloatO,
312- offsetLocalAddr + i * VL);311+ offsetLocalAddr + i * VL);
313 } else if constexpr (IsSameType<T2, bfloat16_t>::value) {312 } else if constexpr (IsSameType<T2, bfloat16_t>::value) {
314- AscendC::Reg::DataCopy<T2, AscendC::Reg::LoadDist::DIST_UNPACK_B16>(vregO,313+ AscendC::Reg::LoadAlign<T2, AscendC::Reg::LoadDist::DIST_UNPACK_B16>(vregO,
315- offsetLocalAddr + i * VL);314+ offsetLocalAddr + i * VL);
316 AscendC::Reg::Cast<float, T2,315 AscendC::Reg::Cast<float, T2,
317 AscendAntiQuantV2Base<T, T1, T2, U, SqrtMode>::CAST_TRAIT_BF16_TO_FP32>(316 AscendAntiQuantV2Base<T, T1, T2, U, SqrtMode>::CAST_TRAIT_BF16_TO_FP32>(
318 vregFloatO, vregO, mask);317 vregFloatO, vregO, mask);
@@ -333,11 +332,11 @@ __aicore__ inline void AscendAntiQuantV2PerChannelNddmaRegbase<T, T1, T2, U, Sqr
333 if constexpr (IsSameType<U, half>::value) {332 if constexpr (IsSameType<U, half>::value) {
334 AscendC::Reg::Cast<half, float, AscendAntiQuantV2Base<T, T1, T2, U, SqrtMode>::CAST_TRAIT_FP32_TO_HALF>(333 AscendC::Reg::Cast<half, float, AscendAntiQuantV2Base<T, T1, T2, U, SqrtMode>::CAST_TRAIT_FP32_TO_HALF>(
335 vregY, vregFloatY, mask);334 vregY, vregFloatY, mask);
336- AscendC::Reg::DataCopy<U, AscendC::Reg::StoreDist::DIST_PACK_B32>(outLocalAddr + i * VL, vregY, mask);335+ AscendC::Reg::StoreAlign<U, AscendC::Reg::StoreDist::DIST_PACK_B32>(outLocalAddr + i * VL, vregY, mask);
337 } else if constexpr (IsSameType<U, bfloat16_t>::value) {336 } else if constexpr (IsSameType<U, bfloat16_t>::value) {
338 AscendC::Reg::Cast<U, float, AscendAntiQuantV2Base<T, T1, T2, U, SqrtMode>::CAST_TRAIT_FP32_TO_BF16>(337 AscendC::Reg::Cast<U, float, AscendAntiQuantV2Base<T, T1, T2, U, SqrtMode>::CAST_TRAIT_FP32_TO_BF16>(
339 vregY, vregFloatY, mask);338 vregY, vregFloatY, mask);
340- AscendC::Reg::DataCopy<U, AscendC::Reg::StoreDist::DIST_PACK_B32>(outLocalAddr + i * VL, vregY, mask);339+ AscendC::Reg::StoreAlign<U, AscendC::Reg::StoreDist::DIST_PACK_B32>(outLocalAddr + i * VL, vregY, mask);
341 }340 }
342 }341 }
343 }342 }
Mquant/ascend_anti_quant_v2/op_kernel/arch35/ascend_anti_quant_v2_per_channel_regbase.h+16-16
@@ -235,9 +235,9 @@ __aicore__ inline void AscendAntiQuantV2PerChannelRegbase<T, T1, T2, U, SqrtMode
235 LocalTensor<xCopyDtype> xLocal = inQueueX_.DeQue<xCopyDtype>();235 LocalTensor<xCopyDtype> xLocal = inQueueX_.DeQue<xCopyDtype>();
236 LocalTensor<U> outLocal = outQueueY_.AllocTensor<U>();236 LocalTensor<U> outLocal = outQueueY_.AllocTensor<U>();
237 237 
238- __local_mem__ xCopyDtype* xLocalAddr = (__local_mem__ xCopyDtype*)xLocal.GetPhyAddr();238+ __ubuf__ xCopyDtype* xLocalAddr = (__ubuf__ xCopyDtype*)xLocal.GetPhyAddr();
239- __local_mem__ T1* scaleLocalAddr = (__local_mem__ T1*)sLocal.GetPhyAddr();239+ __ubuf__ T1* scaleLocalAddr = (__ubuf__ T1*)sLocal.GetPhyAddr();
240- __local_mem__ U* outLocalAddr = (__local_mem__ U*)outLocal.GetPhyAddr();240+ __ubuf__ U* outLocalAddr = (__ubuf__ U*)outLocal.GetPhyAddr();
241 241 
242 uint16_t VL = AscendC::VECTOR_REG_WIDTH / sizeof(float);242 uint16_t VL = AscendC::VECTOR_REG_WIDTH / sizeof(float);
243 uint16_t HalfVL = VL / 2;243 uint16_t HalfVL = VL / 2;
@@ -269,19 +269,19 @@ __aicore__ inline void AscendAntiQuantV2PerChannelRegbase<T, T1, T2, U, SqrtMode
269 for (uint16_t i = 0; i < vfLoopNum; i++) {269 for (uint16_t i = 0; i < vfLoopNum; i++) {
270 mask = AscendC::Reg::UpdateMask<float>(count);270 mask = AscendC::Reg::UpdateMask<float>(count);
271 // ld and cast for x271 // ld and cast for x
272- __local_mem__ xCopyDtype* xSrc = IsSameType<T, int4b_t>::value ?272+ __ubuf__ xCopyDtype* xSrc = IsSameType<T, int4b_t>::value ? xLocalAddr + i * HalfVL + j * xLocalOffset :
273- xLocalAddr + i * HalfVL + j * xLocalOffset :273+ xLocalAddr + i * VL + j * xLocalOffset;
274- xLocalAddr + i * VL + j * xLocalOffset;
275 this->template LoadCastXToFloat<T>(vregX, vregFloatX, xSrc, mask);274 this->template LoadCastXToFloat<T>(vregX, vregFloatX, xSrc, mask);
276 275 
277 // ld and cast for scale276 // ld and cast for scale
278 if constexpr (IsSameType<T1, float>::value) {277 if constexpr (IsSameType<T1, float>::value) {
279 // fp32278 // fp32
280- AscendC::Reg::DataCopy<float, AscendC::Reg::LoadDist::DIST_NORM>(vregFloatS,279+ AscendC::Reg::LoadAlign<float, AscendC::Reg::LoadDist::DIST_NORM>(vregFloatS,
281- scaleLocalAddr + i * VL);280+ scaleLocalAddr + i * VL);
282 } else if constexpr (IsSameType<T1, bfloat16_t>::value) {281 } else if constexpr (IsSameType<T1, bfloat16_t>::value) {
283 // bf16282 // bf16
284- AscendC::Reg::DataCopy<T1, AscendC::Reg::LoadDist::DIST_UNPACK_B16>(vregS, scaleLocalAddr + i * VL);283+ AscendC::Reg::LoadAlign<T1, AscendC::Reg::LoadDist::DIST_UNPACK_B16>(vregS,
284+ scaleLocalAddr + i * VL);
285 AscendC::Reg::Cast<float, T1,285 AscendC::Reg::Cast<float, T1,
286 AscendAntiQuantV2Base<T, T1, T2, U, SqrtMode>::CAST_TRAIT_BF16_TO_FP32>(286 AscendAntiQuantV2Base<T, T1, T2, U, SqrtMode>::CAST_TRAIT_BF16_TO_FP32>(
287 vregFloatS, vregS, mask);287 vregFloatS, vregS, mask);
@@ -290,13 +290,13 @@ __aicore__ inline void AscendAntiQuantV2PerChannelRegbase<T, T1, T2, U, SqrtMode
290 // compute290 // compute
291 if constexpr (HasOffset) {291 if constexpr (HasOffset) {
292 // ld and cast for offset292 // ld and cast for offset
293- __local_mem__ T2* offsetLocalAddr = (__local_mem__ T2*)oLocal.GetPhyAddr();293+ __ubuf__ T2* offsetLocalAddr = (__ubuf__ T2*)oLocal.GetPhyAddr();
294 if constexpr (IsSameType<T2, float>::value) {294 if constexpr (IsSameType<T2, float>::value) {
295- AscendC::Reg::DataCopy<float, AscendC::Reg::LoadDist::DIST_NORM>(vregFloatO,295+ AscendC::Reg::LoadAlign<float, AscendC::Reg::LoadDist::DIST_NORM>(vregFloatO,
296- offsetLocalAddr + i * VL);296+ offsetLocalAddr + i * VL);
297 } else if constexpr (IsSameType<T2, bfloat16_t>::value) {297 } else if constexpr (IsSameType<T2, bfloat16_t>::value) {
298- AscendC::Reg::DataCopy<T2, AscendC::Reg::LoadDist::DIST_UNPACK_B16>(vregO,298+ AscendC::Reg::LoadAlign<T2, AscendC::Reg::LoadDist::DIST_UNPACK_B16>(vregO,
299- offsetLocalAddr + i * VL);299+ offsetLocalAddr + i * VL);
300 AscendC::Reg::Cast<float, T2,300 AscendC::Reg::Cast<float, T2,
301 AscendAntiQuantV2Base<T, T1, T2, U, SqrtMode>::CAST_TRAIT_BF16_TO_FP32>(301 AscendAntiQuantV2Base<T, T1, T2, U, SqrtMode>::CAST_TRAIT_BF16_TO_FP32>(
302 vregFloatO, vregO, mask);302 vregFloatO, vregO, mask);
@@ -319,13 +319,13 @@ __aicore__ inline void AscendAntiQuantV2PerChannelRegbase<T, T1, T2, U, SqrtMode
319 AscendC::Reg::Cast<half, float,319 AscendC::Reg::Cast<half, float,
320 AscendAntiQuantV2Base<T, T1, T2, U, SqrtMode>::CAST_TRAIT_FP32_TO_HALF>(320 AscendAntiQuantV2Base<T, T1, T2, U, SqrtMode>::CAST_TRAIT_FP32_TO_HALF>(
321 vregY, vregFloatY, mask);321 vregY, vregFloatY, mask);
322- AscendC::Reg::DataCopy<U, AscendC::Reg::StoreDist::DIST_PACK_B32>(322+ AscendC::Reg::StoreAlign<U, AscendC::Reg::StoreDist::DIST_PACK_B32>(
323 outLocalAddr + i * VL + j * xLocalOffset, vregY, mask);323 outLocalAddr + i * VL + j * xLocalOffset, vregY, mask);
324 } else if constexpr (IsSameType<U, bfloat16_t>::value) {324 } else if constexpr (IsSameType<U, bfloat16_t>::value) {
325 AscendC::Reg::Cast<U, float,325 AscendC::Reg::Cast<U, float,
326 AscendAntiQuantV2Base<T, T1, T2, U, SqrtMode>::CAST_TRAIT_FP32_TO_BF16>(326 AscendAntiQuantV2Base<T, T1, T2, U, SqrtMode>::CAST_TRAIT_FP32_TO_BF16>(
327 vregY, vregFloatY, mask);327 vregY, vregFloatY, mask);
328- AscendC::Reg::DataCopy<U, AscendC::Reg::StoreDist::DIST_PACK_B32>(328+ AscendC::Reg::StoreAlign<U, AscendC::Reg::StoreDist::DIST_PACK_B32>(
329 outLocalAddr + i * VL + j * xLocalOffset, vregY, mask);329 outLocalAddr + i * VL + j * xLocalOffset, vregY, mask);
330 }330 }
331 }331 }
Mquant/ascend_anti_quant_v2/op_kernel/arch35/ascend_anti_quant_v2_per_head_nddma_regbase.h+16-17
@@ -171,8 +171,8 @@ __aicore__ inline void AscendAntiQuantV2PerHeadNddmaRegbase<T, T1, T2, U, SqrtMo
171 int64_t paramOffset)171 int64_t paramOffset)
172{172{
173 auto paramLocal = inQueue.AllocTensor<dtypeCopyIn>();173 auto paramLocal = inQueue.AllocTensor<dtypeCopyIn>();
174- static constexpr AscendC::MultiCopyConfig copyConfig = {false, 0, 0, false};174+ static constexpr AscendC::NdDmaConfig copyConfig = {false, 0, 0, false};
175- MultiCopyLoopInfo<AscendAntiQuantV2Base<T, T1, T2, U, SqrtMode>::MULTI_COPY_DIM> multiCopyParams;175+ NdDmaLoopInfo<AscendAntiQuantV2Base<T, T1, T2, U, SqrtMode>::MULTI_COPY_DIM> multiCopyParams;
176 // src stride info per loop.176 // src stride info per loop.
177 multiCopyParams.loopSrcStride[0] = 0;177 multiCopyParams.loopSrcStride[0] = 0;
178 multiCopyParams.loopSrcStride[1] = 1;178 multiCopyParams.loopSrcStride[1] = 1;
@@ -183,7 +183,7 @@ __aicore__ inline void AscendAntiQuantV2PerHeadNddmaRegbase<T, T1, T2, U, SqrtMo
183 multiCopyParams.loopSize[0] = tilingData_->baseLen;183 multiCopyParams.loopSize[0] = tilingData_->baseLen;
184 multiCopyParams.loopSize[1] = paramLen;184 multiCopyParams.loopSize[1] = paramLen;
185 dtypeCopyIn constValue = 0;185 dtypeCopyIn constValue = 0;
186- AscendC::MultiCopyParams<dtypeCopyIn, AscendAntiQuantV2Base<T, T1, T2, U, SqrtMode>::MULTI_COPY_DIM> paramsMain = {186+ AscendC::NdDmaParams<dtypeCopyIn, AscendAntiQuantV2Base<T, T1, T2, U, SqrtMode>::MULTI_COPY_DIM> paramsMain = {
187 multiCopyParams, constValue};187 multiCopyParams, constValue};
188 AscendC::DataCopy<dtypeCopyIn, AscendAntiQuantV2Base<T, T1, T2, U, SqrtMode>::MULTI_COPY_DIM, copyConfig>(188 AscendC::DataCopy<dtypeCopyIn, AscendAntiQuantV2Base<T, T1, T2, U, SqrtMode>::MULTI_COPY_DIM, copyConfig>(
189 paramLocal, inGm[paramOffset], paramsMain);189 paramLocal, inGm[paramOffset], paramsMain);
@@ -260,9 +260,9 @@ __aicore__ inline void AscendAntiQuantV2PerHeadNddmaRegbase<T, T1, T2, U, SqrtMo
260 LocalTensor<xCopyDtype> xLocal = inQueueX_.DeQue<xCopyDtype>();260 LocalTensor<xCopyDtype> xLocal = inQueueX_.DeQue<xCopyDtype>();
261 LocalTensor<U> outLocal = outQueueY_.AllocTensor<U>();261 LocalTensor<U> outLocal = outQueueY_.AllocTensor<U>();
262 262 
263- __local_mem__ xCopyDtype* xLocalAddr = (__local_mem__ xCopyDtype*)xLocal.GetPhyAddr();263+ __ubuf__ xCopyDtype* xLocalAddr = (__ubuf__ xCopyDtype*)xLocal.GetPhyAddr();
264- __local_mem__ T1* scaleLocalAddr = (__local_mem__ T1*)sLocal.GetPhyAddr();264+ __ubuf__ T1* scaleLocalAddr = (__ubuf__ T1*)sLocal.GetPhyAddr();
265- __local_mem__ U* outLocalAddr = (__local_mem__ U*)outLocal.GetPhyAddr();265+ __ubuf__ U* outLocalAddr = (__ubuf__ U*)outLocal.GetPhyAddr();
266 266 
267 uint16_t VL = AscendC::VECTOR_REG_WIDTH / sizeof(float);267 uint16_t VL = AscendC::VECTOR_REG_WIDTH / sizeof(float);
268 uint16_t HalfVL = VL / 2;268 uint16_t HalfVL = VL / 2;
@@ -290,32 +290,31 @@ __aicore__ inline void AscendAntiQuantV2PerHeadNddmaRegbase<T, T1, T2, U, SqrtMo
290 uint16_t vfLoopNum = (count + VL - 1) / VL;290 uint16_t vfLoopNum = (count + VL - 1) / VL;
291 for (uint16_t i = 0; i < vfLoopNum; i++) {291 for (uint16_t i = 0; i < vfLoopNum; i++) {
292 mask = AscendC::Reg::UpdateMask<float>(count);292 mask = AscendC::Reg::UpdateMask<float>(count);
293- __local_mem__ xCopyDtype* xSrc = IsSameType<T, int4b_t>::value ? xLocalAddr + i * HalfVL :293+ __ubuf__ xCopyDtype* xSrc = IsSameType<T, int4b_t>::value ? xLocalAddr + i * HalfVL : xLocalAddr + i * VL;
294- xLocalAddr + i * VL;
295 this->template LoadCastXToFloat<T>(vregX, vregFloatX, xSrc, mask);294 this->template LoadCastXToFloat<T>(vregX, vregFloatX, xSrc, mask);
296 295 
297 // ld and cast for scale296 // ld and cast for scale
298 if constexpr (IsSameType<T1, float>::value) {297 if constexpr (IsSameType<T1, float>::value) {
299 // fp32298 // fp32
300- AscendC::Reg::DataCopy<float, AscendC::Reg::LoadDist::DIST_NORM>(vregFloatS, scaleLocalAddr + i * VL);299+ AscendC::Reg::LoadAlign<float, AscendC::Reg::LoadDist::DIST_NORM>(vregFloatS, scaleLocalAddr + i * VL);
301 } else if constexpr (IsSameType<T1, bfloat16_t>::value) {300 } else if constexpr (IsSameType<T1, bfloat16_t>::value) {
302 // bf16301 // bf16
303- AscendC::Reg::DataCopy<T1, AscendC::Reg::LoadDist::DIST_UNPACK_B16>(vregS, scaleLocalAddr + i * VL);302+ AscendC::Reg::LoadAlign<T1, AscendC::Reg::LoadDist::DIST_UNPACK_B16>(vregS, scaleLocalAddr + i * VL);
304 AscendC::Reg::Cast<float, T1, AscendAntiQuantV2Base<T, T1, T2, U, SqrtMode>::CAST_TRAIT_BF16_TO_FP32>(303 AscendC::Reg::Cast<float, T1, AscendAntiQuantV2Base<T, T1, T2, U, SqrtMode>::CAST_TRAIT_BF16_TO_FP32>(
305 vregFloatS, vregS, mask);304 vregFloatS, vregS, mask);
306 }305 }
307 306 
308 if constexpr (HasOffset) {307 if constexpr (HasOffset) {
309 // ld and cast for offset308 // ld and cast for offset
310- __local_mem__ T2* offsetLocalAddr = (__local_mem__ T2*)oLocal.GetPhyAddr();309+ __ubuf__ T2* offsetLocalAddr = (__ubuf__ T2*)oLocal.GetPhyAddr();
311 if constexpr (IsSameType<T2, float>::value) {310 if constexpr (IsSameType<T2, float>::value) {
312 // fp32311 // fp32
313- AscendC::Reg::DataCopy<float, AscendC::Reg::LoadDist::DIST_NORM>(vregFloatO,312+ AscendC::Reg::LoadAlign<float, AscendC::Reg::LoadDist::DIST_NORM>(vregFloatO,
314- offsetLocalAddr + i * VL);313+ offsetLocalAddr + i * VL);
315 } else if constexpr (IsSameType<T2, bfloat16_t>::value) {314 } else if constexpr (IsSameType<T2, bfloat16_t>::value) {
316 // bf16315 // bf16
317- AscendC::Reg::DataCopy<T2, AscendC::Reg::LoadDist::DIST_UNPACK_B16>(vregO,316+ AscendC::Reg::LoadAlign<T2, AscendC::Reg::LoadDist::DIST_UNPACK_B16>(vregO,
318- offsetLocalAddr + i * VL);317+ offsetLocalAddr + i * VL);
319 AscendC::Reg::Cast<float, T2,318 AscendC::Reg::Cast<float, T2,
320 AscendAntiQuantV2Base<T, T1, T2, U, SqrtMode>::CAST_TRAIT_BF16_TO_FP32>(319 AscendAntiQuantV2Base<T, T1, T2, U, SqrtMode>::CAST_TRAIT_BF16_TO_FP32>(
321 vregFloatO, vregO, mask);320 vregFloatO, vregO, mask);
@@ -338,11 +337,11 @@ __aicore__ inline void AscendAntiQuantV2PerHeadNddmaRegbase<T, T1, T2, U, SqrtMo
338 if constexpr (IsSameType<U, half>::value) {337 if constexpr (IsSameType<U, half>::value) {
339 AscendC::Reg::Cast<half, float, AscendAntiQuantV2Base<T, T1, T2, U, SqrtMode>::CAST_TRAIT_FP32_TO_HALF>(338 AscendC::Reg::Cast<half, float, AscendAntiQuantV2Base<T, T1, T2, U, SqrtMode>::CAST_TRAIT_FP32_TO_HALF>(
340 vregY, vregFloatY, mask);339 vregY, vregFloatY, mask);
341- AscendC::Reg::DataCopy<U, AscendC::Reg::StoreDist::DIST_PACK_B32>(outLocalAddr + i * VL, vregY, mask);340+ AscendC::Reg::StoreAlign<U, AscendC::Reg::StoreDist::DIST_PACK_B32>(outLocalAddr + i * VL, vregY, mask);
342 } else if constexpr (IsSameType<U, bfloat16_t>::value) {341 } else if constexpr (IsSameType<U, bfloat16_t>::value) {
343 AscendC::Reg::Cast<U, float, AscendAntiQuantV2Base<T, T1, T2, U, SqrtMode>::CAST_TRAIT_FP32_TO_BF16>(342 AscendC::Reg::Cast<U, float, AscendAntiQuantV2Base<T, T1, T2, U, SqrtMode>::CAST_TRAIT_FP32_TO_BF16>(
344 vregY, vregFloatY, mask);343 vregY, vregFloatY, mask);
345- AscendC::Reg::DataCopy<U, AscendC::Reg::StoreDist::DIST_PACK_B32>(outLocalAddr + i * VL, vregY, mask);344+ AscendC::Reg::StoreAlign<U, AscendC::Reg::StoreDist::DIST_PACK_B32>(outLocalAddr + i * VL, vregY, mask);
346 }345 }
347 }346 }
348 }347 }
Mquant/ascend_anti_quant_v2/op_kernel/arch35/ascend_anti_quant_v2_per_head_regbase.h+14-14
@@ -287,9 +287,9 @@ __aicore__ inline void AscendAntiQuantV2PerHeadRegbase<T, T1, T2, U, SqrtMode, H
287 LocalTensor<xCopyDtype> xLocal = inQueueX_.DeQue<xCopyDtype>();287 LocalTensor<xCopyDtype> xLocal = inQueueX_.DeQue<xCopyDtype>();
288 LocalTensor<U> outLocal = outQueueY_.AllocTensor<U>();288 LocalTensor<U> outLocal = outQueueY_.AllocTensor<U>();
289 289 
290- __local_mem__ xCopyDtype* xLocalAddr = (__local_mem__ xCopyDtype*)xLocal.GetPhyAddr();290+ __ubuf__ xCopyDtype* xLocalAddr = (__ubuf__ xCopyDtype*)xLocal.GetPhyAddr();
291- __local_mem__ T1* scaleLocalAddr = (__local_mem__ T1*)sLocal.GetPhyAddr();291+ __ubuf__ T1* scaleLocalAddr = (__ubuf__ T1*)sLocal.GetPhyAddr();
292- __local_mem__ U* outLocalAddr = (__local_mem__ U*)outLocal.GetPhyAddr();292+ __ubuf__ U* outLocalAddr = (__ubuf__ U*)outLocal.GetPhyAddr();
293 293 
294 uint16_t VL = AscendC::VECTOR_REG_WIDTH / sizeof(float);294 uint16_t VL = AscendC::VECTOR_REG_WIDTH / sizeof(float);
295 uint16_t HalfVL = VL / 2;295 uint16_t HalfVL = VL / 2;
@@ -320,18 +320,18 @@ __aicore__ inline void AscendAntiQuantV2PerHeadRegbase<T, T1, T2, U, SqrtMode, H
320 uint32_t count = dataCount;320 uint32_t count = dataCount;
321 for (uint16_t i = 0; i < vfLoopNum; i++) {321 for (uint16_t i = 0; i < vfLoopNum; i++) {
322 mask = AscendC::Reg::UpdateMask<float>(count);322 mask = AscendC::Reg::UpdateMask<float>(count);
323- __local_mem__ xCopyDtype* xSrc = IsSameType<T, int4b_t>::value ?323+ __ubuf__ xCopyDtype* xSrc = IsSameType<T, int4b_t>::value ? xLocalAddr + i * HalfVL + j * xLocalOffset :
324- xLocalAddr + i * HalfVL + j * xLocalOffset :324+ xLocalAddr + i * VL + j * xLocalOffset;
325- xLocalAddr + i * VL + j * xLocalOffset;
326 this->template LoadCastXToFloat<T>(vregX, vregFloatX, xSrc, mask);325 this->template LoadCastXToFloat<T>(vregX, vregFloatX, xSrc, mask);
327 326 
328 // ld and cast for scale327 // ld and cast for scale
329 if constexpr (IsSameType<T1, float>::value) {328 if constexpr (IsSameType<T1, float>::value) {
330 // fp32329 // fp32
331- AscendC::Reg::DataCopy<float, AscendC::Reg::LoadDist::DIST_BRC_B32>(vregFloatS, scaleLocalAddr + j);330+ AscendC::Reg::LoadAlign<float, AscendC::Reg::LoadDist::DIST_BRC_B32>(vregFloatS,
331+ scaleLocalAddr + j);
332 } else if constexpr (IsSameType<T1, bfloat16_t>::value) {332 } else if constexpr (IsSameType<T1, bfloat16_t>::value) {
333 // bf16333 // bf16
334- AscendC::Reg::DataCopy<T1, AscendC::Reg::LoadDist::DIST_BRC_B16>(vregS, scaleLocalAddr + j);334+ AscendC::Reg::LoadAlign<T1, AscendC::Reg::LoadDist::DIST_BRC_B16>(vregS, scaleLocalAddr + j);
335 AscendC::Reg::Cast<float, T1,335 AscendC::Reg::Cast<float, T1,
336 AscendAntiQuantV2Base<T, T1, T2, U, SqrtMode>::CAST_TRAIT_BF16_TO_FP32>(336 AscendAntiQuantV2Base<T, T1, T2, U, SqrtMode>::CAST_TRAIT_BF16_TO_FP32>(
337 vregFloatS, vregS, mask);337 vregFloatS, vregS, mask);
@@ -339,14 +339,14 @@ __aicore__ inline void AscendAntiQuantV2PerHeadRegbase<T, T1, T2, U, SqrtMode, H
339 339 
340 if constexpr (HasOffset) {340 if constexpr (HasOffset) {
341 // ld and cast for offset341 // ld and cast for offset
342- __local_mem__ T2* offsetLocalAddr = (__local_mem__ T2*)oLocal.GetPhyAddr();342+ __ubuf__ T2* offsetLocalAddr = (__ubuf__ T2*)oLocal.GetPhyAddr();
343 if constexpr (IsSameType<T2, float>::value) {343 if constexpr (IsSameType<T2, float>::value) {
344 // fp32344 // fp32
345- AscendC::Reg::DataCopy<float, AscendC::Reg::LoadDist::DIST_BRC_B32>(vregFloatO,345+ AscendC::Reg::LoadAlign<float, AscendC::Reg::LoadDist::DIST_BRC_B32>(vregFloatO,
346- offsetLocalAddr + j);346+ offsetLocalAddr + j);
347 } else if constexpr (IsSameType<T2, bfloat16_t>::value) {347 } else if constexpr (IsSameType<T2, bfloat16_t>::value) {
348 // bf16348 // bf16
349- AscendC::Reg::DataCopy<T2, AscendC::Reg::LoadDist::DIST_BRC_B16>(vregO, offsetLocalAddr + j);349+ AscendC::Reg::LoadAlign<T2, AscendC::Reg::LoadDist::DIST_BRC_B16>(vregO, offsetLocalAddr + j);
350 AscendC::Reg::Cast<float, T2,350 AscendC::Reg::Cast<float, T2,
351 AscendAntiQuantV2Base<T, T1, T2, U, SqrtMode>::CAST_TRAIT_BF16_TO_FP32>(351 AscendAntiQuantV2Base<T, T1, T2, U, SqrtMode>::CAST_TRAIT_BF16_TO_FP32>(
352 vregFloatO, vregO, mask);352 vregFloatO, vregO, mask);
@@ -371,13 +371,13 @@ __aicore__ inline void AscendAntiQuantV2PerHeadRegbase<T, T1, T2, U, SqrtMode, H
371 AscendC::Reg::Cast<half, float,371 AscendC::Reg::Cast<half, float,
372 AscendAntiQuantV2Base<T, T1, T2, U, SqrtMode>::CAST_TRAIT_FP32_TO_HALF>(372 AscendAntiQuantV2Base<T, T1, T2, U, SqrtMode>::CAST_TRAIT_FP32_TO_HALF>(
373 vregY, vregFloatY, mask);373 vregY, vregFloatY, mask);
374- AscendC::Reg::DataCopy<U, AscendC::Reg::StoreDist::DIST_PACK_B32>(374+ AscendC::Reg::StoreAlign<U, AscendC::Reg::StoreDist::DIST_PACK_B32>(
375 outLocalAddr + i * VL + j * xLocalOffset, vregY, mask);375 outLocalAddr + i * VL + j * xLocalOffset, vregY, mask);
376 } else if constexpr (IsSameType<U, bfloat16_t>::value) {376 } else if constexpr (IsSameType<U, bfloat16_t>::value) {
377 AscendC::Reg::Cast<U, float,377 AscendC::Reg::Cast<U, float,
378 AscendAntiQuantV2Base<T, T1, T2, U, SqrtMode>::CAST_TRAIT_FP32_TO_BF16>(378 AscendAntiQuantV2Base<T, T1, T2, U, SqrtMode>::CAST_TRAIT_FP32_TO_BF16>(
379 vregY, vregFloatY, mask);379 vregY, vregFloatY, mask);
380- AscendC::Reg::DataCopy<U, AscendC::Reg::StoreDist::DIST_PACK_B32>(380+ AscendC::Reg::StoreAlign<U, AscendC::Reg::StoreDist::DIST_PACK_B32>(
381 outLocalAddr + i * VL + j * xLocalOffset, vregY, mask);381 outLocalAddr + i * VL + j * xLocalOffset, vregY, mask);
382 }382 }
383 }383 }
Mquant/ascend_anti_quant_v2/op_kernel/arch35/ascend_anti_quant_v2_per_tensor_regbase.h+11-12
@@ -189,9 +189,9 @@ __aicore__ inline void AscendAntiQuantV2PerTensorRegbase<T, T1, T2, U, SqrtMode,
189 LocalTensor<xCopyDtype> xLocal = inQueueX_.DeQue<xCopyDtype>();189 LocalTensor<xCopyDtype> xLocal = inQueueX_.DeQue<xCopyDtype>();
190 LocalTensor<U> outLocal = outQueueY_.AllocTensor<U>();190 LocalTensor<U> outLocal = outQueueY_.AllocTensor<U>();
191 191 
192- __local_mem__ xCopyDtype* xLocalAddr = (__local_mem__ xCopyDtype*)xLocal.GetPhyAddr();192+ __ubuf__ xCopyDtype* xLocalAddr = (__ubuf__ xCopyDtype*)xLocal.GetPhyAddr();
193- __local_mem__ T1* scaleLocalAddr = (__local_mem__ T1*)sLocal.GetPhyAddr();193+ __ubuf__ T1* scaleLocalAddr = (__ubuf__ T1*)sLocal.GetPhyAddr();
194- __local_mem__ U* outLocalAddr = (__local_mem__ U*)outLocal.GetPhyAddr();194+ __ubuf__ U* outLocalAddr = (__ubuf__ U*)outLocal.GetPhyAddr();
195 195 
196 uint16_t VL = AscendC::VECTOR_REG_WIDTH / sizeof(float);196 uint16_t VL = AscendC::VECTOR_REG_WIDTH / sizeof(float);
197 uint16_t HalfVL = VL / 2;197 uint16_t HalfVL = VL / 2;
@@ -222,17 +222,16 @@ __aicore__ inline void AscendAntiQuantV2PerTensorRegbase<T, T1, T2, U, SqrtMode,
222 for (uint16_t i = 0; i < vfLoopNum; i++) {222 for (uint16_t i = 0; i < vfLoopNum; i++) {
223 mask = AscendC::Reg::UpdateMask<float>(count);223 mask = AscendC::Reg::UpdateMask<float>(count);
224 // ld and cast for x224 // ld and cast for x
225- __local_mem__ xCopyDtype* xSrc = IsSameType<T, int4b_t>::value ? xLocalAddr + i * HalfVL :225+ __ubuf__ xCopyDtype* xSrc = IsSameType<T, int4b_t>::value ? xLocalAddr + i * HalfVL : xLocalAddr + i * VL;
226- xLocalAddr + i * VL;
227 this->template LoadCastXToFloat<T>(vregX, vregFloatX, xSrc, mask);226 this->template LoadCastXToFloat<T>(vregX, vregFloatX, xSrc, mask);
228 227 
229 // ld and cast for scale228 // ld and cast for scale
230 if constexpr (IsSameType<T1, float>::value) {229 if constexpr (IsSameType<T1, float>::value) {
231 // fp32230 // fp32
232- AscendC::Reg::DataCopy<float, AscendC::Reg::LoadDist::DIST_BRC_B32>(vregFloatS, scaleLocalAddr);231+ AscendC::Reg::LoadAlign<float, AscendC::Reg::LoadDist::DIST_BRC_B32>(vregFloatS, scaleLocalAddr);
233 } else if constexpr (IsSameType<T1, bfloat16_t>::value) {232 } else if constexpr (IsSameType<T1, bfloat16_t>::value) {
234 // bf16233 // bf16
235- AscendC::Reg::DataCopy<T1, AscendC::Reg::LoadDist::DIST_BRC_B16>(vregS, scaleLocalAddr);234+ AscendC::Reg::LoadAlign<T1, AscendC::Reg::LoadDist::DIST_BRC_B16>(vregS, scaleLocalAddr);
236 AscendC::Reg::Cast<float, T1, AscendAntiQuantV2Base<T, T1, T2, U, SqrtMode>::CAST_TRAIT_BF16_TO_FP32>(235 AscendC::Reg::Cast<float, T1, AscendAntiQuantV2Base<T, T1, T2, U, SqrtMode>::CAST_TRAIT_BF16_TO_FP32>(
237 vregFloatS, vregS, mask);236 vregFloatS, vregS, mask);
238 }237 }
@@ -240,11 +239,11 @@ __aicore__ inline void AscendAntiQuantV2PerTensorRegbase<T, T1, T2, U, SqrtMode,
240 // compute239 // compute
241 if constexpr (HasOffset) {240 if constexpr (HasOffset) {
242 // ld and cast for offset241 // ld and cast for offset
243- __local_mem__ T2* offsetLocalAddr = (__local_mem__ T2*)oLocal.GetPhyAddr();242+ __ubuf__ T2* offsetLocalAddr = (__ubuf__ T2*)oLocal.GetPhyAddr();
244 if constexpr (IsSameType<T2, float>::value) {243 if constexpr (IsSameType<T2, float>::value) {
245- AscendC::Reg::DataCopy<float, AscendC::Reg::LoadDist::DIST_BRC_B32>(vregFloatO, offsetLocalAddr);244+ AscendC::Reg::LoadAlign<float, AscendC::Reg::LoadDist::DIST_BRC_B32>(vregFloatO, offsetLocalAddr);
246 } else if constexpr (IsSameType<T2, bfloat16_t>::value) {245 } else if constexpr (IsSameType<T2, bfloat16_t>::value) {
247- AscendC::Reg::DataCopy<T2, AscendC::Reg::LoadDist::DIST_BRC_B16>(vregO, offsetLocalAddr);246+ AscendC::Reg::LoadAlign<T2, AscendC::Reg::LoadDist::DIST_BRC_B16>(vregO, offsetLocalAddr);
248 AscendC::Reg::Cast<float, T2,247 AscendC::Reg::Cast<float, T2,
249 AscendAntiQuantV2Base<T, T1, T2, U, SqrtMode>::CAST_TRAIT_BF16_TO_FP32>(248 AscendAntiQuantV2Base<T, T1, T2, U, SqrtMode>::CAST_TRAIT_BF16_TO_FP32>(
250 vregFloatO, vregO, mask);249 vregFloatO, vregO, mask);
@@ -265,11 +264,11 @@ __aicore__ inline void AscendAntiQuantV2PerTensorRegbase<T, T1, T2, U, SqrtMode,
265 if constexpr (IsSameType<U, half>::value) {264 if constexpr (IsSameType<U, half>::value) {
266 AscendC::Reg::Cast<half, float, AscendAntiQuantV2Base<T, T1, T2, U, SqrtMode>::CAST_TRAIT_FP32_TO_HALF>(265 AscendC::Reg::Cast<half, float, AscendAntiQuantV2Base<T, T1, T2, U, SqrtMode>::CAST_TRAIT_FP32_TO_HALF>(
267 vregY, vregFloatY, mask);266 vregY, vregFloatY, mask);
268- AscendC::Reg::DataCopy<U, AscendC::Reg::StoreDist::DIST_PACK_B32>(outLocalAddr + i * VL, vregY, mask);267+ AscendC::Reg::StoreAlign<U, AscendC::Reg::StoreDist::DIST_PACK_B32>(outLocalAddr + i * VL, vregY, mask);
269 } else if constexpr (IsSameType<U, bfloat16_t>::value) {268 } else if constexpr (IsSameType<U, bfloat16_t>::value) {
270 AscendC::Reg::Cast<U, float, AscendAntiQuantV2Base<T, T1, T2, U, SqrtMode>::CAST_TRAIT_FP32_TO_BF16>(269 AscendC::Reg::Cast<U, float, AscendAntiQuantV2Base<T, T1, T2, U, SqrtMode>::CAST_TRAIT_FP32_TO_BF16>(
271 vregY, vregFloatY, mask);270 vregY, vregFloatY, mask);
272- AscendC::Reg::DataCopy<U, AscendC::Reg::StoreDist::DIST_PACK_B32>(outLocalAddr + i * VL, vregY, mask);271+ AscendC::Reg::StoreAlign<U, AscendC::Reg::StoreDist::DIST_PACK_B32>(outLocalAddr + i * VL, vregY, mask);
273 }272 }
274 }273 }
275 }274 }
Mquant/ascend_quant/op_kernel/arch35/ascend_quant_regbase.h+16-16
@@ -126,8 +126,8 @@ __aicore__ inline void AscendQuantPerTensorRegbase<T, U, RoundMode, OffsetZero>:
126 LocalTensor<T> xLocal = inQueueX_.DeQue<T>();126 LocalTensor<T> xLocal = inQueueX_.DeQue<T>();
127 LocalTensor<yCopyDtype> outLocal = outQueueY_.AllocTensor<yCopyDtype>();127 LocalTensor<yCopyDtype> outLocal = outQueueY_.AllocTensor<yCopyDtype>();
128 128 
129- __local_mem__ T* xLocalAddr = (__local_mem__ T*)xLocal.GetPhyAddr();129+ __ubuf__ T* xLocalAddr = (__ubuf__ T*)xLocal.GetPhyAddr();
130- __local_mem__ yCopyDtype* outLocalAddr = (__local_mem__ yCopyDtype*)outLocal.GetPhyAddr();130+ __ubuf__ yCopyDtype* outLocalAddr = (__ubuf__ yCopyDtype*)outLocal.GetPhyAddr();
131 131 
132 uint16_t VL = AscendC::VECTOR_REG_WIDTH / sizeof(float);132 uint16_t VL = AscendC::VECTOR_REG_WIDTH / sizeof(float);
133 133 
@@ -158,12 +158,12 @@ __aicore__ inline void AscendQuantPerTensorRegbase<T, U, RoundMode, OffsetZero>:
158 // ld and cast for x158 // ld and cast for x
159 if constexpr (IsSameType<T, float>::value) {159 if constexpr (IsSameType<T, float>::value) {
160 // fp32160 // fp32
161- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(vregFloatX,161+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(vregFloatX,
162- xLocalAddr + i * VL);162+ xLocalAddr + i * VL);
163 } else if constexpr (IsSameType<T, half>::value) {163 } else if constexpr (IsSameType<T, half>::value) {
164 // fp16164 // fp16
165- AscendC::MicroAPI::DataCopy<half, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(vregX,165+ AscendC::MicroAPI::LoadAlign<half, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(vregX,
166- xLocalAddr + i * VL);166+ xLocalAddr + i * VL);
167 AscendC::MicroAPI::Cast<float, half, AscendQuantBase<T, U, RoundMode>::CAST_TRAIT_HALF_TO_FP32>(167 AscendC::MicroAPI::Cast<float, half, AscendQuantBase<T, U, RoundMode>::CAST_TRAIT_HALF_TO_FP32>(
168 vregFloatX, vregX, mask);168 vregFloatX, vregX, mask);
169 }169 }
@@ -180,20 +180,20 @@ __aicore__ inline void AscendQuantPerTensorRegbase<T, U, RoundMode, OffsetZero>:
180 // hifp8180 // hifp8
181 AscendC::MicroAPI::Cast<U, float, AscendQuantBase<T, U, RoundMode>::CAST_TRAIT_FP32_TO_HIFP8>(181 AscendC::MicroAPI::Cast<U, float, AscendQuantBase<T, U, RoundMode>::CAST_TRAIT_FP32_TO_HIFP8>(
182 vregY, vregFloatY, mask);182 vregY, vregFloatY, mask);
183- AscendC::MicroAPI::DataCopy<U, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(outLocalAddr + i * VL,183+ AscendC::MicroAPI::StoreAlign<U, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(outLocalAddr + i * VL,
184- vregY, mask);184+ vregY, mask);
185 } else if constexpr (IsSameType<U, fp8_e5m2_t>::value) {185 } else if constexpr (IsSameType<U, fp8_e5m2_t>::value) {
186 // fp8_e5m2186 // fp8_e5m2
187 AscendC::MicroAPI::Cast<U, float, AscendQuantBase<T, U, RoundMode>::CAST_TRAIT_FP32_TO_FP8E5M2>(187 AscendC::MicroAPI::Cast<U, float, AscendQuantBase<T, U, RoundMode>::CAST_TRAIT_FP32_TO_FP8E5M2>(
188 vregY, vregFloatY, mask);188 vregY, vregFloatY, mask);
189- AscendC::MicroAPI::DataCopy<U, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(outLocalAddr + i * VL,189+ AscendC::MicroAPI::StoreAlign<U, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(outLocalAddr + i * VL,
190- vregY, mask);190+ vregY, mask);
191 } else if constexpr (IsSameType<U, fp8_e4m3fn_t>::value) {191 } else if constexpr (IsSameType<U, fp8_e4m3fn_t>::value) {
192 // fp8_e4m3192 // fp8_e4m3
193 AscendC::MicroAPI::Cast<U, float, AscendQuantBase<T, U, RoundMode>::CAST_TRAIT_FP32_TO_FP8E4M3>(193 AscendC::MicroAPI::Cast<U, float, AscendQuantBase<T, U, RoundMode>::CAST_TRAIT_FP32_TO_FP8E4M3>(
194 vregY, vregFloatY, mask);194 vregY, vregFloatY, mask);
195- AscendC::MicroAPI::DataCopy<U, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(outLocalAddr + i * VL,195+ AscendC::MicroAPI::StoreAlign<U, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(outLocalAddr + i * VL,
196- vregY, mask);196+ vregY, mask);
197 } else if constexpr (IsSameType<U, int8_t>::value) {197 } else if constexpr (IsSameType<U, int8_t>::value) {
198 // int8198 // int8
199 AscendC::MicroAPI::Cast<int16_t, float, AscendQuantBase<T, U, RoundMode>::CAST_TRAIT_FP32_TO_INT16>(199 AscendC::MicroAPI::Cast<int16_t, float, AscendQuantBase<T, U, RoundMode>::CAST_TRAIT_FP32_TO_INT16>(
@@ -202,8 +202,8 @@ __aicore__ inline void AscendQuantPerTensorRegbase<T, U, RoundMode, OffsetZero>:
202 vregHalfY, vregInt16Y, mask);202 vregHalfY, vregInt16Y, mask);
203 AscendC::MicroAPI::Cast<int8_t, half, AscendQuantBase<T, U, RoundMode>::CAST_TRAIT_HALF_TO_INT8>(203 AscendC::MicroAPI::Cast<int8_t, half, AscendQuantBase<T, U, RoundMode>::CAST_TRAIT_HALF_TO_INT8>(
204 vregY, vregHalfY, mask);204 vregY, vregHalfY, mask);
205- AscendC::MicroAPI::DataCopy<U, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(outLocalAddr + i * VL,205+ AscendC::MicroAPI::StoreAlign<U, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(outLocalAddr + i * VL,
206- vregY, mask);206+ vregY, mask);
207 } else if constexpr (IsSameType<U, int4b_t>::value) {207 } else if constexpr (IsSameType<U, int4b_t>::value) {
208 AscendC::MicroAPI::RegTensor<int16_t> vregInt16Y;208 AscendC::MicroAPI::RegTensor<int16_t> vregInt16Y;
209 AscendC::MicroAPI::RegTensor<uint16_t> vregTmp1Y;209 AscendC::MicroAPI::RegTensor<uint16_t> vregTmp1Y;
@@ -217,7 +217,7 @@ __aicore__ inline void AscendQuantPerTensorRegbase<T, U, RoundMode, OffsetZero>:
217 AscendC::MicroAPI::Cast<int4x2_t, half, AscendQuantBase<T, U, RoundMode>::CAST_TRAIT_F16_TO_I8>(217 AscendC::MicroAPI::Cast<int4x2_t, half, AscendQuantBase<T, U, RoundMode>::CAST_TRAIT_F16_TO_I8>(
218 (AscendC::MicroAPI::RegTensor<int4x2_t>&)vregTmp2Y, (AscendC::MicroAPI::RegTensor<half>&)vregTmp1Y,218 (AscendC::MicroAPI::RegTensor<int4x2_t>&)vregTmp2Y, (AscendC::MicroAPI::RegTensor<half>&)vregTmp1Y,
219 mask);219 mask);
220- AscendC::MicroAPI::DataCopy<yCopyDtype, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(220+ AscendC::MicroAPI::StoreAlign<yCopyDtype, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(
221 outLocalAddr + (i * VL / 2), vregTmp2Y, mask4Int4);221 outLocalAddr + (i * VL / 2), vregTmp2Y, mask4Int4);
222 }222 }
223 }223 }
@@ -240,4 +240,4 @@ __aicore__ inline void AscendQuantPerTensorRegbase<T, U, RoundMode, OffsetZero>:
240 outQueueY_.FreeTensor(outLocal);240 outQueueY_.FreeTensor(outLocal);
241}241}
242} // namespace AscendQuantOp242} // namespace AscendQuantOp
243-#endif243+#endif
Mquant/dynamic_block_mx_quant/op_kernel/arch35/dynamic_block_mx_quant_base.h+27-24
@@ -554,7 +554,7 @@ __aicore__ inline void DynamicBlockMxQuantBase<T, U, roundMode, scaleAlg>::Compu
554 // 计算x0和x1的最大值，相当于计算原始相邻两个数据的最大值554 // 计算x0和x1的最大值，相当于计算原始相邻两个数据的最大值
555 Reg::Max(expMaxDim, expMax1Dim2, expMax2Dim2, maskAll);555 Reg::Max(expMaxDim, expMax1Dim2, expMax2Dim2, maskAll);
556 // ReduceMax一个block，即16个数，配合上一步，可以计算出每32*32个数的最大值，一共256/32个556 // ReduceMax一个block，即16个数，配合上一步，可以计算出每32*32个数的最大值，一共256/32个
557- Reg::ReduceMaxWithDataBlock(expMaxDim, expMaxDim, maskAll);557+ Reg::ReduceDataBlock<Reg::ReduceType::MAX>(expMaxDim, expMaxDim, maskAll);
558 558 
559 // inf/nan值单独处理，结果为E8M0的nan559 // inf/nan值单独处理，结果为E8M0的nan
560 Reg::Compare<uint16_t, CMPMODE::NE>(infMask, expMaxDim, expMaskBF16, maskAll);560 Reg::Compare<uint16_t, CMPMODE::NE>(infMask, expMaxDim, expMaskBF16, maskAll);
@@ -691,7 +691,7 @@ __aicore__ inline void DynamicBlockMxQuantBase<T, U, roundMode, scaleAlg>::Compu
691 // 计算x0和x1的最大值，相当于计算原始相邻两个数据的最大值691 // 计算x0和x1的最大值，相当于计算原始相邻两个数据的最大值
692 Reg::Max(expMaxDim, expMax1Dim2, expMax2Dim2, maskAll);692 Reg::Max(expMaxDim, expMax1Dim2, expMax2Dim2, maskAll);
693 // ReduceMax一个block，即16个数，配合上一步，可以计算出每32*32个数的最大值，一共256/32个693 // ReduceMax一个block，即16个数，配合上一步，可以计算出每32*32个数的最大值，一共256/32个
694- Reg::ReduceMaxWithDataBlock(expMaxDim, expMaxDim, maskAll);694+ Reg::ReduceDataBlock<Reg::ReduceType::MAX>(expMaxDim, expMaxDim, maskAll);
695 695 
696 Reg::And(expTmpMaxDim, expMaxDim, expMaskBF16, maskAll);696 Reg::And(expTmpMaxDim, expMaxDim, expMaskBF16, maskAll);
697 697 
@@ -773,10 +773,10 @@ __aicore__ inline void DynamicBlockMxQuantBase<T, U, roundMode, scaleAlg>::Compu
773 Reg::RegTensor<U> x0FP4;773 Reg::RegTensor<U> x0FP4;
774 Reg::RegTensor<U> x1FP4;774 Reg::RegTensor<U> x1FP4;
775 775 
776- Reg::DataCopy<uint16_t, Reg::PostLiteral::POST_MODE_UPDATE, Reg::LoadDist::DIST_E2B_B16>(776+ Reg::LoadAlign<uint16_t, Reg::PostLiteral::POST_MODE_UPDATE, Reg::LoadDist::DIST_E2B_B16>(
777 scaleForMulFP16, mxScale1ReciprocalAddr, ubBlockSize_ / sizeof(uint16_t));777 scaleForMulFP16, mxScale1ReciprocalAddr, ubBlockSize_ / sizeof(uint16_t));
778 for (uint16_t i = 0; i < blockCount; i++) {778 for (uint16_t i = 0; i < blockCount; i++) {
779- Reg::DataCopy<T, Reg::PostLiteral::POST_MODE_UPDATE, Reg::LoadDist::DIST_DINTLV_B16>(779+ Reg::LoadAlign<T, Reg::PostLiteral::POST_MODE_UPDATE, Reg::LoadDist::DIST_DINTLV_B16>(
780 x0, x1, xAddr, vlForHalfNumber_ * DIGIT_TWO);780 x0, x1, xAddr, vlForHalfNumber_ * DIGIT_TWO);
781 if constexpr (IsSameType<T, half>::value) {781 if constexpr (IsSameType<T, half>::value) {
782 Reg::Cast<float, bfloat16_t, castTraitXdtypetoFp32Zero>(782 Reg::Cast<float, bfloat16_t, castTraitXdtypetoFp32Zero>(
@@ -827,9 +827,9 @@ __aicore__ inline void DynamicBlockMxQuantBase<T, U, roundMode, scaleAlg>::Compu
827 }827 }
828 828 
829 // copy to ub829 // copy to ub
830- Reg::DataCopy<uint8_t, Reg::PostLiteral::POST_MODE_UPDATE, Reg::StoreDist::DIST_PACK4_B32>(830+ Reg::StoreAlign<uint8_t, Reg::PostLiteral::POST_MODE_UPDATE, Reg::StoreDist::DIST_PACK4_B32>(
831 y1Addr, (Reg::RegTensor<uint8_t>&)x0FP4, OUT_ELE_NUM_ONE_BLK, dataMaskB8);831 y1Addr, (Reg::RegTensor<uint8_t>&)x0FP4, OUT_ELE_NUM_ONE_BLK, dataMaskB8);
832- Reg::DataCopy<uint8_t, Reg::PostLiteral::POST_MODE_UPDATE, Reg::StoreDist::DIST_PACK4_B32>(832+ Reg::StoreAlign<uint8_t, Reg::PostLiteral::POST_MODE_UPDATE, Reg::StoreDist::DIST_PACK4_B32>(
833 y1Addr, (Reg::RegTensor<uint8_t>&)x1FP4, OUT_ELE_NUM_ONE_BLK, dataMaskB8);833 y1Addr, (Reg::RegTensor<uint8_t>&)x1FP4, OUT_ELE_NUM_ONE_BLK, dataMaskB8);
834 }834 }
835 }835 }
@@ -841,6 +841,7 @@ __aicore__ inline void DynamicBlockMxQuantBase<T, U, roundMode, scaleAlg>::Compu
841 uint16_t dataLen, uint16_t blockCount, __ubuf__ T* xAddr, __ubuf__ uint16_t* mxScale1ReciprocalAddr,841 uint16_t dataLen, uint16_t blockCount, __ubuf__ T* xAddr, __ubuf__ uint16_t* mxScale1ReciprocalAddr,
842 __ubuf__ uint8_t* y1Addr)842 __ubuf__ uint8_t* y1Addr)
843{843{
844+#ifndef ASCENDC_CPU_DEBUG
844 __VEC_SCOPE__845 __VEC_SCOPE__
845 {846 {
846 Reg::MaskReg maskAll = Reg::CreateMask<uint16_t, Reg::MaskPattern::ALL>();847 Reg::MaskReg maskAll = Reg::CreateMask<uint16_t, Reg::MaskPattern::ALL>();
@@ -859,10 +860,10 @@ __aicore__ inline void DynamicBlockMxQuantBase<T, U, roundMode, scaleAlg>::Compu
859 Reg::RegTensor<U> x1ZeroFP8;860 Reg::RegTensor<U> x1ZeroFP8;
860 Reg::RegTensor<U> x1OneFP8;861 Reg::RegTensor<U> x1OneFP8;
861 862 
862- Reg::DataCopy<uint16_t, Reg::PostLiteral::POST_MODE_UPDATE, Reg::LoadDist::DIST_E2B_B16>(863+ Reg::LoadAlign<uint16_t, Reg::PostLiteral::POST_MODE_UPDATE, Reg::LoadDist::DIST_E2B_B16>(
863 scaleForMulFP16, mxScale1ReciprocalAddr, ubBlockSize_ / sizeof(uint16_t));864 scaleForMulFP16, mxScale1ReciprocalAddr, ubBlockSize_ / sizeof(uint16_t));
864 for (uint16_t i = 0; i < blockCount; i++) {865 for (uint16_t i = 0; i < blockCount; i++) {
865- Reg::DataCopy<T, Reg::PostLiteral::POST_MODE_UPDATE, Reg::LoadDist::DIST_DINTLV_B16>(866+ Reg::LoadAlign<T, Reg::PostLiteral::POST_MODE_UPDATE, Reg::LoadDist::DIST_DINTLV_B16>(
866 x0, x1, xAddr, vlForHalfNumber_ * DIGIT_TWO);867 x0, x1, xAddr, vlForHalfNumber_ * DIGIT_TWO);
867 if constexpr (IsSameType<T, half>::value) {868 if constexpr (IsSameType<T, half>::value) {
868 Reg::Cast<float, T, castTraitXdtypetoFp32Zero>(x0ZeroFP32, x0, maskAll);869 Reg::Cast<float, T, castTraitXdtypetoFp32Zero>(x0ZeroFP32, x0, maskAll);
@@ -898,16 +899,17 @@ __aicore__ inline void DynamicBlockMxQuantBase<T, U, roundMode, scaleAlg>::Compu
898 Reg::Cast<U, float, castTraitFp32toYdtype>(x1ZeroFP8, x1ZeroFP32, maskAll);899 Reg::Cast<U, float, castTraitFp32toYdtype>(x1ZeroFP8, x1ZeroFP32, maskAll);
899 Reg::Cast<U, float, castTraitFp32toYdtype>(x1OneFP8, x1OneFP32, maskAll);900 Reg::Cast<U, float, castTraitFp32toYdtype>(x1OneFP8, x1OneFP32, maskAll);
900 }901 }
901- Reg::DataCopy<uint8_t, Reg::PostLiteral::POST_MODE_UPDATE, Reg::StoreDist::DIST_PACK4_B32>(902+ Reg::StoreAlign<uint8_t, Reg::PostLiteral::POST_MODE_UPDATE, Reg::StoreDist::DIST_PACK4_B32>(
902 y1Addr, (Reg::RegTensor<uint8_t>&)x0ZeroFP8, OUT_ELE_NUM_ONE_BLK, maskAll);903 y1Addr, (Reg::RegTensor<uint8_t>&)x0ZeroFP8, OUT_ELE_NUM_ONE_BLK, maskAll);
903- Reg::DataCopy<uint8_t, Reg::PostLiteral::POST_MODE_UPDATE, Reg::StoreDist::DIST_PACK4_B32>(904+ Reg::StoreAlign<uint8_t, Reg::PostLiteral::POST_MODE_UPDATE, Reg::StoreDist::DIST_PACK4_B32>(
904 y1Addr, (Reg::RegTensor<uint8_t>&)x0OneFP8, OUT_ELE_NUM_ONE_BLK, maskAll);905 y1Addr, (Reg::RegTensor<uint8_t>&)x0OneFP8, OUT_ELE_NUM_ONE_BLK, maskAll);
905- Reg::DataCopy<uint8_t, Reg::PostLiteral::POST_MODE_UPDATE, Reg::StoreDist::DIST_PACK4_B32>(906+ Reg::StoreAlign<uint8_t, Reg::PostLiteral::POST_MODE_UPDATE, Reg::StoreDist::DIST_PACK4_B32>(
906 y1Addr, (Reg::RegTensor<uint8_t>&)x1ZeroFP8, OUT_ELE_NUM_ONE_BLK, maskAll);907 y1Addr, (Reg::RegTensor<uint8_t>&)x1ZeroFP8, OUT_ELE_NUM_ONE_BLK, maskAll);
907- Reg::DataCopy<uint8_t, Reg::PostLiteral::POST_MODE_UPDATE, Reg::StoreDist::DIST_PACK4_B32>(908+ Reg::StoreAlign<uint8_t, Reg::PostLiteral::POST_MODE_UPDATE, Reg::StoreDist::DIST_PACK4_B32>(
908 y1Addr, (Reg::RegTensor<uint8_t>&)x1OneFP8, OUT_ELE_NUM_ONE_BLK, maskAll);909 y1Addr, (Reg::RegTensor<uint8_t>&)x1OneFP8, OUT_ELE_NUM_ONE_BLK, maskAll);
909 }910 }
910 }911 }
912+#endif // ASCENDC_CPU_DEBUG
911 return;913 return;
912}914}
913 915 
@@ -930,7 +932,7 @@ __aicore__ inline void DynamicBlockMxQuantBase<T, U, roundMode, scaleAlg>::Compu
930 Reg::Compare<int32_t, CMPMODE::EQ>(negInfMask, (Reg::RegTensor<int32_t>&)Reg, negZero, pregAll32);932 Reg::Compare<int32_t, CMPMODE::EQ>(negInfMask, (Reg::RegTensor<int32_t>&)Reg, negZero, pregAll32);
931 if constexpr (IsSameType<U, fp4x2_e1m2_t>::value) {933 if constexpr (IsSameType<U, fp4x2_e1m2_t>::value) {
932 Reg::Muls(Reg, Reg, FOUR, pregAll32);934 Reg::Muls(Reg, Reg, FOUR, pregAll32);
933- Reg::CompareScalar<float, CMPMODE::LT>(specialMask, Reg, 0, pregAll32);935+ Reg::Compares<float, CMPMODE::LT>(specialMask, Reg, 0, pregAll32);
934 Reg::Truncate<float, roundMode>(Reg, Reg, pregAll32);936 Reg::Truncate<float, roundMode>(Reg, Reg, pregAll32);
935 Reg::Muls(Reg, Reg, ONE_FOURTH, pregAll32);937 Reg::Muls(Reg, Reg, ONE_FOURTH, pregAll32);
936 } else {938 } else {
@@ -948,13 +950,13 @@ __aicore__ inline void DynamicBlockMxQuantBase<T, U, roundMode, scaleAlg>::Compu
948 Reg::Mul(Reg, Reg, (Reg::RegTensor<float>&)exp1FP32, pregAll32);950 Reg::Mul(Reg, Reg, (Reg::RegTensor<float>&)exp1FP32, pregAll32);
949 Reg::Adds(exp0FP32, exp0FP32, FP32_BIAS, pregAll32);951 Reg::Adds(exp0FP32, exp0FP32, FP32_BIAS, pregAll32);
950 Reg::ShiftLefts(exp0FP32, exp0FP32, SHR_NUM_FOR_FP32, pregAll32);952 Reg::ShiftLefts(exp0FP32, exp0FP32, SHR_NUM_FOR_FP32, pregAll32);
951- Reg::CompareScalar<float, CMPMODE::LT>(specialMask, Reg, 0, pregAll32);953+ Reg::Compares<float, CMPMODE::LT>(specialMask, Reg, 0, pregAll32);
952 Reg::Truncate<float, roundMode>(Reg, Reg, pregAll32);954 Reg::Truncate<float, roundMode>(Reg, Reg, pregAll32);
953 Reg::Mul(Reg, Reg, (Reg::RegTensor<float>&)exp0FP32, pregAll32);955 Reg::Mul(Reg, Reg, (Reg::RegTensor<float>&)exp0FP32, pregAll32);
954 }956 }
955- Reg::CompareScalar<float, CMPMODE::EQ>(zeroMask, Reg, 0, pregAll32);957+ Reg::Compares<float, CMPMODE::EQ>(zeroMask, Reg, 0, pregAll32);
956- Reg::MaskAnd(zeroMask, specialMask, zeroMask, pregAll32);958+ Reg::And(zeroMask, specialMask, zeroMask, pregAll32);
957- Reg::MaskOr(zeroMask, negInfMask, zeroMask, pregAll32);959+ Reg::Or(zeroMask, negInfMask, zeroMask, pregAll32);
958 Reg::Select<int32_t>((Reg::RegTensor<int32_t>&)Reg, negZero, (Reg::RegTensor<int32_t>&)Reg, zeroMask);960 Reg::Select<int32_t>((Reg::RegTensor<int32_t>&)Reg, negZero, (Reg::RegTensor<int32_t>&)Reg, zeroMask);
959}961}
960 962 
@@ -965,22 +967,22 @@ __aicore__ inline void DynamicBlockMxQuantBase<T, U, roundMode, scaleAlg>::Compu
965 int64_t nowRowBlock)967 int64_t nowRowBlock)
966{968{
967 LocalTensor<T> inLocal = inQueue_.DeQue<T>();969 LocalTensor<T> inLocal = inQueue_.DeQue<T>();
968- __local_mem__ T* xLocalAddr = (__local_mem__ T*)inLocal.GetPhyAddr();970+ __ubuf__ T* xLocalAddr = (__ubuf__ T*)inLocal.GetPhyAddr();
969 971 
970 LocalTensor<uint8_t> scale1Local = scale1Queue_.AllocTensor<uint8_t>();972 LocalTensor<uint8_t> scale1Local = scale1Queue_.AllocTensor<uint8_t>();
971- __local_mem__ uint8_t* scale1LocalAddr = (__local_mem__ uint8_t*)scale1Local.GetPhyAddr();973+ __ubuf__ uint8_t* scale1LocalAddr = (__ubuf__ uint8_t*)scale1Local.GetPhyAddr();
972 974 
973 LocalTensor<uint8_t> scale2Local = scale2Queue_.AllocTensor<uint8_t>();975 LocalTensor<uint8_t> scale2Local = scale2Queue_.AllocTensor<uint8_t>();
974- __local_mem__ uint8_t* scale2LocalAddr = (__local_mem__ uint8_t*)scale2Local.GetPhyAddr();976+ __ubuf__ uint8_t* scale2LocalAddr = (__ubuf__ uint8_t*)scale2Local.GetPhyAddr();
975 977 
976 LocalTensor<uint8_t> outLocal = outQueue_.AllocTensor<uint8_t>();978 LocalTensor<uint8_t> outLocal = outQueue_.AllocTensor<uint8_t>();
977- __local_mem__ uint8_t* outLocalAddr = (__local_mem__ uint8_t*)outLocal.GetPhyAddr();979+ __ubuf__ uint8_t* outLocalAddr = (__ubuf__ uint8_t*)outLocal.GetPhyAddr();
978 980 
979 LocalTensor<uint16_t> mxScaleReciprocalLocal = mxScaleReciprocalBuf_.Get<uint16_t>();981 LocalTensor<uint16_t> mxScaleReciprocalLocal = mxScaleReciprocalBuf_.Get<uint16_t>();
980- auto mxScaleReciprocalAddr = (__local_mem__ uint16_t*)mxScaleReciprocalLocal.GetPhyAddr();982+ auto mxScaleReciprocalAddr = (__ubuf__ uint16_t*)mxScaleReciprocalLocal.GetPhyAddr();
981 983 
982 LocalTensor<uint8_t> tempIndexLocal = tempIndexBuf_.Get<uint8_t>();984 LocalTensor<uint8_t> tempIndexLocal = tempIndexBuf_.Get<uint8_t>();
983- auto tempIndexLocalAddr = (__local_mem__ uint8_t*)tempIndexLocal.GetPhyAddr();985+ auto tempIndexLocalAddr = (__ubuf__ uint8_t*)tempIndexLocal.GetPhyAddr();
984 986 
985 int64_t ubOffset = 0;987 int64_t ubOffset = 0;
986 int64_t yOffset = 0;988 int64_t yOffset = 0;
@@ -1166,4 +1168,5 @@ __aicore__ inline void DynamicBlockMxQuantBase<T, U, roundMode, scaleAlg>::CopyO
1166}1168}
1167 1169 
1168} // namespace DynamicBlockMxQuant1170} // namespace DynamicBlockMxQuant
1169-#endif // DYNAMIC_BLOCK_MX_QUANT_BASE_H1171+ 
1172+#endif // DYNAMIC_BLOCK_MX_QUANT_BASE_H
Mquant/dynamic_block_quant/op_kernel/arch35/dynamic_block_quant_large_blocksize_kernel.h+35-35
@@ -36,11 +36,11 @@ private:
36 __aicore__ inline void CopyOutY(int64_t rowNum, int64_t colNum, int64_t baseXGmOffset);36 __aicore__ inline void CopyOutY(int64_t rowNum, int64_t colNum, int64_t baseXGmOffset);
37 __aicore__ inline void CopyOutScale(int64_t baseScaleOffset);37 __aicore__ inline void CopyOutScale(int64_t baseScaleOffset);
38 __aicore__ inline void ParseTilingData(const DynamicBlockQuantTilingData& tilingData);38 __aicore__ inline void ParseTilingData(const DynamicBlockQuantTilingData& tilingData);
39- __aicore__ inline void ComputeXTmpMax(int64_t rowNum, int64_t blockColNum, __local_mem__ T* xLocalAddr,39+ __aicore__ inline void ComputeXTmpMax(int64_t rowNum, int64_t blockColNum, __ubuf__ T* xLocalAddr,
40- __local_mem__ T* xLocalMaxTmp);40+ __ubuf__ T* xLocalMaxTmp);
41- __aicore__ inline void ComputeScaleVF(__local_mem__ float* scaleLocalTmp, __local_mem__ T* xLocalMaxTmp);41+ __aicore__ inline void ComputeScaleVF(__ubuf__ float* scaleLocalTmp, __ubuf__ T* xLocalMaxTmp);
42- __aicore__ inline void ComputeOutVF(int64_t rowNum, int64_t colNum, __local_mem__ T* xLocalAddr,42+ __aicore__ inline void ComputeOutVF(int64_t rowNum, int64_t colNum, __ubuf__ T* xLocalAddr,
43- __local_mem__ float* scaleLocal, __local_mem__ U* outLocal);43+ __ubuf__ float* scaleLocal, __ubuf__ U* outLocal);
44 __aicore__ inline void InitBuffer();44 __aicore__ inline void InitBuffer();
45 __aicore__ inline void InitGmOffset(GM_ADDR x, GM_ADDR y, GM_ADDR scale);45 __aicore__ inline void InitGmOffset(GM_ADDR x, GM_ADDR y, GM_ADDR scale);
46 __aicore__ inline void ProcessBlock(int64_t blockRowNum, int64_t blockColNum, int64_t baseXGmOffset,46 __aicore__ inline void ProcessBlock(int64_t blockRowNum, int64_t blockColNum, int64_t baseXGmOffset,
@@ -327,9 +327,9 @@ __aicore__ inline void DynamicBlockQuantLargeBlockSize<T, U, RMode>::ProcessBloc
327 int64_t singleLoopHandleRowNum;327 int64_t singleLoopHandleRowNum;
328 LocalTensor<T> xLocalMaxTmp = xLocalMaxBuffer_.Get<T>();328 LocalTensor<T> xLocalMaxTmp = xLocalMaxBuffer_.Get<T>();
329 AscendC::Duplicate(xLocalMaxTmp, static_cast<T>(0), xLocalMaxTmp.GetSize());329 AscendC::Duplicate(xLocalMaxTmp, static_cast<T>(0), xLocalMaxTmp.GetSize());
330- __local_mem__ T* xLocalMaxTmpAddr = (__local_mem__ T*)xLocalMaxTmp.GetPhyAddr();330+ __ubuf__ T* xLocalMaxTmpAddr = (__ubuf__ T*)xLocalMaxTmp.GetPhyAddr();
331 LocalTensor<T> inLocal;331 LocalTensor<T> inLocal;
332- __local_mem__ T* xLocalAddr;332+ __ubuf__ T* xLocalAddr;
333 int64_t nowXGmOffset;333 int64_t nowXGmOffset;
334 334 
335 // 分段搬入，计算max(input)335 // 分段搬入，计算max(input)
@@ -340,7 +340,7 @@ __aicore__ inline void DynamicBlockQuantLargeBlockSize<T, U, RMode>::ProcessBloc
340 nowXGmOffset = baseXGmOffset + rowLoopIdx * singleLoopHandleRow * colNum_;340 nowXGmOffset = baseXGmOffset + rowLoopIdx * singleLoopHandleRow * colNum_;
341 CopyIn(singleLoopHandleRowNum, blockColNum, nowXGmOffset);341 CopyIn(singleLoopHandleRowNum, blockColNum, nowXGmOffset);
342 inLocal = inQueue_.DeQue<T>();342 inLocal = inQueue_.DeQue<T>();
343- xLocalAddr = (__local_mem__ T*)inLocal.GetPhyAddr();343+ xLocalAddr = (__ubuf__ T*)inLocal.GetPhyAddr();
344 ComputeXTmpMax(singleLoopHandleRowNum, blockColNum, xLocalAddr, xLocalMaxTmpAddr);344 ComputeXTmpMax(singleLoopHandleRowNum, blockColNum, xLocalAddr, xLocalMaxTmpAddr);
345 if (rowLoopIdx != 0) {345 if (rowLoopIdx != 0) {
346 inQueue_.FreeTensor(inLocal);346 inQueue_.FreeTensor(inLocal);
@@ -360,11 +360,11 @@ __aicore__ inline void DynamicBlockQuantLargeBlockSize<T, U, RMode>::ProcessBloc
360 }360 }
361 361 
362 LocalTensor<float> scaleLocal = scaleQueue_.AllocTensor<float>();362 LocalTensor<float> scaleLocal = scaleQueue_.AllocTensor<float>();
363- __local_mem__ float* scaleLocalAddr = (__local_mem__ float*)scaleLocal.GetPhyAddr();363+ __ubuf__ float* scaleLocalAddr = (__ubuf__ float*)scaleLocal.GetPhyAddr();
364 ComputeScaleVF(scaleLocalAddr, xLocalMaxTmpAddr);364 ComputeScaleVF(scaleLocalAddr, xLocalMaxTmpAddr);
365 365 
366 LocalTensor<U> outLocal = outQueue_.AllocTensor<U>();366 LocalTensor<U> outLocal = outQueue_.AllocTensor<U>();
367- __local_mem__ U* outLocalAddr = (__local_mem__ U*)outLocal.GetPhyAddr();367+ __ubuf__ U* outLocalAddr = (__ubuf__ U*)outLocal.GetPhyAddr();
368 368 
369 // 分段计算Y369 // 分段计算Y
370 ComputeOutVF(singleLoopHandleRowNum, blockColNum, xLocalAddr, scaleLocalAddr, outLocalAddr);370 ComputeOutVF(singleLoopHandleRowNum, blockColNum, xLocalAddr, scaleLocalAddr, outLocalAddr);
@@ -380,10 +380,10 @@ __aicore__ inline void DynamicBlockQuantLargeBlockSize<T, U, RMode>::ProcessBloc
380 CopyIn(singleLoopHandleRowNum, blockColNum, nowXGmOffset);380 CopyIn(singleLoopHandleRowNum, blockColNum, nowXGmOffset);
381 381 
382 inLocal = inQueue_.DeQue<T>();382 inLocal = inQueue_.DeQue<T>();
383- xLocalAddr = (__local_mem__ T*)inLocal.GetPhyAddr();383+ xLocalAddr = (__ubuf__ T*)inLocal.GetPhyAddr();
384 384 
385 outLocal = outQueue_.AllocTensor<U>();385 outLocal = outQueue_.AllocTensor<U>();
386- outLocalAddr = (__local_mem__ U*)outLocal.GetPhyAddr();386+ outLocalAddr = (__ubuf__ U*)outLocal.GetPhyAddr();
387 ComputeOutVF(singleLoopHandleRowNum, blockColNum, xLocalAddr, scaleLocalAddr, outLocalAddr);387 ComputeOutVF(singleLoopHandleRowNum, blockColNum, xLocalAddr, scaleLocalAddr, outLocalAddr);
388 388 
389 inQueue_.FreeTensor(inLocal);389 inQueue_.FreeTensor(inLocal);
@@ -409,8 +409,8 @@ __aicore__ inline void DynamicBlockQuantLargeBlockSize<T, U, RMode>::CopyIn(int6
409 409 
410template <typename T, typename U, int64_t RMode>410template <typename T, typename U, int64_t RMode>
411__aicore__ inline void DynamicBlockQuantLargeBlockSize<T, U, RMode>::ComputeXTmpMax(int64_t rowNum, int64_t blockColNum,411__aicore__ inline void DynamicBlockQuantLargeBlockSize<T, U, RMode>::ComputeXTmpMax(int64_t rowNum, int64_t blockColNum,
412- __local_mem__ T* xLocalAddr,412+ __ubuf__ T* xLocalAddr,
413- __local_mem__ T* xLocalMaxTmp)413+ __ubuf__ T* xLocalMaxTmp)
414{414{
415 uint32_t xTotalNum = rowNum * blockColNum;415 uint32_t xTotalNum = rowNum * blockColNum;
416 uint32_t dtypeSize = sizeof(T);416 uint32_t dtypeSize = sizeof(T);
@@ -426,7 +426,7 @@ __aicore__ inline void DynamicBlockQuantLargeBlockSize<T, U, RMode>::ComputeXTmp
426 AscendC::MicroAPI::MaskReg preg0;426 AscendC::MicroAPI::MaskReg preg0;
427 AscendC::MicroAPI::MaskReg maskAll;427 AscendC::MicroAPI::MaskReg maskAll;
428 428 
429- AscendC::MicroAPI::DataCopy(vLocalTmpMaxReg, xLocalMaxTmp);429+ AscendC::MicroAPI::LoadAlign(vLocalTmpMaxReg, xLocalMaxTmp);
430 430 
431 if constexpr (IsSameType<T, float>::value) {431 if constexpr (IsSameType<T, float>::value) {
432 // ===== float32 类型处理 =====432 // ===== float32 类型处理 =====
@@ -435,7 +435,7 @@ __aicore__ inline void DynamicBlockQuantLargeBlockSize<T, U, RMode>::ComputeXTmp
435 435 
436 for (uint16_t i = 0; i < vfLoop; i++) {436 for (uint16_t i = 0; i < vfLoop; i++) {
437 preg0 = AscendC::MicroAPI::UpdateMask<T>(xTotalNum);437 preg0 = AscendC::MicroAPI::UpdateMask<T>(xTotalNum);
438- AscendC::MicroAPI::DataCopy(vreg1, xLocalAddr + i * VL);438+ AscendC::MicroAPI::LoadAlign(vreg1, xLocalAddr + i * VL);
439 AscendC::MicroAPI::And((AscendC::MicroAPI::RegTensor<uint32_t>&)vreg3,439 AscendC::MicroAPI::And((AscendC::MicroAPI::RegTensor<uint32_t>&)vreg3,
440 (AscendC::MicroAPI::RegTensor<uint32_t>&)vreg1,440 (AscendC::MicroAPI::RegTensor<uint32_t>&)vreg1,
441 (AscendC::MicroAPI::RegTensor<uint32_t>&)vreg2, preg0);441 (AscendC::MicroAPI::RegTensor<uint32_t>&)vreg2, preg0);
@@ -451,7 +451,7 @@ __aicore__ inline void DynamicBlockQuantLargeBlockSize<T, U, RMode>::ComputeXTmp
451 451 
452 for (uint16_t i = 0; i < vfLoop; i++) {452 for (uint16_t i = 0; i < vfLoop; i++) {
453 preg0 = AscendC::MicroAPI::UpdateMask<T>(xTotalNum);453 preg0 = AscendC::MicroAPI::UpdateMask<T>(xTotalNum);
454- AscendC::MicroAPI::DataCopy(vreg1, xLocalAddr + i * VL);454+ AscendC::MicroAPI::LoadAlign(vreg1, xLocalAddr + i * VL);
455 AscendC::MicroAPI::And((AscendC::MicroAPI::RegTensor<uint16_t>&)vreg3,455 AscendC::MicroAPI::And((AscendC::MicroAPI::RegTensor<uint16_t>&)vreg3,
456 (AscendC::MicroAPI::RegTensor<uint16_t>&)vreg1,456 (AscendC::MicroAPI::RegTensor<uint16_t>&)vreg1,
457 (AscendC::MicroAPI::RegTensor<uint16_t>&)vreg2, preg0);457 (AscendC::MicroAPI::RegTensor<uint16_t>&)vreg2, preg0);
@@ -459,13 +459,13 @@ __aicore__ inline void DynamicBlockQuantLargeBlockSize<T, U, RMode>::ComputeXTmp
459 vreg3, preg0);459 vreg3, preg0);
460 }460 }
461 }461 }
462- AscendC::MicroAPI::DataCopy(xLocalMaxTmp, vLocalTmpMaxReg, maskAll);462+ AscendC::MicroAPI::StoreAlign(xLocalMaxTmp, vLocalTmpMaxReg, maskAll);
463 }463 }
464}464}
465 465 
466template <typename T, typename U, int64_t RMode>466template <typename T, typename U, int64_t RMode>
467-__aicore__ inline void DynamicBlockQuantLargeBlockSize<T, U, RMode>::ComputeScaleVF(__local_mem__ float* scaleLocalTmp,467+__aicore__ inline void DynamicBlockQuantLargeBlockSize<T, U, RMode>::ComputeScaleVF(__ubuf__ float* scaleLocalTmp,
468- __local_mem__ T* xLocalMaxTmp)468+ __ubuf__ T* xLocalMaxTmp)
469{469{
470 static constexpr AscendC::MicroAPI::DivSpecificMode mode = {AscendC::MicroAPI::MaskMergeMode::ZEROING, false};470 static constexpr AscendC::MicroAPI::DivSpecificMode mode = {AscendC::MicroAPI::MaskMergeMode::ZEROING, false};
471 uint32_t scaleNum = 1;471 uint32_t scaleNum = 1;
@@ -491,32 +491,32 @@ __aicore__ inline void DynamicBlockQuantLargeBlockSize<T, U, RMode>::ComputeScal
491 491 
492 if constexpr (IsSameType<T, float>::value) {492 if constexpr (IsSameType<T, float>::value) {
493 // float32: 位模式本身就是 float，不需要 Cast 和 UNPACK493 // float32: 位模式本身就是 float，不需要 Cast 和 UNPACK
494- AscendC::MicroAPI::DataCopy(vreg1, xLocalMaxTmp);494+ AscendC::MicroAPI::LoadAlign(vreg1, xLocalMaxTmp);
495 // 位模式直接参与计算495 // 位模式直接参与计算
496 AscendC::MicroAPI::Div<float, &mode>(vreg5, (AscendC::MicroAPI::RegTensor<float>&)vreg1, vreg3, preg0);496 AscendC::MicroAPI::Div<float, &mode>(vreg5, (AscendC::MicroAPI::RegTensor<float>&)vreg1, vreg3, preg0);
497 } else {497 } else {
498 // FP16/BF16: 需要 UNPACK 和 Cast498 // FP16/BF16: 需要 UNPACK 和 Cast
499- AscendC::MicroAPI::DataCopy<T, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(vreg1, xLocalMaxTmp);499+ AscendC::MicroAPI::LoadAlign<T, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(vreg1, xLocalMaxTmp);
500 AscendC::MicroAPI::Cast<float, T, castTrait0>(vreg2, vreg1, preg0);500 AscendC::MicroAPI::Cast<float, T, castTrait0>(vreg2, vreg1, preg0);
501 AscendC::MicroAPI::Div<float, &mode>(vreg5, vreg2, vreg3, preg0);501 AscendC::MicroAPI::Div<float, &mode>(vreg5, vreg2, vreg3, preg0);
502 }502 }
503 503 
504- AscendC::MicroAPI::CompareScalar<uint32_t, CMPMODE::LT>(504+ AscendC::MicroAPI::Compares<uint32_t, CMPMODE::LT>(scaleMaskReg, (AscendC::MicroAPI::RegTensor<uint32_t>&)vreg5,
505- scaleMaskReg, (AscendC::MicroAPI::RegTensor<uint32_t>&)vreg5, infValue_, preg0);505+ infValue_, preg0);
506 // Min(input_max / FP_MAX, 1 / minScale)506 // Min(input_max / FP_MAX, 1 / minScale)
507 AscendC::MicroAPI::Min<float, AscendC::MicroAPI::MaskMergeMode::MERGING>(vreg5, vreg5, reciprocalScale,507 AscendC::MicroAPI::Min<float, AscendC::MicroAPI::MaskMergeMode::MERGING>(vreg5, vreg5, reciprocalScale,
508 scaleMaskReg);508 scaleMaskReg);
509 509 
510- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::StoreDist::DIST_FIRST_ELEMENT_B32>(scaleLocalTmp, vreg5,510+ AscendC::MicroAPI::StoreAlign<float, AscendC::MicroAPI::StoreDist::DIST_FIRST_ELEMENT_B32>(scaleLocalTmp, vreg5,
511- preg0);511+ preg0);
512 }512 }
513}513}
514 514 
515template <typename T, typename U, int64_t RMode>515template <typename T, typename U, int64_t RMode>
516__aicore__ inline void DynamicBlockQuantLargeBlockSize<T, U, RMode>::ComputeOutVF(int64_t rowNum, int64_t colNum,516__aicore__ inline void DynamicBlockQuantLargeBlockSize<T, U, RMode>::ComputeOutVF(int64_t rowNum, int64_t colNum,
517- __local_mem__ T* xLocalAddr,517+ __ubuf__ T* xLocalAddr,
518- __local_mem__ float* scaleLocal,518+ __ubuf__ float* scaleLocal,
519- __local_mem__ U* outLocal)519+ __ubuf__ U* outLocal)
520{520{
521 uint32_t xTotalNum = rowNum * colNum;521 uint32_t xTotalNum = rowNum * colNum;
522 uint32_t dtypeSize = sizeof(float);522 uint32_t dtypeSize = sizeof(float);
@@ -538,12 +538,12 @@ __aicore__ inline void DynamicBlockQuantLargeBlockSize<T, U, RMode>::ComputeOutV
538 538 
539 preg0 = AscendC::MicroAPI::CreateMask<float, MicroAPI::MaskPattern::ALL>();539 preg0 = AscendC::MicroAPI::CreateMask<float, MicroAPI::MaskPattern::ALL>();
540 540 
541- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_BRC_B32>(vreg2, scaleLocal);541+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_BRC_B32>(vreg2, scaleLocal);
542 542 
543 if constexpr (IsSameType<T, float>::value) {543 if constexpr (IsSameType<T, float>::value) {
544 for (uint16_t i = 0; i < static_cast<uint16_t>(vfLoop); i++) {544 for (uint16_t i = 0; i < static_cast<uint16_t>(vfLoop); i++) {
545 preg0 = AscendC::MicroAPI::UpdateMask<float>(xTotalNum);545 preg0 = AscendC::MicroAPI::UpdateMask<float>(xTotalNum);
546- AscendC::MicroAPI::DataCopy(vreg1, xLocalAddr + i * VL);546+ AscendC::MicroAPI::LoadAlign(vreg1, xLocalAddr + i * VL);
547 AscendC::MicroAPI::Div<float, &mode>(vreg5, vreg1, vreg2, preg0);547 AscendC::MicroAPI::Div<float, &mode>(vreg5, vreg1, vreg2, preg0);
548 548 
549 if constexpr (IsSameType<U, hifloat8_t>::value) {549 if constexpr (IsSameType<U, hifloat8_t>::value) {
@@ -555,13 +555,13 @@ __aicore__ inline void DynamicBlockQuantLargeBlockSize<T, U, RMode>::ComputeOutV
555 } else {555 } else {
556 AscendC::MicroAPI::Cast<U, float, castTrait32tofp8>(outReg, vreg5, preg0);556 AscendC::MicroAPI::Cast<U, float, castTrait32tofp8>(outReg, vreg5, preg0);
557 }557 }
558- MicroAPI::DataCopy<U, MicroAPI::StoreDist::DIST_PACK4_B32>(outLocal + i * VL, outReg, preg0);558+ MicroAPI::StoreAlign<U, MicroAPI::StoreDist::DIST_PACK4_B32>(outLocal + i * VL, outReg, preg0);
559 }559 }
560 } else {560 } else {
561 for (uint16_t i = 0; i < static_cast<uint16_t>(vfLoop); i++) {561 for (uint16_t i = 0; i < static_cast<uint16_t>(vfLoop); i++) {
562 preg0 = AscendC::MicroAPI::UpdateMask<float>(xTotalNum);562 preg0 = AscendC::MicroAPI::UpdateMask<float>(xTotalNum);
563- AscendC::MicroAPI::DataCopy<T, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(vreg1,563+ AscendC::MicroAPI::LoadAlign<T, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(vreg1,
564- xLocalAddr + i * VL);564+ xLocalAddr + i * VL);
565 AscendC::MicroAPI::Cast<float, T, castTrait0>(vreg4, vreg1, preg0);565 AscendC::MicroAPI::Cast<float, T, castTrait0>(vreg4, vreg1, preg0);
566 AscendC::MicroAPI::Div<float, &mode>(vreg5, vreg4, vreg2, preg0);566 AscendC::MicroAPI::Div<float, &mode>(vreg5, vreg4, vreg2, preg0);
567 567 
@@ -574,7 +574,7 @@ __aicore__ inline void DynamicBlockQuantLargeBlockSize<T, U, RMode>::ComputeOutV
574 } else {574 } else {
575 AscendC::MicroAPI::Cast<U, float, castTrait32tofp8>(outReg, vreg5, preg0);575 AscendC::MicroAPI::Cast<U, float, castTrait32tofp8>(outReg, vreg5, preg0);
576 }576 }
577- MicroAPI::DataCopy<U, MicroAPI::StoreDist::DIST_PACK4_B32>(outLocal + i * VL, outReg, preg0);577+ MicroAPI::StoreAlign<U, MicroAPI::StoreDist::DIST_PACK4_B32>(outLocal + i * VL, outReg, preg0);
578 }578 }
579 }579 }
580 }580 }
Mquant/dynamic_block_quant/op_kernel/arch35/dynamic_block_quant_single_row_kernel.h+52-50
@@ -40,9 +40,8 @@ private:
40 __aicore__ inline void Compute(uint64_t colSize, uint64_t rowBlockSize, uint64_t colBlockSize);40 __aicore__ inline void Compute(uint64_t colSize, uint64_t rowBlockSize, uint64_t colBlockSize);
41 __aicore__ inline void CopyOut(uint64_t offset, uint64_t rowSize, uint64_t colSize);41 __aicore__ inline void CopyOut(uint64_t offset, uint64_t rowSize, uint64_t colSize);
42 __aicore__ inline void CopyOutScale(uint64_t offset, uint64_t rowSize, uint64_t colSize);42 __aicore__ inline void CopyOutScale(uint64_t offset, uint64_t rowSize, uint64_t colSize);
43- __aicore__ inline void ComputeVF(__local_mem__ OUT_TYPE* outLocal, __local_mem__ float* scaleLocal,43+ __aicore__ inline void ComputeVF(__ubuf__ OUT_TYPE* outLocal, __ubuf__ float* scaleLocal, __ubuf__ IN_TYPE* xLocal,
44- __local_mem__ IN_TYPE* xLocal, uint64_t colSize, uint64_t rowBlockSize,44+ uint64_t colSize, uint64_t rowBlockSize, uint64_t colBlockSize);
45- uint64_t colBlockSize);
46 45 
47private:46private:
48 TPipe* tPipe_ = nullptr;47 TPipe* tPipe_ = nullptr;
@@ -222,9 +221,9 @@ inline __aicore__ void DynamicBlockQuantSingleRow<IN_TYPE, OUT_TYPE, ROUND_MODE>
222 LocalTensor<OUT_TYPE> yLocal = outQueue_.AllocTensor<OUT_TYPE>();221 LocalTensor<OUT_TYPE> yLocal = outQueue_.AllocTensor<OUT_TYPE>();
223 LocalTensor<float> scaleLocal = scaleQueue_.AllocTensor<float>();222 LocalTensor<float> scaleLocal = scaleQueue_.AllocTensor<float>();
224 223 
225- __local_mem__ IN_TYPE* xLocalPtr = (__local_mem__ IN_TYPE*)xLocal.GetPhyAddr();224+ __ubuf__ IN_TYPE* xLocalPtr = (__ubuf__ IN_TYPE*)xLocal.GetPhyAddr();
226- __local_mem__ OUT_TYPE* yLocalPtr = (__local_mem__ OUT_TYPE*)yLocal.GetPhyAddr();225+ __ubuf__ OUT_TYPE* yLocalPtr = (__ubuf__ OUT_TYPE*)yLocal.GetPhyAddr();
227- __local_mem__ float* scaleLocalPtr = (__local_mem__ float*)scaleLocal.GetPhyAddr();226+ __ubuf__ float* scaleLocalPtr = (__ubuf__ float*)scaleLocal.GetPhyAddr();
228 227 
229 ComputeVF(yLocalPtr, scaleLocalPtr, xLocalPtr, colSize, rowBlockSize, colBlockSize);228 ComputeVF(yLocalPtr, scaleLocalPtr, xLocalPtr, colSize, rowBlockSize, colBlockSize);
230 229 
@@ -263,7 +262,7 @@ inline __aicore__ void DynamicBlockQuantSingleRow<IN_TYPE, OUT_TYPE, ROUND_MODE>
263 262 
264template <typename IN_TYPE, typename OUT_TYPE, int64_t ROUND_MODE>263template <typename IN_TYPE, typename OUT_TYPE, int64_t ROUND_MODE>
265inline __aicore__ void DynamicBlockQuantSingleRow<IN_TYPE, OUT_TYPE, ROUND_MODE>::ComputeVF(264inline __aicore__ void DynamicBlockQuantSingleRow<IN_TYPE, OUT_TYPE, ROUND_MODE>::ComputeVF(
266- __local_mem__ OUT_TYPE* outLocal, __local_mem__ float* scaleLocal, __local_mem__ IN_TYPE* xLocal, uint64_t colSize,265+ __ubuf__ OUT_TYPE* outLocal, __ubuf__ float* scaleLocal, __ubuf__ IN_TYPE* xLocal, uint64_t colSize,
267 uint64_t rowBlockSize, uint64_t colBlockSize)266 uint64_t rowBlockSize, uint64_t colBlockSize)
268{267{
269 IN_TYPE zero = 0.0;268 IN_TYPE zero = 0.0;
@@ -338,8 +337,8 @@ inline __aicore__ void DynamicBlockQuantSingleRow<IN_TYPE, OUT_TYPE, ROUND_MODE>
338 curSize = tilingData_->blockSizeCol;337 curSize = tilingData_->blockSizeCol;
339 for (uint16_t vlLoopIdx = 0; vlLoopIdx < normalLoopNum; vlLoopIdx++) {338 for (uint16_t vlLoopIdx = 0; vlLoopIdx < normalLoopNum; vlLoopIdx++) {
340 inputMaskReg = AscendC::MicroAPI::UpdateMask<IN_TYPE>(curSize);339 inputMaskReg = AscendC::MicroAPI::UpdateMask<IN_TYPE>(curSize);
341- AscendC::MicroAPI::DataCopy(vReg0, xLocal + rowIdx * inputColAlign +340+ AscendC::MicroAPI::LoadAlign(vReg0, xLocal + rowIdx * inputColAlign +
342- colIdx * tilingData_->blockSizeCol + vlLoopIdx * VL);341+ colIdx * tilingData_->blockSizeCol + vlLoopIdx * VL);
343 AscendC::MicroAPI::And((AscendC::MicroAPI::RegTensor<uint32_t>&)vReg3,342 AscendC::MicroAPI::And((AscendC::MicroAPI::RegTensor<uint32_t>&)vReg3,
344 (AscendC::MicroAPI::RegTensor<uint32_t>&)vReg0,343 (AscendC::MicroAPI::RegTensor<uint32_t>&)vReg0,
345 (AscendC::MicroAPI::RegTensor<uint32_t>&)vRegFp32Max, inputMaskReg);344 (AscendC::MicroAPI::RegTensor<uint32_t>&)vRegFp32Max, inputMaskReg);
@@ -349,16 +348,16 @@ inline __aicore__ void DynamicBlockQuantSingleRow<IN_TYPE, OUT_TYPE, ROUND_MODE>
349 (AscendC::MicroAPI::RegTensor<uint32_t>&)vReg3, inputMaskReg);348 (AscendC::MicroAPI::RegTensor<uint32_t>&)vReg3, inputMaskReg);
350 }349 }
351 350 
352- AscendC::MicroAPI::ReduceMax<uint32_t>((AscendC::MicroAPI::RegTensor<uint32_t>&)vReg3,351+ AscendC::MicroAPI::Reduce<MicroAPI::ReduceType::MAX, uint32_t>(
353- (AscendC::MicroAPI::RegTensor<uint32_t>&)vReg2,352+ (AscendC::MicroAPI::RegTensor<uint32_t>&)vReg3, (AscendC::MicroAPI::RegTensor<uint32_t>&)vReg2,
354- defaultMaskReg);353+ defaultMaskReg);
355 354 
356 AscendC::MicroAPI::Duplicate(vReg4, vReg3, defaultMaskReg);355 AscendC::MicroAPI::Duplicate(vReg4, vReg3, defaultMaskReg);
357 356 
358 // brc(input_max) - 位模式直接参与计算357 // brc(input_max) - 位模式直接参与计算
359 AscendC::MicroAPI::Div<float, &mode>(vReg8, (AscendC::MicroAPI::RegTensor<float>&)vReg4,358 AscendC::MicroAPI::Div<float, &mode>(vReg8, (AscendC::MicroAPI::RegTensor<float>&)vReg4,
360 fp8MaxValue, defaultMaskReg);359 fp8MaxValue, defaultMaskReg);
361- AscendC::MicroAPI::CompareScalar<uint32_t, CMPMODE::LT>(360+ AscendC::MicroAPI::Compares<uint32_t, CMPMODE::LT>(
362 scaleMaskReg, (AscendC::MicroAPI::RegTensor<uint32_t>&)vReg8, infValue_, defaultMaskReg);361 scaleMaskReg, (AscendC::MicroAPI::RegTensor<uint32_t>&)vReg8, infValue_, defaultMaskReg);
363 AscendC::MicroAPI::Min<float, AscendC::MicroAPI::MaskMergeMode::MERGING>(362 AscendC::MicroAPI::Min<float, AscendC::MicroAPI::MaskMergeMode::MERGING>(
364 vReg8, vReg8, reciprocalScale, scaleMaskReg);363 vReg8, vReg8, reciprocalScale, scaleMaskReg);
@@ -372,8 +371,8 @@ inline __aicore__ void DynamicBlockQuantSingleRow<IN_TYPE, OUT_TYPE, ROUND_MODE>
372 curSize = tilingData_->blockSizeCol;371 curSize = tilingData_->blockSizeCol;
373 for (uint16_t vlLoopIdx = 0; vlLoopIdx < normalLoopNum; vlLoopIdx++) {372 for (uint16_t vlLoopIdx = 0; vlLoopIdx < normalLoopNum; vlLoopIdx++) {
374 inputMaskReg = AscendC::MicroAPI::UpdateMask<IN_TYPE>(curSize);373 inputMaskReg = AscendC::MicroAPI::UpdateMask<IN_TYPE>(curSize);
375- AscendC::MicroAPI::DataCopy(vReg0, xLocal + rowIdx * inputColAlign +374+ AscendC::MicroAPI::LoadAlign(vReg0, xLocal + rowIdx * inputColAlign +
376- colIdx * tilingData_->blockSizeCol + vlLoopIdx * VL);375+ colIdx * tilingData_->blockSizeCol + vlLoopIdx * VL);
377 AscendC::MicroAPI::Div<float, &mode>(vReg13, vReg0, vReg8, defaultMaskReg);376 AscendC::MicroAPI::Div<float, &mode>(vReg13, vReg0, vReg8, defaultMaskReg);
378 377 
379 if constexpr (IsSameType<OUT_TYPE, hifloat8_t>::value) {378 if constexpr (IsSameType<OUT_TYPE, hifloat8_t>::value) {
@@ -387,7 +386,7 @@ inline __aicore__ void DynamicBlockQuantSingleRow<IN_TYPE, OUT_TYPE, ROUND_MODE>
387 AscendC::MicroAPI::Cast<OUT_TYPE, float, castTrait32tofp8>(vReg15, vReg13, defaultMaskReg);386 AscendC::MicroAPI::Cast<OUT_TYPE, float, castTrait32tofp8>(vReg15, vReg13, defaultMaskReg);
388 }387 }
389 388 
390- AscendC::MicroAPI::DataCopy<OUT_TYPE, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(389+ AscendC::MicroAPI::StoreAlign<OUT_TYPE, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(
391 outLocal + rowIdx * colSizeAlign + colIdx * tilingData_->blockSizeCol + vlLoopIdx * VL,390 outLocal + rowIdx * colSizeAlign + colIdx * tilingData_->blockSizeCol + vlLoopIdx * VL,
392 vReg15, inputMaskReg);391 vReg15, inputMaskReg);
393 }392 }
@@ -398,9 +397,9 @@ inline __aicore__ void DynamicBlockQuantSingleRow<IN_TYPE, OUT_TYPE, ROUND_MODE>
398 curSize = tailBlockSize;397 curSize = tailBlockSize;
399 for (uint16_t vlLoopIdx = 0; vlLoopIdx < tailLoopNum; vlLoopIdx++) {398 for (uint16_t vlLoopIdx = 0; vlLoopIdx < tailLoopNum; vlLoopIdx++) {
400 inputMaskReg = AscendC::MicroAPI::UpdateMask<IN_TYPE>(curSize);399 inputMaskReg = AscendC::MicroAPI::UpdateMask<IN_TYPE>(curSize);
401- AscendC::MicroAPI::DataCopy(vReg0, xLocal + rowIdx * inputColAlign +400+ AscendC::MicroAPI::LoadAlign(vReg0, xLocal + rowIdx * inputColAlign +
402- normalColBlockLoop * tilingData_->blockSizeCol +401+ normalColBlockLoop * tilingData_->blockSizeCol +
403- vlLoopIdx * VL);402+ vlLoopIdx * VL);
404 AscendC::MicroAPI::And((AscendC::MicroAPI::RegTensor<uint32_t>&)vReg3,403 AscendC::MicroAPI::And((AscendC::MicroAPI::RegTensor<uint32_t>&)vReg3,
405 (AscendC::MicroAPI::RegTensor<uint32_t>&)vReg0,404 (AscendC::MicroAPI::RegTensor<uint32_t>&)vReg0,
406 (AscendC::MicroAPI::RegTensor<uint32_t>&)vRegFp32Max, inputMaskReg);405 (AscendC::MicroAPI::RegTensor<uint32_t>&)vRegFp32Max, inputMaskReg);
@@ -409,15 +408,16 @@ inline __aicore__ void DynamicBlockQuantSingleRow<IN_TYPE, OUT_TYPE, ROUND_MODE>
409 (AscendC::MicroAPI::RegTensor<uint32_t>&)vReg3, inputMaskReg);408 (AscendC::MicroAPI::RegTensor<uint32_t>&)vReg3, inputMaskReg);
410 }409 }
411 410 
412- AscendC::MicroAPI::ReduceMax<uint32_t>((AscendC::MicroAPI::RegTensor<uint32_t>&)vReg3,411+ AscendC::MicroAPI::Reduce<MicroAPI::ReduceType::MAX, uint32_t>(
413- (AscendC::MicroAPI::RegTensor<uint32_t>&)vReg2, defaultMaskReg);412+ (AscendC::MicroAPI::RegTensor<uint32_t>&)vReg3, (AscendC::MicroAPI::RegTensor<uint32_t>&)vReg2,
413+ defaultMaskReg);
414 414 
415 AscendC::MicroAPI::Duplicate(vReg4, vReg3, defaultMaskReg);415 AscendC::MicroAPI::Duplicate(vReg4, vReg3, defaultMaskReg);
416 416 
417 // brc(input_max) - 位模式直接参与计算417 // brc(input_max) - 位模式直接参与计算
418 AscendC::MicroAPI::Div<float, &mode>(vReg8, (AscendC::MicroAPI::RegTensor<float>&)vReg4, fp8MaxValue,418 AscendC::MicroAPI::Div<float, &mode>(vReg8, (AscendC::MicroAPI::RegTensor<float>&)vReg4, fp8MaxValue,
419 defaultMaskReg);419 defaultMaskReg);
420- AscendC::MicroAPI::CompareScalar<uint32_t, CMPMODE::LT>(420+ AscendC::MicroAPI::Compares<uint32_t, CMPMODE::LT>(
421 scaleMaskReg, (AscendC::MicroAPI::RegTensor<uint32_t>&)vReg8, infValue_, defaultMaskReg);421 scaleMaskReg, (AscendC::MicroAPI::RegTensor<uint32_t>&)vReg8, infValue_, defaultMaskReg);
422 AscendC::MicroAPI::Min<float, AscendC::MicroAPI::MaskMergeMode::MERGING>(vReg8, vReg8, reciprocalScale,422 AscendC::MicroAPI::Min<float, AscendC::MicroAPI::MaskMergeMode::MERGING>(vReg8, vReg8, reciprocalScale,
423 scaleMaskReg);423 scaleMaskReg);
@@ -431,9 +431,9 @@ inline __aicore__ void DynamicBlockQuantSingleRow<IN_TYPE, OUT_TYPE, ROUND_MODE>
431 curSize = tailBlockSize;431 curSize = tailBlockSize;
432 for (uint16_t vlLoopIdx = 0; vlLoopIdx < tailLoopNum; vlLoopIdx++) {432 for (uint16_t vlLoopIdx = 0; vlLoopIdx < tailLoopNum; vlLoopIdx++) {
433 inputMaskReg = AscendC::MicroAPI::UpdateMask<IN_TYPE>(curSize);433 inputMaskReg = AscendC::MicroAPI::UpdateMask<IN_TYPE>(curSize);
434- AscendC::MicroAPI::DataCopy(vReg0, xLocal + rowIdx * inputColAlign +434+ AscendC::MicroAPI::LoadAlign(vReg0, xLocal + rowIdx * inputColAlign +
435- normalColBlockLoop * tilingData_->blockSizeCol +435+ normalColBlockLoop * tilingData_->blockSizeCol +
436- vlLoopIdx * VL);436+ vlLoopIdx * VL);
437 AscendC::MicroAPI::Div<float, &mode>(vReg13, vReg0, vReg8, defaultMaskReg);437 AscendC::MicroAPI::Div<float, &mode>(vReg13, vReg0, vReg8, defaultMaskReg);
438 438 
439 if constexpr (IsSameType<OUT_TYPE, hifloat8_t>::value) {439 if constexpr (IsSameType<OUT_TYPE, hifloat8_t>::value) {
@@ -446,7 +446,7 @@ inline __aicore__ void DynamicBlockQuantSingleRow<IN_TYPE, OUT_TYPE, ROUND_MODE>
446 AscendC::MicroAPI::Cast<OUT_TYPE, float, castTrait32tofp8>(vReg15, vReg13, defaultMaskReg);446 AscendC::MicroAPI::Cast<OUT_TYPE, float, castTrait32tofp8>(vReg15, vReg13, defaultMaskReg);
447 }447 }
448 448 
449- AscendC::MicroAPI::DataCopy<OUT_TYPE, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(449+ AscendC::MicroAPI::StoreAlign<OUT_TYPE, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(
450 outLocal + rowIdx * colSizeAlign + normalColBlockLoop * tilingData_->blockSizeCol +450 outLocal + rowIdx * colSizeAlign + normalColBlockLoop * tilingData_->blockSizeCol +
451 vlLoopIdx * VL,451 vlLoopIdx * VL,
452 vReg15, inputMaskReg);452 vReg15, inputMaskReg);
@@ -463,8 +463,8 @@ inline __aicore__ void DynamicBlockQuantSingleRow<IN_TYPE, OUT_TYPE, ROUND_MODE>
463 curSize = tilingData_->blockSizeCol;463 curSize = tilingData_->blockSizeCol;
464 for (uint16_t vlLoopIdx = 0; vlLoopIdx < normalLoopNum; vlLoopIdx++) {464 for (uint16_t vlLoopIdx = 0; vlLoopIdx < normalLoopNum; vlLoopIdx++) {
465 inputMaskReg = AscendC::MicroAPI::UpdateMask<IN_TYPE>(curSize);465 inputMaskReg = AscendC::MicroAPI::UpdateMask<IN_TYPE>(curSize);
466- AscendC::MicroAPI::DataCopy(vReg0, xLocal + rowIdx * inputColAlign +466+ AscendC::MicroAPI::LoadAlign(vReg0, xLocal + rowIdx * inputColAlign +
467- colIdx * tilingData_->blockSizeCol + vlLoopIdx * VL);467+ colIdx * tilingData_->blockSizeCol + vlLoopIdx * VL);
468 AscendC::MicroAPI::And((AscendC::MicroAPI::RegTensor<uint16_t>&)vReg3,468 AscendC::MicroAPI::And((AscendC::MicroAPI::RegTensor<uint16_t>&)vReg3,
469 (AscendC::MicroAPI::RegTensor<uint16_t>&)vReg0,469 (AscendC::MicroAPI::RegTensor<uint16_t>&)vReg0,
470 (AscendC::MicroAPI::RegTensor<uint16_t>&)vRegFp16Max, inputMaskReg);470 (AscendC::MicroAPI::RegTensor<uint16_t>&)vRegFp16Max, inputMaskReg);
@@ -473,8 +473,9 @@ inline __aicore__ void DynamicBlockQuantSingleRow<IN_TYPE, OUT_TYPE, ROUND_MODE>
473 inputMaskReg);473 inputMaskReg);
474 }474 }
475 475 
476- AscendC::MicroAPI::ReduceMax((AscendC::MicroAPI::RegTensor<uint16_t>&)vReg3,476+ AscendC::MicroAPI::Reduce<MicroAPI::ReduceType::MAX>((AscendC::MicroAPI::RegTensor<uint16_t>&)vReg3,
477- (AscendC::MicroAPI::RegTensor<uint16_t>&)vReg2, defaultMaskReg);477+ (AscendC::MicroAPI::RegTensor<uint16_t>&)vReg2,
478+ defaultMaskReg);
478 479 
479 // brc(input_max)480 // brc(input_max)
480 AscendC::MicroAPI::Duplicate(vReg4, vReg3, defaultMaskReg);481 AscendC::MicroAPI::Duplicate(vReg4, vReg3, defaultMaskReg);
@@ -484,7 +485,7 @@ inline __aicore__ void DynamicBlockQuantSingleRow<IN_TYPE, OUT_TYPE, ROUND_MODE>
484 485 
485 // input_max / FP_MAX486 // input_max / FP_MAX
486 AscendC::MicroAPI::Div<float, &mode>(vReg8, vReg5, fp8MaxValue, defaultMaskReg);487 AscendC::MicroAPI::Div<float, &mode>(vReg8, vReg5, fp8MaxValue, defaultMaskReg);
487- AscendC::MicroAPI::CompareScalar<uint32_t, CMPMODE::LT>(488+ AscendC::MicroAPI::Compares<uint32_t, CMPMODE::LT>(
488 scaleMaskReg, (AscendC::MicroAPI::RegTensor<uint32_t>&)vReg8, infValue_, defaultMaskReg);489 scaleMaskReg, (AscendC::MicroAPI::RegTensor<uint32_t>&)vReg8, infValue_, defaultMaskReg);
489 // Min(input_max / FP_MAX, 1 / minScale)490 // Min(input_max / FP_MAX, 1 / minScale)
490 AscendC::MicroAPI::Min<float, AscendC::MicroAPI::MaskMergeMode::MERGING>(491 AscendC::MicroAPI::Min<float, AscendC::MicroAPI::MaskMergeMode::MERGING>(
@@ -499,8 +500,8 @@ inline __aicore__ void DynamicBlockQuantSingleRow<IN_TYPE, OUT_TYPE, ROUND_MODE>
499 curSize = tilingData_->blockSizeCol;500 curSize = tilingData_->blockSizeCol;
500 for (uint16_t vlLoopIdx = 0; vlLoopIdx < normalLoopNum; vlLoopIdx++) {501 for (uint16_t vlLoopIdx = 0; vlLoopIdx < normalLoopNum; vlLoopIdx++) {
501 inputMaskReg = AscendC::MicroAPI::UpdateMask<IN_TYPE>(curSize);502 inputMaskReg = AscendC::MicroAPI::UpdateMask<IN_TYPE>(curSize);
502- AscendC::MicroAPI::DataCopy(vReg0, xLocal + rowIdx * inputColAlign +503+ AscendC::MicroAPI::LoadAlign(vReg0, xLocal + rowIdx * inputColAlign +
503- colIdx * tilingData_->blockSizeCol + vlLoopIdx * VL);504+ colIdx * tilingData_->blockSizeCol + vlLoopIdx * VL);
504 AscendC::MicroAPI::Cast<float, IN_TYPE, castTraitZero>(vReg9, vReg0, defaultMaskReg);505 AscendC::MicroAPI::Cast<float, IN_TYPE, castTraitZero>(vReg9, vReg0, defaultMaskReg);
505 AscendC::MicroAPI::Cast<float, IN_TYPE, castTraitOne>(vReg10, vReg0, defaultMaskReg);506 AscendC::MicroAPI::Cast<float, IN_TYPE, castTraitOne>(vReg10, vReg0, defaultMaskReg);
506 AscendC::MicroAPI::Interleave(vReg11, vReg12, vReg9, vReg10);507 AscendC::MicroAPI::Interleave(vReg11, vReg12, vReg9, vReg10);
@@ -524,13 +525,13 @@ inline __aicore__ void DynamicBlockQuantSingleRow<IN_TYPE, OUT_TYPE, ROUND_MODE>
524 AscendC::MicroAPI::Cast<OUT_TYPE, float, castTrait32tofp8>(vReg16, vReg14, defaultMaskReg);525 AscendC::MicroAPI::Cast<OUT_TYPE, float, castTrait32tofp8>(vReg16, vReg14, defaultMaskReg);
525 }526 }
526 527 
527- AscendC::MicroAPI::MaskUnPack<AscendC::MicroAPI::HighLowPart::LOWEST>(maskReg2, inputMaskReg);528+ AscendC::MicroAPI::UnPack<AscendC::MicroAPI::HighLowPart::LOWEST>(maskReg2, inputMaskReg);
528- AscendC::MicroAPI::MaskUnPack<AscendC::MicroAPI::HighLowPart::HIGHEST>(maskReg3, inputMaskReg);529+ AscendC::MicroAPI::UnPack<AscendC::MicroAPI::HighLowPart::HIGHEST>(maskReg3, inputMaskReg);
529 530 
530- AscendC::MicroAPI::DataCopy<OUT_TYPE, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(531+ AscendC::MicroAPI::StoreAlign<OUT_TYPE, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(
531 outLocal + rowIdx * colSizeAlign + colIdx * tilingData_->blockSizeCol + vlLoopIdx * VL,532 outLocal + rowIdx * colSizeAlign + colIdx * tilingData_->blockSizeCol + vlLoopIdx * VL,
532 vReg15, maskReg2);533 vReg15, maskReg2);
533- AscendC::MicroAPI::DataCopy<OUT_TYPE, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(534+ AscendC::MicroAPI::StoreAlign<OUT_TYPE, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(
534 outLocal + rowIdx * colSizeAlign + colIdx * tilingData_->blockSizeCol + vlLoopIdx * VL + 64,535 outLocal + rowIdx * colSizeAlign + colIdx * tilingData_->blockSizeCol + vlLoopIdx * VL + 64,
535 vReg16, maskReg3);536 vReg16, maskReg3);
536 }537 }
@@ -540,9 +541,9 @@ inline __aicore__ void DynamicBlockQuantSingleRow<IN_TYPE, OUT_TYPE, ROUND_MODE>
540 curSize = tailBlockSize;541 curSize = tailBlockSize;
541 for (uint16_t vlLoopIdx = 0; vlLoopIdx < tailLoopNum; vlLoopIdx++) {542 for (uint16_t vlLoopIdx = 0; vlLoopIdx < tailLoopNum; vlLoopIdx++) {
542 inputMaskReg = AscendC::MicroAPI::UpdateMask<IN_TYPE>(curSize);543 inputMaskReg = AscendC::MicroAPI::UpdateMask<IN_TYPE>(curSize);
543- AscendC::MicroAPI::DataCopy(vReg0, xLocal + rowIdx * inputColAlign +544+ AscendC::MicroAPI::LoadAlign(vReg0, xLocal + rowIdx * inputColAlign +
544- normalColBlockLoop * tilingData_->blockSizeCol +545+ normalColBlockLoop * tilingData_->blockSizeCol +
545- vlLoopIdx * VL);546+ vlLoopIdx * VL);
546 AscendC::MicroAPI::And((AscendC::MicroAPI::RegTensor<uint16_t>&)vReg3,547 AscendC::MicroAPI::And((AscendC::MicroAPI::RegTensor<uint16_t>&)vReg3,
547 (AscendC::MicroAPI::RegTensor<uint16_t>&)vReg0,548 (AscendC::MicroAPI::RegTensor<uint16_t>&)vReg0,
548 (AscendC::MicroAPI::RegTensor<uint16_t>&)vRegFp16Max, inputMaskReg);549 (AscendC::MicroAPI::RegTensor<uint16_t>&)vRegFp16Max, inputMaskReg);
@@ -551,8 +552,9 @@ inline __aicore__ void DynamicBlockQuantSingleRow<IN_TYPE, OUT_TYPE, ROUND_MODE>
551 inputMaskReg);552 inputMaskReg);
552 }553 }
553 554 
554- AscendC::MicroAPI::ReduceMax((AscendC::MicroAPI::RegTensor<uint16_t>&)vReg3,555+ AscendC::MicroAPI::Reduce<MicroAPI::ReduceType::MAX>((AscendC::MicroAPI::RegTensor<uint16_t>&)vReg3,
555- (AscendC::MicroAPI::RegTensor<uint16_t>&)vReg2, defaultMaskReg);556+ (AscendC::MicroAPI::RegTensor<uint16_t>&)vReg2,
557+ defaultMaskReg);
556 558 
557 // brc(input_max)559 // brc(input_max)
558 AscendC::MicroAPI::Duplicate(vReg4, vReg3, defaultMaskReg);560 AscendC::MicroAPI::Duplicate(vReg4, vReg3, defaultMaskReg);
@@ -562,7 +564,7 @@ inline __aicore__ void DynamicBlockQuantSingleRow<IN_TYPE, OUT_TYPE, ROUND_MODE>
562 564 
563 // input_max / FP_MAX565 // input_max / FP_MAX
564 AscendC::MicroAPI::Div<float, &mode>(vReg8, vReg5, fp8MaxValue, defaultMaskReg);566 AscendC::MicroAPI::Div<float, &mode>(vReg8, vReg5, fp8MaxValue, defaultMaskReg);
565- AscendC::MicroAPI::CompareScalar<uint32_t, CMPMODE::LT>(567+ AscendC::MicroAPI::Compares<uint32_t, CMPMODE::LT>(
566 scaleMaskReg, (AscendC::MicroAPI::RegTensor<uint32_t>&)vReg8, infValue_, defaultMaskReg);568 scaleMaskReg, (AscendC::MicroAPI::RegTensor<uint32_t>&)vReg8, infValue_, defaultMaskReg);
567 // Min(input_max / FP_MAX, 1 / minScale)569 // Min(input_max / FP_MAX, 1 / minScale)
568 AscendC::MicroAPI::Min<float, AscendC::MicroAPI::MaskMergeMode::MERGING>(vReg8, vReg8, reciprocalScale,570 AscendC::MicroAPI::Min<float, AscendC::MicroAPI::MaskMergeMode::MERGING>(vReg8, vReg8, reciprocalScale,
@@ -577,9 +579,9 @@ inline __aicore__ void DynamicBlockQuantSingleRow<IN_TYPE, OUT_TYPE, ROUND_MODE>
577 curSize = tailBlockSize;579 curSize = tailBlockSize;
578 for (uint16_t vlLoopIdx = 0; vlLoopIdx < tailLoopNum; vlLoopIdx++) {580 for (uint16_t vlLoopIdx = 0; vlLoopIdx < tailLoopNum; vlLoopIdx++) {
579 inputMaskReg = AscendC::MicroAPI::UpdateMask<IN_TYPE>(curSize);581 inputMaskReg = AscendC::MicroAPI::UpdateMask<IN_TYPE>(curSize);
580- AscendC::MicroAPI::DataCopy(vReg0, xLocal + rowIdx * inputColAlign +582+ AscendC::MicroAPI::LoadAlign(vReg0, xLocal + rowIdx * inputColAlign +
581- normalColBlockLoop * tilingData_->blockSizeCol +583+ normalColBlockLoop * tilingData_->blockSizeCol +
582- vlLoopIdx * VL);584+ vlLoopIdx * VL);
583 AscendC::MicroAPI::Cast<float, IN_TYPE, castTraitZero>(vReg9, vReg0, defaultMaskReg);585 AscendC::MicroAPI::Cast<float, IN_TYPE, castTraitZero>(vReg9, vReg0, defaultMaskReg);
584 AscendC::MicroAPI::Cast<float, IN_TYPE, castTraitOne>(vReg10, vReg0, defaultMaskReg);586 AscendC::MicroAPI::Cast<float, IN_TYPE, castTraitOne>(vReg10, vReg0, defaultMaskReg);
585 AscendC::MicroAPI::Interleave(vReg11, vReg12, vReg9, vReg10);587 AscendC::MicroAPI::Interleave(vReg11, vReg12, vReg9, vReg10);
@@ -601,14 +603,14 @@ inline __aicore__ void DynamicBlockQuantSingleRow<IN_TYPE, OUT_TYPE, ROUND_MODE>
601 AscendC::MicroAPI::Cast<OUT_TYPE, float, castTrait32tofp8>(vReg16, vReg14, defaultMaskReg);603 AscendC::MicroAPI::Cast<OUT_TYPE, float, castTrait32tofp8>(vReg16, vReg14, defaultMaskReg);
602 }604 }
603 605 
604- AscendC::MicroAPI::MaskUnPack<AscendC::MicroAPI::HighLowPart::LOWEST>(maskReg2, inputMaskReg);606+ AscendC::MicroAPI::UnPack<AscendC::MicroAPI::HighLowPart::LOWEST>(maskReg2, inputMaskReg);
605- AscendC::MicroAPI::MaskUnPack<AscendC::MicroAPI::HighLowPart::HIGHEST>(maskReg3, inputMaskReg);607+ AscendC::MicroAPI::UnPack<AscendC::MicroAPI::HighLowPart::HIGHEST>(maskReg3, inputMaskReg);
606 608 
607- AscendC::MicroAPI::DataCopy<OUT_TYPE, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(609+ AscendC::MicroAPI::StoreAlign<OUT_TYPE, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(
608 outLocal + rowIdx * colSizeAlign + normalColBlockLoop * tilingData_->blockSizeCol +610 outLocal + rowIdx * colSizeAlign + normalColBlockLoop * tilingData_->blockSizeCol +
609 vlLoopIdx * VL,611 vlLoopIdx * VL,
610 vReg15, maskReg2);612 vReg15, maskReg2);
611- AscendC::MicroAPI::DataCopy<OUT_TYPE, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(613+ AscendC::MicroAPI::StoreAlign<OUT_TYPE, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(
612 outLocal + rowIdx * colSizeAlign + normalColBlockLoop * tilingData_->blockSizeCol +614 outLocal + rowIdx * colSizeAlign + normalColBlockLoop * tilingData_->blockSizeCol +
613 vlLoopIdx * VL + 64,615 vlLoopIdx * VL + 64,
614 vReg16, maskReg3);616 vReg16, maskReg3);
Mquant/dynamic_block_quant/op_kernel/arch35/dynamic_block_quant_small_blocksize_kernel.h+17-17
@@ -370,7 +370,7 @@ __aicore__ inline void DynamicBlockQuantSmallBlockSize<T, U, RMode>::ComputeVF(i
370 // comput Max(x) - float32 使用 uint32_t 位运算370 // comput Max(x) - float32 使用 uint32_t 位运算
371 for (uint16_t i = 0; i < inputVfLoop; i++) {371 for (uint16_t i = 0; i < inputVfLoop; i++) {
372 preg0 = AscendC::MicroAPI::UpdateMask<T>(inputNum);372 preg0 = AscendC::MicroAPI::UpdateMask<T>(inputNum);
373- AscendC::MicroAPI::DataCopy(vreg1, xAddr + i * vfLen);373+ AscendC::MicroAPI::LoadAlign(vreg1, xAddr + i * vfLen);
374 AscendC::MicroAPI::And((AscendC::MicroAPI::RegTensor<uint32_t>&)vreg3,374 AscendC::MicroAPI::And((AscendC::MicroAPI::RegTensor<uint32_t>&)vreg3,
375 (AscendC::MicroAPI::RegTensor<uint32_t>&)vreg1,375 (AscendC::MicroAPI::RegTensor<uint32_t>&)vreg1,
376 (AscendC::MicroAPI::RegTensor<uint32_t>&)vreg2, preg0);376 (AscendC::MicroAPI::RegTensor<uint32_t>&)vreg2, preg0);
@@ -379,7 +379,7 @@ __aicore__ inline void DynamicBlockQuantSmallBlockSize<T, U, RMode>::ComputeVF(i
379 }379 }
380 380 
381 // comput scale - float32 直接用位模式参与计算381 // comput scale - float32 直接用位模式参与计算
382- AscendC::MicroAPI::ReduceMax<uint32_t>(expMaxRegTensor, expMaxRegTensor, maskAll);382+ AscendC::MicroAPI::Reduce<MicroAPI::ReduceType::MAX, uint32_t>(expMaxRegTensor, expMaxRegTensor, maskAll);
383 AscendC::MicroAPI::Duplicate(expMaxRegTensor, expMaxRegTensor, maskAll);383 AscendC::MicroAPI::Duplicate(expMaxRegTensor, expMaxRegTensor, maskAll);
384 AscendC::MicroAPI::Duplicate(fp8MaxReg, fp8MaxValue_);384 AscendC::MicroAPI::Duplicate(fp8MaxReg, fp8MaxValue_);
385 AscendC::MicroAPI::Duplicate(reciprocalScale, 1.0f);385 AscendC::MicroAPI::Duplicate(reciprocalScale, 1.0f);
@@ -391,17 +391,17 @@ __aicore__ inline void DynamicBlockQuantSmallBlockSize<T, U, RMode>::ComputeVF(i
391 AscendC::MicroAPI::Div<float, &mode>(scaleRegTensor, (AscendC::MicroAPI::RegTensor<float>&)expMaxRegTensor,391 AscendC::MicroAPI::Div<float, &mode>(scaleRegTensor, (AscendC::MicroAPI::RegTensor<float>&)expMaxRegTensor,
392 fp8MaxReg, ymaskAll);392 fp8MaxReg, ymaskAll);
393 393 
394- AscendC::MicroAPI::CompareScalar<uint32_t, CMPMODE::LT>(394+ AscendC::MicroAPI::Compares<uint32_t, CMPMODE::LT>(
395 scaleMaskReg, (AscendC::MicroAPI::RegTensor<uint32_t>&)scaleRegTensor, infValue_, ymaskAll);395 scaleMaskReg, (AscendC::MicroAPI::RegTensor<uint32_t>&)scaleRegTensor, infValue_, ymaskAll);
396 AscendC::MicroAPI::Min<float, AscendC::MicroAPI::MaskMergeMode::MERGING>(scaleRegTensor, scaleRegTensor,396 AscendC::MicroAPI::Min<float, AscendC::MicroAPI::MaskMergeMode::MERGING>(scaleRegTensor, scaleRegTensor,
397 reciprocalScale, scaleMaskReg);397 reciprocalScale, scaleMaskReg);
398- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::StoreDist::DIST_FIRST_ELEMENT_B32>(398+ AscendC::MicroAPI::StoreAlign<float, AscendC::MicroAPI::StoreDist::DIST_FIRST_ELEMENT_B32>(
399 scaleOutAddr, scaleRegTensor, ymaskAll);399 scaleOutAddr, scaleRegTensor, ymaskAll);
400 400 
401 // compute y - float32 直接加载，不需要 UNPACK401 // compute y - float32 直接加载，不需要 UNPACK
402 for (uint16_t i = 0; i < vfLoop; i++) {402 for (uint16_t i = 0; i < vfLoop; i++) {
403 yMaskReg0 = AscendC::MicroAPI::UpdateMask<float>(outputNum);403 yMaskReg0 = AscendC::MicroAPI::UpdateMask<float>(outputNum);
404- AscendC::MicroAPI::DataCopy(yReg1, xAddr + i * vfNum);404+ AscendC::MicroAPI::LoadAlign(yReg1, xAddr + i * vfNum);
405 AscendC::MicroAPI::Div<float, &mode>(yReg3, yReg1, scaleRegTensor, yMaskReg0);405 AscendC::MicroAPI::Div<float, &mode>(yReg3, yReg1, scaleRegTensor, yMaskReg0);
406 406 
407 if constexpr (IsSameType<U, hifloat8_t>::value) {407 if constexpr (IsSameType<U, hifloat8_t>::value) {
@@ -413,8 +413,8 @@ __aicore__ inline void DynamicBlockQuantSmallBlockSize<T, U, RMode>::ComputeVF(i
413 } else {413 } else {
414 AscendC::MicroAPI::Cast<U, float, castTrait32tofp8>(outReg, yReg3, yMaskReg0);414 AscendC::MicroAPI::Cast<U, float, castTrait32tofp8>(outReg, yReg3, yMaskReg0);
415 }415 }
416- AscendC::MicroAPI::DataCopy<U, MicroAPI::StoreDist::DIST_PACK4_B32>(yOutAddr + i * vfNum, outReg,416+ AscendC::MicroAPI::StoreAlign<U, MicroAPI::StoreDist::DIST_PACK4_B32>(yOutAddr + i * vfNum, outReg,
417- yMaskReg0);417+ yMaskReg0);
418 }418 }
419 } else {419 } else {
420 // ===== FP16/BF16 类型处理 =====420 // ===== FP16/BF16 类型处理 =====
@@ -430,7 +430,7 @@ __aicore__ inline void DynamicBlockQuantSmallBlockSize<T, U, RMode>::ComputeVF(i
430 // comput Max(x) - FP16/BF16 使用 uint16_t 位运算430 // comput Max(x) - FP16/BF16 使用 uint16_t 位运算
431 for (uint16_t i = 0; i < inputVfLoop; i++) {431 for (uint16_t i = 0; i < inputVfLoop; i++) {
432 preg0 = AscendC::MicroAPI::UpdateMask<T>(inputNum);432 preg0 = AscendC::MicroAPI::UpdateMask<T>(inputNum);
433- AscendC::MicroAPI::DataCopy(vreg1, xAddr + i * vfLen);433+ AscendC::MicroAPI::LoadAlign(vreg1, xAddr + i * vfLen);
434 AscendC::MicroAPI::And((AscendC::MicroAPI::RegTensor<uint16_t>&)vreg3,434 AscendC::MicroAPI::And((AscendC::MicroAPI::RegTensor<uint16_t>&)vreg3,
435 (AscendC::MicroAPI::RegTensor<uint16_t>&)vreg1,435 (AscendC::MicroAPI::RegTensor<uint16_t>&)vreg1,
436 (AscendC::MicroAPI::RegTensor<uint16_t>&)vreg2, preg0);436 (AscendC::MicroAPI::RegTensor<uint16_t>&)vreg2, preg0);
@@ -439,7 +439,7 @@ __aicore__ inline void DynamicBlockQuantSmallBlockSize<T, U, RMode>::ComputeVF(i
439 }439 }
440 440 
441 // comput scale441 // comput scale
442- AscendC::MicroAPI::ReduceMax<uint16_t>(expMaxRegTensor, expMaxRegTensor, maskAll);442+ AscendC::MicroAPI::Reduce<MicroAPI::ReduceType::MAX, uint16_t>(expMaxRegTensor, expMaxRegTensor, maskAll);
443 AscendC::MicroAPI::Duplicate(expMaxRegTensor, expMaxRegTensor, maskAll);443 AscendC::MicroAPI::Duplicate(expMaxRegTensor, expMaxRegTensor, maskAll);
444 AscendC::MicroAPI::Duplicate(fp8MaxReg, fp8MaxValue_);444 AscendC::MicroAPI::Duplicate(fp8MaxReg, fp8MaxValue_);
445 AscendC::MicroAPI::Duplicate(reciprocalScale, 1.0f);445 AscendC::MicroAPI::Duplicate(reciprocalScale, 1.0f);
@@ -450,17 +450,17 @@ __aicore__ inline void DynamicBlockQuantSmallBlockSize<T, U, RMode>::ComputeVF(i
450 (AscendC::MicroAPI::RegTensor<T>&)expMaxRegTensor, maskAll);450 (AscendC::MicroAPI::RegTensor<T>&)expMaxRegTensor, maskAll);
451 451 
452 AscendC::MicroAPI::Div<float, &mode>(scaleRegTensor, scaleRegTensor, fp8MaxReg, ymaskAll);452 AscendC::MicroAPI::Div<float, &mode>(scaleRegTensor, scaleRegTensor, fp8MaxReg, ymaskAll);
453- AscendC::MicroAPI::CompareScalar<uint32_t, CMPMODE::LT>(453+ AscendC::MicroAPI::Compares<uint32_t, CMPMODE::LT>(
454 scaleMaskReg, (AscendC::MicroAPI::RegTensor<uint32_t>&)scaleRegTensor, infValue_, ymaskAll);454 scaleMaskReg, (AscendC::MicroAPI::RegTensor<uint32_t>&)scaleRegTensor, infValue_, ymaskAll);
455 AscendC::MicroAPI::Min<float, AscendC::MicroAPI::MaskMergeMode::MERGING>(scaleRegTensor, scaleRegTensor,455 AscendC::MicroAPI::Min<float, AscendC::MicroAPI::MaskMergeMode::MERGING>(scaleRegTensor, scaleRegTensor,
456 reciprocalScale, scaleMaskReg);456 reciprocalScale, scaleMaskReg);
457- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::StoreDist::DIST_FIRST_ELEMENT_B32>(457+ AscendC::MicroAPI::StoreAlign<float, AscendC::MicroAPI::StoreDist::DIST_FIRST_ELEMENT_B32>(
458 scaleOutAddr, scaleRegTensor, ymaskAll);458 scaleOutAddr, scaleRegTensor, ymaskAll);
459 459 
460 // compute y460 // compute y
461 for (uint16_t i = 0; i < vfLoop; i++) {461 for (uint16_t i = 0; i < vfLoop; i++) {
462 yMaskReg0 = AscendC::MicroAPI::UpdateMask<float>(outputNum);462 yMaskReg0 = AscendC::MicroAPI::UpdateMask<float>(outputNum);
463- AscendC::MicroAPI::DataCopy<T, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(yReg1, xAddr + i * vfNum);463+ AscendC::MicroAPI::LoadAlign<T, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(yReg1, xAddr + i * vfNum);
464 AscendC::MicroAPI::Cast<float, T, castTrait0>(yReg2, yReg1, yMaskReg0);464 AscendC::MicroAPI::Cast<float, T, castTrait0>(yReg2, yReg1, yMaskReg0);
465 AscendC::MicroAPI::Div<float, &mode>(yReg3, yReg2, scaleRegTensor, yMaskReg0);465 AscendC::MicroAPI::Div<float, &mode>(yReg3, yReg2, scaleRegTensor, yMaskReg0);
466 466 
@@ -473,8 +473,8 @@ __aicore__ inline void DynamicBlockQuantSmallBlockSize<T, U, RMode>::ComputeVF(i
473 } else {473 } else {
474 AscendC::MicroAPI::Cast<U, float, castTrait32tofp8>(outReg, yReg3, yMaskReg0);474 AscendC::MicroAPI::Cast<U, float, castTrait32tofp8>(outReg, yReg3, yMaskReg0);
475 }475 }
476- AscendC::MicroAPI::DataCopy<U, MicroAPI::StoreDist::DIST_PACK4_B32>(yOutAddr + i * vfNum, outReg,476+ AscendC::MicroAPI::StoreAlign<U, MicroAPI::StoreDist::DIST_PACK4_B32>(yOutAddr + i * vfNum, outReg,
477- yMaskReg0);477+ yMaskReg0);
478 }478 }
479 }479 }
480 }480 }
@@ -486,13 +486,13 @@ __aicore__ inline void DynamicBlockQuantSmallBlockSize<T, U, RMode>::Compute(int
486 int64_t nowRowBlock)486 int64_t nowRowBlock)
487{487{
488 LocalTensor<T> inLocal = inQueue_.DeQue<T>();488 LocalTensor<T> inLocal = inQueue_.DeQue<T>();
489- __local_mem__ T* xLocalAddr = (__local_mem__ T*)inLocal.GetPhyAddr();489+ __ubuf__ T* xLocalAddr = (__ubuf__ T*)inLocal.GetPhyAddr();
490 490 
491 LocalTensor<float> scaleLocal = scaleQueue_.AllocTensor<float>();491 LocalTensor<float> scaleLocal = scaleQueue_.AllocTensor<float>();
492- __local_mem__ float* scaleLocalAddr = (__local_mem__ float*)scaleLocal.GetPhyAddr();492+ __ubuf__ float* scaleLocalAddr = (__ubuf__ float*)scaleLocal.GetPhyAddr();
493 493 
494 LocalTensor<U> outLocal = outQueue_.AllocTensor<U>();494 LocalTensor<U> outLocal = outQueue_.AllocTensor<U>();
495- __local_mem__ U* outLocalAddr = (__local_mem__ U*)outLocal.GetPhyAddr();495+ __ubuf__ U* outLocalAddr = (__ubuf__ U*)outLocal.GetPhyAddr();
496 int64_t ubOffset = 0;496 int64_t ubOffset = 0;
497 int64_t yOffset = 0;497 int64_t yOffset = 0;
498 int32_t inputColAlign = BLOCK_BYTE_32 / sizeof(T);498 int32_t inputColAlign = BLOCK_BYTE_32 / sizeof(T);
Mquant/dynamic_dual_level_mx_quant/op_kernel/arch35/dynamic_dual_level_mx_quant_base.h+23-21
@@ -497,22 +497,22 @@ __simd_vf__ inline void DynamicDualLevelMxQuantBase<xDtype, roundMode, needSmoot
497 }497 }
498 498 
499 for (uint16_t j = 0; j < static_cast<uint16_t>(loopNum); j++) {499 for (uint16_t j = 0; j < static_cast<uint16_t>(loopNum); j++) {
500- MicroAPI::DataCopy<xDtype, MicroAPI::PostLiteral::POST_MODE_UPDATE, MicroAPI::LoadDist::DIST_NORM>(500+ MicroAPI::LoadAlign<xDtype, MicroAPI::PostLiteral::POST_MODE_UPDATE, MicroAPI::LoadDist::DIST_NORM>(
501 x0, xUbAddr, vlForHalfNumber_);501 x0, xUbAddr, vlForHalfNumber_);
502- MicroAPI::DataCopy<xDtype, MicroAPI::PostLiteral::POST_MODE_UPDATE, MicroAPI::LoadDist::DIST_NORM>(502+ MicroAPI::LoadAlign<xDtype, MicroAPI::PostLiteral::POST_MODE_UPDATE, MicroAPI::LoadDist::DIST_NORM>(
503 x1, xUbAddr, vlForHalfNumber_);503 x1, xUbAddr, vlForHalfNumber_);
504- MicroAPI::DataCopy<xDtype, MicroAPI::PostLiteral::POST_MODE_UPDATE, MicroAPI::LoadDist::DIST_NORM>(504+ MicroAPI::LoadAlign<xDtype, MicroAPI::PostLiteral::POST_MODE_UPDATE, MicroAPI::LoadDist::DIST_NORM>(
505 x2, xUbAddr, vlForHalfNumber_);505 x2, xUbAddr, vlForHalfNumber_);
506- MicroAPI::DataCopy<xDtype, MicroAPI::PostLiteral::POST_MODE_UPDATE, MicroAPI::LoadDist::DIST_NORM>(506+ MicroAPI::LoadAlign<xDtype, MicroAPI::PostLiteral::POST_MODE_UPDATE, MicroAPI::LoadDist::DIST_NORM>(
507 x3, xUbAddr, vlForHalfNumber_);507 x3, xUbAddr, vlForHalfNumber_);
508 if constexpr (needSmoothScale) {508 if constexpr (needSmoothScale) {
509- MicroAPI::DataCopy<xDtype, MicroAPI::PostLiteral::POST_MODE_UPDATE, MicroAPI::LoadDist::DIST_NORM>(509+ MicroAPI::LoadAlign<xDtype, MicroAPI::PostLiteral::POST_MODE_UPDATE, MicroAPI::LoadDist::DIST_NORM>(
510 smoothScale0, smoothScaleUbAddr, vlForHalfNumber_);510 smoothScale0, smoothScaleUbAddr, vlForHalfNumber_);
511- MicroAPI::DataCopy<xDtype, MicroAPI::PostLiteral::POST_MODE_UPDATE, MicroAPI::LoadDist::DIST_NORM>(511+ MicroAPI::LoadAlign<xDtype, MicroAPI::PostLiteral::POST_MODE_UPDATE, MicroAPI::LoadDist::DIST_NORM>(
512 smoothScale1, smoothScaleUbAddr, vlForHalfNumber_);512 smoothScale1, smoothScaleUbAddr, vlForHalfNumber_);
513- MicroAPI::DataCopy<xDtype, MicroAPI::PostLiteral::POST_MODE_UPDATE, MicroAPI::LoadDist::DIST_NORM>(513+ MicroAPI::LoadAlign<xDtype, MicroAPI::PostLiteral::POST_MODE_UPDATE, MicroAPI::LoadDist::DIST_NORM>(
514 smoothScale2, smoothScaleUbAddr, vlForHalfNumber_);514 smoothScale2, smoothScaleUbAddr, vlForHalfNumber_);
515- MicroAPI::DataCopy<xDtype, MicroAPI::PostLiteral::POST_MODE_UPDATE, MicroAPI::LoadDist::DIST_NORM>(515+ MicroAPI::LoadAlign<xDtype, MicroAPI::PostLiteral::POST_MODE_UPDATE, MicroAPI::LoadDist::DIST_NORM>(
516 smoothScale3, smoothScaleUbAddr, vlForHalfNumber_);516 smoothScale3, smoothScaleUbAddr, vlForHalfNumber_);
517 MicroAPI::Mul(x0, x0, smoothScale0, maskAll16);517 MicroAPI::Mul(x0, x0, smoothScale0, maskAll16);
518 MicroAPI::Mul(x1, x1, smoothScale1, maskAll16);518 MicroAPI::Mul(x1, x1, smoothScale1, maskAll16);
@@ -536,7 +536,7 @@ __simd_vf__ inline void DynamicDualLevelMxQuantBase<xDtype, roundMode, needSmoot
536 MicroAPI::Max(absX0, absX0, absX1, maskAll16);536 MicroAPI::Max(absX0, absX0, absX1, maskAll16);
537 MicroAPI::Max(absX2, absX2, absX3, maskAll16);537 MicroAPI::Max(absX2, absX2, absX3, maskAll16);
538 MicroAPI::Max(absX0, absX0, absX2, maskAll16);538 MicroAPI::Max(absX0, absX0, absX2, maskAll16);
539- MicroAPI::ReduceMax(absX0, absX0, maskAll16);539+ MicroAPI::Reduce<MicroAPI::ReduceType::MAX>(absX0, absX0, maskAll16);
540 540 
541 MicroAPI::Cast<float, xDtype, castTraitXdtypetoFp32Zero>(541 MicroAPI::Cast<float, xDtype, castTraitXdtypetoFp32Zero>(
542 level0Scale, (AscendC::MicroAPI::RegTensor<xDtype>&)absX0, maskAll16);542 level0Scale, (AscendC::MicroAPI::RegTensor<xDtype>&)absX0, maskAll16);
@@ -597,7 +597,7 @@ __simd_callee__ inline void DynamicDualLevelMxQuantBase<xDtype, roundMode, needS
597 MicroAPI::Select<xDtype>(xZero, xReg, xZero, zeroMask);597 MicroAPI::Select<xDtype>(xZero, xReg, xZero, zeroMask);
598 598 
599 // 连续搬出128个float32的xTmp599 // 连续搬出128个float32的xTmp
600- MicroAPI::DataCopy(xTmpUbAddr, xZero, maskAll16);600+ MicroAPI::StoreAlign(xTmpUbAddr, xZero, maskAll16);
601}601}
602 602 
603template <typename xDtype, AscendC::RoundMode roundMode, bool needSmoothScale>603template <typename xDtype, AscendC::RoundMode roundMode, bool needSmoothScale>
@@ -647,7 +647,7 @@ __simd_vf__ inline void DynamicDualLevelMxQuantBase<xDtype, roundMode, needSmoot
647 647 
648 for (uint16_t i = 0; i < static_cast<uint16_t>(loopNum); i++) {648 for (uint16_t i = 0; i < static_cast<uint16_t>(loopNum); i++) {
649 // 交织搬运，一次搬256个B16649 // 交织搬运，一次搬256个B16
650- MicroAPI::DataCopy<xDtype, MicroAPI::PostLiteral::POST_MODE_UPDATE, MicroAPI::LoadDist::DIST_DINTLV_B16>(650+ MicroAPI::LoadAlign<xDtype, MicroAPI::PostLiteral::POST_MODE_UPDATE, MicroAPI::LoadDist::DIST_DINTLV_B16>(
651 xTmp0, xTmp1, xTmpUbAddr, vlForHalfNumber_ * DIGIT_TWO);651 xTmp0, xTmp1, xTmpUbAddr, vlForHalfNumber_ * DIGIT_TWO);
652 652 
653 if constexpr (IsSameType<xDtype, half>::value) {653 if constexpr (IsSameType<xDtype, half>::value) {
@@ -674,7 +674,7 @@ __simd_vf__ inline void DynamicDualLevelMxQuantBase<xDtype, roundMode, needSmoot
674 // 计算奇偶位置最大值，相当于计算原始相邻两个数据的最大值674 // 计算奇偶位置最大值，相当于计算原始相邻两个数据的最大值
675 MicroAPI::Max(xTmp0ExpBF16, xTmp1ExpBF16, xTmp0ExpBF16, maskAll16);675 MicroAPI::Max(xTmp0ExpBF16, xTmp1ExpBF16, xTmp0ExpBF16, maskAll16);
676 // ReduceMax一个block，即16个数，配合上一步，可以计算出每32个数的最大值，一共256/32个676 // ReduceMax一个block，即16个数，配合上一步，可以计算出每32个数的最大值，一共256/32个
677- MicroAPI::ReduceMaxWithDataBlock(xTmp0ExpBF16, xTmp0ExpBF16, maskAll16);677+ MicroAPI::ReduceDataBlock<MicroAPI::ReduceType::MAX>(xTmp0ExpBF16, xTmp0ExpBF16, maskAll16);
678 678 
679 // 计算-1轴的scale和1/scale679 // 计算-1轴的scale和1/scale
680 // inf/nan值单独处理，结果为E8M0的nan680 // inf/nan值单独处理，结果为E8M0的nan
@@ -708,7 +708,7 @@ __simd_vf__ inline void DynamicDualLevelMxQuantBase<xDtype, roundMode, needSmoot
708 MicroAPI::Select<uint16_t>(reversedShareExp1, reversedShareExp1, zero, zeroMask);708 MicroAPI::Select<uint16_t>(reversedShareExp1, reversedShareExp1, zero, zeroMask);
709 MicroAPI::Select<uint16_t>(reversedShareExp1, specialExp, reversedShareExp1, invalidDataMask);709 MicroAPI::Select<uint16_t>(reversedShareExp1, specialExp, reversedShareExp1, invalidDataMask);
710 // 搬出8位1/scale，占一个UBBlock方便后续取值计算710 // 搬出8位1/scale，占一个UBBlock方便后续取值计算
711- MicroAPI::DataCopy<uint16_t, MicroAPI::PostLiteral::POST_MODE_UPDATE>(711+ MicroAPI::StoreAlign<uint16_t, MicroAPI::PostLiteral::POST_MODE_UPDATE>(
712 level1ScaleReciprocalUbAddr, reversedShareExp1, UBBlockSize_ / sizeof(uint16_t), maskReduceB16);712 level1ScaleReciprocalUbAddr, reversedShareExp1, UBBlockSize_ / sizeof(uint16_t), maskReduceB16);
713 }713 }
714}714}
@@ -718,6 +718,7 @@ __simd_vf__ inline void DynamicDualLevelMxQuantBase<xDtype, roundMode, needSmoot
718 int64_t loopNum, __ubuf__ xDtype* xTmpUbAddr, __ubuf__ uint8_t* yAddr,718 int64_t loopNum, __ubuf__ xDtype* xTmpUbAddr, __ubuf__ uint8_t* yAddr,
719 __ubuf__ uint16_t* level1ScaleReciprocalUbAddr)719 __ubuf__ uint16_t* level1ScaleReciprocalUbAddr)
720{720{
721+#ifndef ASCENDC_CPU_DEBUG
721 MicroAPI::RegTensor<xDtype> xTmp0;722 MicroAPI::RegTensor<xDtype> xTmp0;
722 MicroAPI::RegTensor<xDtype> xTmp1;723 MicroAPI::RegTensor<xDtype> xTmp1;
723 MicroAPI::RegTensor<uint16_t> scaleForMulFP16;724 MicroAPI::RegTensor<uint16_t> scaleForMulFP16;
@@ -741,11 +742,11 @@ __simd_vf__ inline void DynamicDualLevelMxQuantBase<xDtype, roundMode, needSmoot
741 742 
742 for (uint16_t i = 0; i < static_cast<uint16_t>(loopNum); i++) {743 for (uint16_t i = 0; i < static_cast<uint16_t>(loopNum); i++) {
743 // 搬入8个uint16_t元素,单个元素广播到一个UBBlock中744 // 搬入8个uint16_t元素,单个元素广播到一个UBBlock中
744- MicroAPI::DataCopy<uint16_t, MicroAPI::PostLiteral::POST_MODE_UPDATE, MicroAPI::LoadDist::DIST_E2B_B16>(745+ MicroAPI::LoadAlign<uint16_t, MicroAPI::PostLiteral::POST_MODE_UPDATE, MicroAPI::LoadDist::DIST_E2B_B16>(
745 scaleForMulFP16, level1ScaleReciprocalUbAddr, UBBlockSize_ / sizeof(uint16_t));746 scaleForMulFP16, level1ScaleReciprocalUbAddr, UBBlockSize_ / sizeof(uint16_t));
746 747 
747 // 交织搬入256个xTmp(bfloat16_t)748 // 交织搬入256个xTmp(bfloat16_t)
748- MicroAPI::DataCopy<xDtype, MicroAPI::PostLiteral::POST_MODE_UPDATE, MicroAPI::LoadDist::DIST_DINTLV_B16>(749+ MicroAPI::LoadAlign<xDtype, MicroAPI::PostLiteral::POST_MODE_UPDATE, MicroAPI::LoadDist::DIST_DINTLV_B16>(
749 xTmp0, xTmp1, xTmpUbAddr, vlForHalfNumber_ * DIGIT_TWO);750 xTmp0, xTmp1, xTmpUbAddr, vlForHalfNumber_ * DIGIT_TWO);
750 751 
751 if constexpr (IsSameType<xDtype, half>::value) {752 if constexpr (IsSameType<xDtype, half>::value) {
@@ -799,11 +800,12 @@ __simd_vf__ inline void DynamicDualLevelMxQuantBase<xDtype, roundMode, needSmoot
799 }800 }
800 801 
801 // 256个fp4元素作为128个uint8元素搬出802 // 256个fp4元素作为128个uint8元素搬出
802- MicroAPI::DataCopy<uint8_t, MicroAPI::PostLiteral::POST_MODE_UPDATE, MicroAPI::StoreDist::DIST_PACK4_B32>(803+ MicroAPI::StoreAlign<uint8_t, MicroAPI::PostLiteral::POST_MODE_UPDATE, MicroAPI::StoreDist::DIST_PACK4_B32>(
803 yAddr, (MicroAPI::RegTensor<uint8_t>&)y0FP4, DIGIT_64, dataMaskB8);804 yAddr, (MicroAPI::RegTensor<uint8_t>&)y0FP4, DIGIT_64, dataMaskB8);
804- MicroAPI::DataCopy<uint8_t, MicroAPI::PostLiteral::POST_MODE_UPDATE, MicroAPI::StoreDist::DIST_PACK4_B32>(805+ MicroAPI::StoreAlign<uint8_t, MicroAPI::PostLiteral::POST_MODE_UPDATE, MicroAPI::StoreDist::DIST_PACK4_B32>(
805 yAddr, (MicroAPI::RegTensor<uint8_t>&)y1FP4, DIGIT_64, dataMaskB8);806 yAddr, (MicroAPI::RegTensor<uint8_t>&)y1FP4, DIGIT_64, dataMaskB8);
806 }807 }
808+#endif // ASCENDC_CPU_DEBUG
807}809}
808 810 
809template <typename xDtype, AscendC::RoundMode roundMode, bool needSmoothScale>811template <typename xDtype, AscendC::RoundMode roundMode, bool needSmoothScale>
@@ -838,13 +840,13 @@ __simd_callee__ inline void DynamicDualLevelMxQuantBase<xDtype, roundMode, needS
838 MicroAPI::Mul(Reg, Reg, (MicroAPI::RegTensor<float>&)exp1FP32, pregAll32);840 MicroAPI::Mul(Reg, Reg, (MicroAPI::RegTensor<float>&)exp1FP32, pregAll32);
839 MicroAPI::Adds(exp0FP32, exp0FP32, FP32_BIAS, pregAll32);841 MicroAPI::Adds(exp0FP32, exp0FP32, FP32_BIAS, pregAll32);
840 MicroAPI::ShiftLefts(exp0FP32, exp0FP32, SHR_NUM_FOR_FP32, pregAll32);842 MicroAPI::ShiftLefts(exp0FP32, exp0FP32, SHR_NUM_FOR_FP32, pregAll32);
841- MicroAPI::CompareScalar<float, CMPMODE::LT>(specialMask, Reg, 0, pregAll32);843+ MicroAPI::Compares<float, CMPMODE::LT>(specialMask, Reg, 0, pregAll32);
842 MicroAPI::Truncate<float, roundMode>(Reg, Reg, pregAll32);844 MicroAPI::Truncate<float, roundMode>(Reg, Reg, pregAll32);
843 MicroAPI::Mul(Reg, Reg, (MicroAPI::RegTensor<float>&)exp0FP32, pregAll32);845 MicroAPI::Mul(Reg, Reg, (MicroAPI::RegTensor<float>&)exp0FP32, pregAll32);
844 846 
845- MicroAPI::CompareScalar<float, CMPMODE::EQ>(zeroMask, Reg, 0, pregAll32);847+ MicroAPI::Compares<float, CMPMODE::EQ>(zeroMask, Reg, 0, pregAll32);
846- MicroAPI::MaskAnd(zeroMask, specialMask, zeroMask, pregAll32);848+ MicroAPI::And(zeroMask, specialMask, zeroMask, pregAll32);
847- MicroAPI::MaskOr(zeroMask, negInfMask, zeroMask, pregAll32);849+ MicroAPI::Or(zeroMask, negInfMask, zeroMask, pregAll32);
848 MicroAPI::Select<int32_t>((MicroAPI::RegTensor<int32_t>&)Reg, negZero, (MicroAPI::RegTensor<int32_t>&)Reg,850 MicroAPI::Select<int32_t>((MicroAPI::RegTensor<int32_t>&)Reg, negZero, (MicroAPI::RegTensor<int32_t>&)Reg,
849 zeroMask);851 zeroMask);
850}852}
Mquant/dynamic_mx_quant/op_kernel/arch35/dynamic_mx_quant_common.h+8-8
@@ -222,8 +222,8 @@ __aicore__ inline void CalcElement(AscendC::Reg::RegTensor<inType>& in, AscendC:
222 AscendC::Reg::MaskReg zeroNegMask;222 AscendC::Reg::MaskReg zeroNegMask;
223 AscendC::Reg::RegTensor<int32_t> negZero;223 AscendC::Reg::RegTensor<int32_t> negZero;
224 AscendC::Reg::Duplicate(negZero, FP32_NEG_ZERO_BITS);224 AscendC::Reg::Duplicate(negZero, FP32_NEG_ZERO_BITS);
225- AscendC::Reg::CompareScalar<int32_t, AscendC::CMPMODE::EQ>(zeroNegMask, (AscendC::Reg::RegTensor<int32_t>&)in,225+ AscendC::Reg::Compares<int32_t, AscendC::CMPMODE::EQ>(zeroNegMask, (AscendC::Reg::RegTensor<int32_t>&)in,
226- FP32_NEG_ZERO_BITS, mask);226+ FP32_NEG_ZERO_BITS, mask);
227 if constexpr (IsSame<outType, fp4x2_e2m1_t>::value) {227 if constexpr (IsSame<outType, fp4x2_e2m1_t>::value) {
228 AscendC::Reg::RegTensor<int32_t> exp1;228 AscendC::Reg::RegTensor<int32_t> exp1;
229 AscendC::Reg::RegTensor<int32_t> exp2;229 AscendC::Reg::RegTensor<int32_t> exp2;
@@ -239,19 +239,19 @@ __aicore__ inline void CalcElement(AscendC::Reg::RegTensor<inType>& in, AscendC:
239 AscendC::Reg::Mul(y1, in, (AscendC::Reg::RegTensor<float>&)exp2, mask);239 AscendC::Reg::Mul(y1, in, (AscendC::Reg::RegTensor<float>&)exp2, mask);
240 AscendC::Reg::Adds(exp1, exp1, FP32_BIAS_VALUE, mask);240 AscendC::Reg::Adds(exp1, exp1, FP32_BIAS_VALUE, mask);
241 AscendC::Reg::ShiftLefts(exp1, exp1, FP32_SHR_NUM, mask);241 AscendC::Reg::ShiftLefts(exp1, exp1, FP32_SHR_NUM, mask);
242- AscendC::Reg::CompareScalar<float, AscendC::CMPMODE::LT>(negValueMask, y1, 0, mask);242+ AscendC::Reg::Compares<float, AscendC::CMPMODE::LT>(negValueMask, y1, 0, mask);
243 AscendC::Reg::Truncate<float, roundMode>(y1, y1, mask);243 AscendC::Reg::Truncate<float, roundMode>(y1, y1, mask);
244 AscendC::Reg::Mul(in, y1, (AscendC::Reg::RegTensor<float>&)exp1, mask);244 AscendC::Reg::Mul(in, y1, (AscendC::Reg::RegTensor<float>&)exp1, mask);
245 } else {245 } else {
246 AscendC::Reg::Muls(y1, in, FP4_SCALE_FACTOR, mask);246 AscendC::Reg::Muls(y1, in, FP4_SCALE_FACTOR, mask);
247- AscendC::Reg::CompareScalar<float, AscendC::CMPMODE::LT>(negValueMask, y1, 0, mask);247+ AscendC::Reg::Compares<float, AscendC::CMPMODE::LT>(negValueMask, y1, 0, mask);
248 AscendC::Reg::Truncate<float, roundMode>(y1, y1, mask);248 AscendC::Reg::Truncate<float, roundMode>(y1, y1, mask);
249 AscendC::Reg::Muls(in, y1, FP4_INV_SCALE_FACTOR, mask);249 AscendC::Reg::Muls(in, y1, FP4_INV_SCALE_FACTOR, mask);
250 }250 }
251- AscendC::Reg::CompareScalar<float, AscendC::CMPMODE::EQ>(zeroMask, in, 0, mask);251+ AscendC::Reg::Compares<float, AscendC::CMPMODE::EQ>(zeroMask, in, 0, mask);
252- AscendC::Reg::MaskAnd(negZeroMask, zeroMask, negValueMask, mask);252+ AscendC::Reg::And(negZeroMask, zeroMask, negValueMask, mask);
253- AscendC::Reg::MaskOr(zeroMask, negZeroMask, zeroNegMask, mask);253+ AscendC::Reg::Or(zeroMask, negZeroMask, zeroNegMask, mask);
254- AscendC::Reg::Copy((AscendC::Reg::RegTensor<int32_t>&)in, negZero, zeroMask);254+ AscendC::Reg::Move((AscendC::Reg::RegTensor<int32_t>&)in, negZero, zeroMask);
255}255}
256 256 
257template <AscendC::RoundMode roundMode, typename outType, typename inType, typename calcTypeInt>257template <AscendC::RoundMode roundMode, typename outType, typename inType, typename calcTypeInt>
Mquant/dynamic_mx_quant/op_kernel/arch35/dynamic_mx_quant_not_tail_axis.h+53-53
@@ -319,7 +319,7 @@ __aicore__ inline void DynamicMxQuantNotTailAxis<T, U, ISTAIL>::ComputeOcp(int64
319 Reg::RegTensor<calcTypeInt> nan;319 Reg::RegTensor<calcTypeInt> nan;
320 Reg::RegTensor<calcTypeInt> subNumForScale;320 Reg::RegTensor<calcTypeInt> subNumForScale;
321 Reg::RegTensor<uint8_t> out;321 Reg::RegTensor<uint8_t> out;
322- Reg::UnalignReg u1;322+ Reg::UnalignRegForStore u1;
323 Reg::MaskReg infMask;323 Reg::MaskReg infMask;
324 Reg::MaskReg zeroMask;324 Reg::MaskReg zeroMask;
325 Reg::MaskReg invalidDataMask;325 Reg::MaskReg invalidDataMask;
@@ -370,8 +370,8 @@ __aicore__ inline void DynamicMxQuantNotTailAxis<T, U, ISTAIL>::ComputeOcp(int64
370 } else if constexpr (IsSame<T, bfloat16_t>::value) {370 } else if constexpr (IsSame<T, bfloat16_t>::value) {
371 Reg::Pack(mxScale, mxScaleInt);371 Reg::Pack(mxScale, mxScaleInt);
372 }372 }
373- Reg::DataCopyUnAlign(mxScaleAddr, mxScale, u1, vfLen);373+ Reg::StoreUnAlign(mxScaleAddr, mxScale, u1, vfLen);
374- Reg::DataCopyUnAlignPost(mxScaleAddr, u1, 0);374+ Reg::StoreUnAlignPost(mxScaleAddr, u1, 0);
375 // 求1/scale375 // 求1/scale
376 Reg::Compare<calcTypeInt, CMPMODE::EQ>(specialDataMask, expMax, bias, p0);376 Reg::Compare<calcTypeInt, CMPMODE::EQ>(specialDataMask, expMax, bias, p0);
377 Reg::Compare<calcTypeInt, CMPMODE::NE>(zeroMask, expMax, zero, p0);377 Reg::Compare<calcTypeInt, CMPMODE::NE>(zeroMask, expMax, zero, p0);
@@ -385,8 +385,8 @@ __aicore__ inline void DynamicMxQuantNotTailAxis<T, U, ISTAIL>::ComputeOcp(int64
385 this->template LoadData<calcType>(xAddr, j * dataLen + i * vfLen, x, p0);385 this->template LoadData<calcType>(xAddr, j * dataLen + i * vfLen, x, p0);
386 CalcElement<roundMode, U, calcType, calcTypeInt>(x, scaleReprocal, maxEle, out, p0);386 CalcElement<roundMode, U, calcType, calcTypeInt>(x, scaleReprocal, maxEle, out, p0);
387 auto addr = yAddr + (j * outDataLenAlign + i * vfLen) / DIGIT_TWO;387 auto addr = yAddr + (j * outDataLenAlign + i * vfLen) / DIGIT_TWO;
388- Reg::DataCopyUnAlign(addr, out, u1, vfLen / DIGIT_TWO);388+ Reg::StoreUnAlign(addr, out, u1, vfLen / DIGIT_TWO);
389- Reg::DataCopyUnAlignPost(addr, u1, 0);389+ Reg::StoreUnAlignPost(addr, u1, 0);
390 }390 }
391 }391 }
392 if (tailVfLen != 0) {392 if (tailVfLen != 0) {
@@ -416,8 +416,8 @@ __aicore__ inline void DynamicMxQuantNotTailAxis<T, U, ISTAIL>::ComputeOcp(int64
416 } else if constexpr (IsSame<T, bfloat16_t>::value) {416 } else if constexpr (IsSame<T, bfloat16_t>::value) {
417 Reg::Pack(mxScale, mxScaleInt);417 Reg::Pack(mxScale, mxScaleInt);
418 }418 }
419- Reg::DataCopyUnAlign(mxScaleAddr, mxScale, u1, tailVfLen);419+ Reg::StoreUnAlign(mxScaleAddr, mxScale, u1, tailVfLen);
420- Reg::DataCopyUnAlignPost(mxScaleAddr, u1, 0);420+ Reg::StoreUnAlignPost(mxScaleAddr, u1, 0);
421 // 求1/scale421 // 求1/scale
422 Reg::Compare<calcTypeInt, CMPMODE::NE>(zeroMask, expMax, zero, p1);422 Reg::Compare<calcTypeInt, CMPMODE::NE>(zeroMask, expMax, zero, p1);
423 Reg::Compare<calcTypeInt, CMPMODE::EQ>(specialDataMask, expMax, bias, p1);423 Reg::Compare<calcTypeInt, CMPMODE::EQ>(specialDataMask, expMax, bias, p1);
@@ -434,8 +434,8 @@ __aicore__ inline void DynamicMxQuantNotTailAxis<T, U, ISTAIL>::ComputeOcp(int64
434 this->template LoadData<calcType>(xAddr, regLoop * vfLen + j * dataLen, x, p1);434 this->template LoadData<calcType>(xAddr, regLoop * vfLen + j * dataLen, x, p1);
435 CalcElement<roundMode, U, calcType, calcTypeInt>(x, scaleReprocal, maxEle, out, p1);435 CalcElement<roundMode, U, calcType, calcTypeInt>(x, scaleReprocal, maxEle, out, p1);
436 auto addr = yAddr + (regLoop * vfLen + j * outDataLenAlign) / DIGIT_TWO;436 auto addr = yAddr + (regLoop * vfLen + j * outDataLenAlign) / DIGIT_TWO;
437- Reg::DataCopyUnAlign(addr, out, u1, tailVfLen / DIGIT_TWO);437+ Reg::StoreUnAlign(addr, out, u1, tailVfLen / DIGIT_TWO);
438- Reg::DataCopyUnAlignPost(addr, u1, 0);438+ Reg::StoreUnAlignPost(addr, u1, 0);
439 }439 }
440 }440 }
441 } else {441 } else {
@@ -463,8 +463,8 @@ __aicore__ inline void DynamicMxQuantNotTailAxis<T, U, ISTAIL>::ComputeOcp(int64
463 } else if constexpr (IsSame<T, bfloat16_t>::value) {463 } else if constexpr (IsSame<T, bfloat16_t>::value) {
464 Reg::Pack(mxScale, mxScaleInt);464 Reg::Pack(mxScale, mxScaleInt);
465 }465 }
466- Reg::DataCopyUnAlign(mxScaleAddr, mxScale, u1, vfLen);466+ Reg::StoreUnAlign(mxScaleAddr, mxScale, u1, vfLen);
467- Reg::DataCopyUnAlignPost(mxScaleAddr, u1, 0);467+ Reg::StoreUnAlignPost(mxScaleAddr, u1, 0);
468 // 求1/scale468 // 求1/scale
469 Reg::Compare<calcTypeInt, CMPMODE::NE>(zeroMask, expMax, zero, p0);469 Reg::Compare<calcTypeInt, CMPMODE::NE>(zeroMask, expMax, zero, p0);
470 Reg::Compare<calcTypeInt, CMPMODE::EQ>(specialDataMask, expMax, bias, p0);470 Reg::Compare<calcTypeInt, CMPMODE::EQ>(specialDataMask, expMax, bias, p0);
@@ -478,8 +478,8 @@ __aicore__ inline void DynamicMxQuantNotTailAxis<T, U, ISTAIL>::ComputeOcp(int64
478 this->template LoadData<calcType>(xAddr, j * dataLen + i * vfLen, x, p0);478 this->template LoadData<calcType>(xAddr, j * dataLen + i * vfLen, x, p0);
479 CalcElement<roundMode, U, calcType, calcTypeInt>(x, scaleReprocal, maxEle, out, p0);479 CalcElement<roundMode, U, calcType, calcTypeInt>(x, scaleReprocal, maxEle, out, p0);
480 auto addr = yAddr + (j * outDataLenAlign + i * vfLen) / DIGIT_TWO;480 auto addr = yAddr + (j * outDataLenAlign + i * vfLen) / DIGIT_TWO;
481- Reg::DataCopyUnAlign(addr, out, u1, vfLen / DIGIT_TWO);481+ Reg::StoreUnAlign(addr, out, u1, vfLen / DIGIT_TWO);
482- Reg::DataCopyUnAlignPost(addr, u1, 0);482+ Reg::StoreUnAlignPost(addr, u1, 0);
483 }483 }
484 }484 }
485 if (tailVfLen != 0) {485 if (tailVfLen != 0) {
@@ -506,8 +506,8 @@ __aicore__ inline void DynamicMxQuantNotTailAxis<T, U, ISTAIL>::ComputeOcp(int64
506 } else if constexpr (IsSame<T, bfloat16_t>::value) {506 } else if constexpr (IsSame<T, bfloat16_t>::value) {
507 Reg::Pack(mxScale, mxScaleInt);507 Reg::Pack(mxScale, mxScaleInt);
508 }508 }
509- Reg::DataCopyUnAlign(mxScaleAddr, mxScale, u1, tailVfLen);509+ Reg::StoreUnAlign(mxScaleAddr, mxScale, u1, tailVfLen);
510- Reg::DataCopyUnAlignPost(mxScaleAddr, u1, 0);510+ Reg::StoreUnAlignPost(mxScaleAddr, u1, 0);
511 // 求1/scale511 // 求1/scale
512 Reg::Compare<calcTypeInt, CMPMODE::NE>(zeroMask, expMax, zero, p1);512 Reg::Compare<calcTypeInt, CMPMODE::NE>(zeroMask, expMax, zero, p1);
513 Reg::Compare<calcTypeInt, CMPMODE::EQ>(specialDataMask, expMax, bias, p1);513 Reg::Compare<calcTypeInt, CMPMODE::EQ>(specialDataMask, expMax, bias, p1);
@@ -524,8 +524,8 @@ __aicore__ inline void DynamicMxQuantNotTailAxis<T, U, ISTAIL>::ComputeOcp(int64
524 this->template LoadData<calcType>(xAddr, regLoop * vfLen + j * dataLen, x, p1);524 this->template LoadData<calcType>(xAddr, regLoop * vfLen + j * dataLen, x, p1);
525 CalcElement<roundMode, U, calcType, calcTypeInt>(x, scaleReprocal, maxEle, out, p1);525 CalcElement<roundMode, U, calcType, calcTypeInt>(x, scaleReprocal, maxEle, out, p1);
526 auto addr = yAddr + (regLoop * vfLen + j * outDataLenAlign) / DIGIT_TWO;526 auto addr = yAddr + (regLoop * vfLen + j * outDataLenAlign) / DIGIT_TWO;
527- Reg::DataCopyUnAlign(addr, out, u1, tailVfLen / DIGIT_TWO);527+ Reg::StoreUnAlign(addr, out, u1, tailVfLen / DIGIT_TWO);
528- Reg::DataCopyUnAlignPost(addr, u1, 0);528+ Reg::StoreUnAlignPost(addr, u1, 0);
529 }529 }
530 }530 }
531 }531 }
@@ -586,8 +586,8 @@ __aicore__ inline void DynamicMxQuantNotTailAxis<T, U, ISTAIL>::ComputeCuBLAS(in
586 Reg::RegTensor<uint32_t> expAndreShareExpFP32RegTensor;586 Reg::RegTensor<uint32_t> expAndreShareExpFP32RegTensor;
587 Reg::RegTensor<uint32_t> manAndmxScaleFP32RegTensor;587 Reg::RegTensor<uint32_t> manAndmxScaleFP32RegTensor;
588 588 
589- Reg::UnalignReg u0;589+ Reg::UnalignRegForLoad u0;
590- Reg::UnalignReg u1;590+ Reg::UnalignRegForStore u1;
591 Reg::MaskReg p2;591 Reg::MaskReg p2;
592 Reg::MaskReg infMask;592 Reg::MaskReg infMask;
593 Reg::MaskReg specialDataMask;593 Reg::MaskReg specialDataMask;
@@ -642,12 +642,12 @@ __aicore__ inline void DynamicMxQuantNotTailAxis<T, U, ISTAIL>::ComputeCuBLAS(in
642 // And获取尾数位642 // And获取尾数位
643 AscendC::Reg::And(manFP32Reg, xFP32MaxReg, infForFP32Reg, preMaskScale);643 AscendC::Reg::And(manFP32Reg, xFP32MaxReg, infForFP32Reg, preMaskScale);
644 AscendC::Reg::And(manFP32Reg2, xFP32MaxReg2, infForFP32Reg, preMaskScale);644 AscendC::Reg::And(manFP32Reg2, xFP32MaxReg2, infForFP32Reg, preMaskScale);
645- AscendC::Reg::CompareScalar<uint32_t, CMPMODE::GT>(p1, expFP32Reg, FP32_NUMBER_ZERO, preMaskScale);645+ AscendC::Reg::Compares<uint32_t, CMPMODE::GT>(p1, expFP32Reg, FP32_NUMBER_ZERO, preMaskScale);
646- AscendC::Reg::CompareScalar<uint32_t, CMPMODE::LT>(p1, expFP32Reg, FP32_NUMBER_254, p1);646+ AscendC::Reg::Compares<uint32_t, CMPMODE::LT>(p1, expFP32Reg, FP32_NUMBER_254, p1);
647- AscendC::Reg::CompareScalar<uint32_t, CMPMODE::GT>(p1, manFP32Reg, FP32_NUMBER_ZERO, p1);647+ AscendC::Reg::Compares<uint32_t, CMPMODE::GT>(p1, manFP32Reg, FP32_NUMBER_ZERO, p1);
648- AscendC::Reg::CompareScalar<uint32_t, CMPMODE::GT>(p2, expFP32Reg2, FP32_NUMBER_ZERO, preMaskScale);648+ AscendC::Reg::Compares<uint32_t, CMPMODE::GT>(p2, expFP32Reg2, FP32_NUMBER_ZERO, preMaskScale);
649- AscendC::Reg::CompareScalar<uint32_t, CMPMODE::LT>(p2, expFP32Reg2, FP32_NUMBER_254, p2);649+ AscendC::Reg::Compares<uint32_t, CMPMODE::LT>(p2, expFP32Reg2, FP32_NUMBER_254, p2);
650- AscendC::Reg::CompareScalar<uint32_t, CMPMODE::GT>(p2, manFP32Reg2, FP32_NUMBER_ZERO, p2);650+ AscendC::Reg::Compares<uint32_t, CMPMODE::GT>(p2, manFP32Reg2, FP32_NUMBER_ZERO, p2);
651 AscendC::Reg::Adds(extractExpReg, expFP32Reg, 1, preMaskScale);651 AscendC::Reg::Adds(extractExpReg, expFP32Reg, 1, preMaskScale);
652 AscendC::Reg::Adds(extractExpReg2, expFP32Reg2, 1, preMaskScale);652 AscendC::Reg::Adds(extractExpReg2, expFP32Reg2, 1, preMaskScale);
653 // 根据情况选择指数位是否加一653 // 根据情况选择指数位是否加一
@@ -657,8 +657,8 @@ __aicore__ inline void DynamicMxQuantNotTailAxis<T, U, ISTAIL>::ComputeCuBLAS(in
657 (Reg::RegTensor<uint16_t>&)expFP32Reg, (Reg::RegTensor<uint16_t>&)expFP32Reg2,657 (Reg::RegTensor<uint16_t>&)expFP32Reg, (Reg::RegTensor<uint16_t>&)expFP32Reg2,
658 (Reg::RegTensor<uint16_t>&)expFP32Reg, (Reg::RegTensor<uint16_t>&)expFP32Reg2);658 (Reg::RegTensor<uint16_t>&)expFP32Reg, (Reg::RegTensor<uint16_t>&)expFP32Reg2);
659 AscendC::Reg::Pack(mxScale, (Reg::RegTensor<uint16_t>&)expFP32Reg);659 AscendC::Reg::Pack(mxScale, (Reg::RegTensor<uint16_t>&)expFP32Reg);
660- AscendC::Reg::DataCopyUnAlign(mxScaleAddr, mxScale, u1, vfLen);660+ AscendC::Reg::StoreUnAlign(mxScaleAddr, mxScale, u1, vfLen);
661- AscendC::Reg::DataCopyUnAlignPost(mxScaleAddr, u1, 0);661+ AscendC::Reg::StoreUnAlignPost(mxScaleAddr, u1, 0);
662 AscendC::Reg::ShiftLefts((Reg::RegTensor<uint16_t>&)expFP32Reg, (Reg::RegTensor<uint16_t>&)expFP32Reg,662 AscendC::Reg::ShiftLefts((Reg::RegTensor<uint16_t>&)expFP32Reg, (Reg::RegTensor<uint16_t>&)expFP32Reg,
663 BF16_SHR_NUM, pregAll16);663 BF16_SHR_NUM, pregAll16);
664 AscendC::Reg::Compare<calcTypeInt, CMPMODE::NE>(infMask, (Reg::RegTensor<uint16_t>&)expFP32Reg,664 AscendC::Reg::Compare<calcTypeInt, CMPMODE::NE>(infMask, (Reg::RegTensor<uint16_t>&)expFP32Reg,
@@ -673,8 +673,8 @@ __aicore__ inline void DynamicMxQuantNotTailAxis<T, U, ISTAIL>::ComputeCuBLAS(in
673 CalcElement<roundMode, U, calcType, calcTypeInt>(xReg, scaleReprocal, infForXReg, out,673 CalcElement<roundMode, U, calcType, calcTypeInt>(xReg, scaleReprocal, infForXReg, out,
674 preMaskScale2);674 preMaskScale2);
675 auto addr = yAddr + (j * outDataLenAlign + i * vfLen) / DIGIT_TWO;675 auto addr = yAddr + (j * outDataLenAlign + i * vfLen) / DIGIT_TWO;
676- AscendC::Reg::DataCopyUnAlign(addr, out, u1, vfLen / DIGIT_TWO);676+ AscendC::Reg::StoreUnAlign(addr, out, u1, vfLen / DIGIT_TWO);
677- AscendC::Reg::DataCopyUnAlignPost(addr, u1, 0);677+ AscendC::Reg::StoreUnAlignPost(addr, u1, 0);
678 }678 }
679 } else if constexpr (IsSame<T, half>::value) {679 } else if constexpr (IsSame<T, half>::value) {
680 AscendC::Reg::Mul((AscendC::Reg::RegTensor<float>&)xFP32MaxReg,680 AscendC::Reg::Mul((AscendC::Reg::RegTensor<float>&)xFP32MaxReg,
@@ -683,17 +683,17 @@ __aicore__ inline void DynamicMxQuantNotTailAxis<T, U, ISTAIL>::ComputeCuBLAS(in
683 AscendC::Reg::ShiftRights(expFP32Reg, xFP32MaxReg, FP32_SHR_NUM, preMaskScale);683 AscendC::Reg::ShiftRights(expFP32Reg, xFP32MaxReg, FP32_SHR_NUM, preMaskScale);
684 // And获取尾数位684 // And获取尾数位
685 AscendC::Reg::And(manFP32Reg, xFP32MaxReg, infForFP32Reg, preMaskScale);685 AscendC::Reg::And(manFP32Reg, xFP32MaxReg, infForFP32Reg, preMaskScale);
686- AscendC::Reg::CompareScalar<uint32_t, CMPMODE::GT>(p1, expFP32Reg, FP32_NUMBER_ZERO, preMaskScale);686+ AscendC::Reg::Compares<uint32_t, CMPMODE::GT>(p1, expFP32Reg, FP32_NUMBER_ZERO, preMaskScale);
687- AscendC::Reg::CompareScalar<uint32_t, CMPMODE::LT>(p1, expFP32Reg, FP32_NUMBER_254, p1);687+ AscendC::Reg::Compares<uint32_t, CMPMODE::LT>(p1, expFP32Reg, FP32_NUMBER_254, p1);
688- AscendC::Reg::CompareScalar<uint32_t, CMPMODE::GT>(p1, manFP32Reg, FP32_NUMBER_ZERO, p1);688+ AscendC::Reg::Compares<uint32_t, CMPMODE::GT>(p1, manFP32Reg, FP32_NUMBER_ZERO, p1);
689 AscendC::Reg::Adds(extractExpReg, expFP32Reg, 1, preMaskScale);689 AscendC::Reg::Adds(extractExpReg, expFP32Reg, 1, preMaskScale);
690 // 根据情况选择指数位是否加一690 // 根据情况选择指数位是否加一
691 AscendC::Reg::Select<uint32_t>(expFP32Reg, extractExpReg, expFP32Reg, p1);691 AscendC::Reg::Select<uint32_t>(expFP32Reg, extractExpReg, expFP32Reg, p1);
692 692 
693 AscendC::Reg::Pack(expBF16Reg, expFP32Reg);693 AscendC::Reg::Pack(expBF16Reg, expFP32Reg);
694 AscendC::Reg::Pack(mxScale, expBF16Reg);694 AscendC::Reg::Pack(mxScale, expBF16Reg);
695- AscendC::Reg::DataCopyUnAlign(mxScaleAddr, mxScale, u1, vfLen);695+ AscendC::Reg::StoreUnAlign(mxScaleAddr, mxScale, u1, vfLen);
696- AscendC::Reg::DataCopyUnAlignPost(mxScaleAddr, u1, 0);696+ AscendC::Reg::StoreUnAlignPost(mxScaleAddr, u1, 0);
697 697 
698 AscendC::Reg::ShiftLefts(expFP32Reg, expFP32Reg, FP32_SHR_NUM, preMaskScale);698 AscendC::Reg::ShiftLefts(expFP32Reg, expFP32Reg, FP32_SHR_NUM, preMaskScale);
699 699 
@@ -708,8 +708,8 @@ __aicore__ inline void DynamicMxQuantNotTailAxis<T, U, ISTAIL>::ComputeCuBLAS(in
708 this->template LoadData<calcType>(xAddr, j * dataLen + i * vfLen, xReg, p0);708 this->template LoadData<calcType>(xAddr, j * dataLen + i * vfLen, xReg, p0);
709 CalcElement<roundMode, U, calcType, calcTypeInt>(xReg, scaleReprocal, infForXReg, out, p0);709 CalcElement<roundMode, U, calcType, calcTypeInt>(xReg, scaleReprocal, infForXReg, out, p0);
710 auto addr = yAddr + (j * outDataLenAlign + i * vfLen) / DIGIT_TWO;710 auto addr = yAddr + (j * outDataLenAlign + i * vfLen) / DIGIT_TWO;
711- AscendC::Reg::DataCopyUnAlign(addr, out, u1, vfLen / DIGIT_TWO);711+ AscendC::Reg::StoreUnAlign(addr, out, u1, vfLen / DIGIT_TWO);
712- AscendC::Reg::DataCopyUnAlignPost(addr, u1, 0);712+ AscendC::Reg::StoreUnAlignPost(addr, u1, 0);
713 }713 }
714 }714 }
715 }715 }
@@ -739,12 +739,12 @@ __aicore__ inline void DynamicMxQuantNotTailAxis<T, U, ISTAIL>::ComputeCuBLAS(in
739 // And获取尾数位739 // And获取尾数位
740 AscendC::Reg::And(manFP32Reg, xFP32MaxReg, infForFP32Reg, preMaskScale);740 AscendC::Reg::And(manFP32Reg, xFP32MaxReg, infForFP32Reg, preMaskScale);
741 AscendC::Reg::And(manFP32Reg2, xFP32MaxReg2, infForFP32Reg, preMaskScale);741 AscendC::Reg::And(manFP32Reg2, xFP32MaxReg2, infForFP32Reg, preMaskScale);
742- AscendC::Reg::CompareScalar<uint32_t, CMPMODE::GT>(p1, expFP32Reg, FP32_NUMBER_ZERO, preMaskScale);742+ AscendC::Reg::Compares<uint32_t, CMPMODE::GT>(p1, expFP32Reg, FP32_NUMBER_ZERO, preMaskScale);
743- AscendC::Reg::CompareScalar<uint32_t, CMPMODE::LT>(p1, expFP32Reg, FP32_NUMBER_254, p1);743+ AscendC::Reg::Compares<uint32_t, CMPMODE::LT>(p1, expFP32Reg, FP32_NUMBER_254, p1);
744- AscendC::Reg::CompareScalar<uint32_t, CMPMODE::GT>(p1, manFP32Reg, FP32_NUMBER_ZERO, p1);744+ AscendC::Reg::Compares<uint32_t, CMPMODE::GT>(p1, manFP32Reg, FP32_NUMBER_ZERO, p1);
745- AscendC::Reg::CompareScalar<uint32_t, CMPMODE::GT>(p2, expFP32Reg2, FP32_NUMBER_ZERO, preMaskScale);745+ AscendC::Reg::Compares<uint32_t, CMPMODE::GT>(p2, expFP32Reg2, FP32_NUMBER_ZERO, preMaskScale);
746- AscendC::Reg::CompareScalar<uint32_t, CMPMODE::LT>(p2, expFP32Reg2, FP32_NUMBER_254, p2);746+ AscendC::Reg::Compares<uint32_t, CMPMODE::LT>(p2, expFP32Reg2, FP32_NUMBER_254, p2);
747- AscendC::Reg::CompareScalar<uint32_t, CMPMODE::GT>(p2, manFP32Reg2, FP32_NUMBER_ZERO, p2);747+ AscendC::Reg::Compares<uint32_t, CMPMODE::GT>(p2, manFP32Reg2, FP32_NUMBER_ZERO, p2);
748 AscendC::Reg::Adds(extractExpReg, expFP32Reg, 1, preMaskScale);748 AscendC::Reg::Adds(extractExpReg, expFP32Reg, 1, preMaskScale);
749 AscendC::Reg::Adds(extractExpReg2, expFP32Reg2, 1, preMaskScale);749 AscendC::Reg::Adds(extractExpReg2, expFP32Reg2, 1, preMaskScale);
750 // 根据情况选择指数位是否加一750 // 根据情况选择指数位是否加一
@@ -754,8 +754,8 @@ __aicore__ inline void DynamicMxQuantNotTailAxis<T, U, ISTAIL>::ComputeCuBLAS(in
754 (Reg::RegTensor<uint16_t>&)expFP32Reg, (Reg::RegTensor<uint16_t>&)expFP32Reg2,754 (Reg::RegTensor<uint16_t>&)expFP32Reg, (Reg::RegTensor<uint16_t>&)expFP32Reg2,
755 (Reg::RegTensor<uint16_t>&)expFP32Reg, (Reg::RegTensor<uint16_t>&)expFP32Reg2);755 (Reg::RegTensor<uint16_t>&)expFP32Reg, (Reg::RegTensor<uint16_t>&)expFP32Reg2);
756 AscendC::Reg::Pack(mxScale, (Reg::RegTensor<uint16_t>&)expFP32Reg);756 AscendC::Reg::Pack(mxScale, (Reg::RegTensor<uint16_t>&)expFP32Reg);
757- AscendC::Reg::DataCopyUnAlign(mxScaleAddr, mxScale, u1, tailVfLen);757+ AscendC::Reg::StoreUnAlign(mxScaleAddr, mxScale, u1, tailVfLen);
758- AscendC::Reg::DataCopyUnAlignPost(mxScaleAddr, u1, 0);758+ AscendC::Reg::StoreUnAlignPost(mxScaleAddr, u1, 0);
759 759 
760 AscendC::Reg::ShiftLefts((Reg::RegTensor<uint16_t>&)expFP32Reg, (Reg::RegTensor<uint16_t>&)expFP32Reg,760 AscendC::Reg::ShiftLefts((Reg::RegTensor<uint16_t>&)expFP32Reg, (Reg::RegTensor<uint16_t>&)expFP32Reg,
761 BF16_SHR_NUM, pregAll16);761 BF16_SHR_NUM, pregAll16);
@@ -774,8 +774,8 @@ __aicore__ inline void DynamicMxQuantNotTailAxis<T, U, ISTAIL>::ComputeCuBLAS(in
774 CalcElement<roundMode, U, calcType, calcTypeInt>(xReg, scaleReprocal, infForXReg, out,774 CalcElement<roundMode, U, calcType, calcTypeInt>(xReg, scaleReprocal, infForXReg, out,
775 preMaskScale2);775 preMaskScale2);
776 auto addr = yAddr + (regLoop * vfLen + j * outDataLenAlign) / DIGIT_TWO;776 auto addr = yAddr + (regLoop * vfLen + j * outDataLenAlign) / DIGIT_TWO;
777- Reg::DataCopyUnAlign(addr, out, u1, tailVfLen / DIGIT_TWO);777+ Reg::StoreUnAlign(addr, out, u1, tailVfLen / DIGIT_TWO);
778- Reg::DataCopyUnAlignPost(addr, u1, 0);778+ Reg::StoreUnAlignPost(addr, u1, 0);
779 }779 }
780 780 
781 } else if constexpr (IsSame<T, half>::value) {781 } else if constexpr (IsSame<T, half>::value) {
@@ -785,17 +785,17 @@ __aicore__ inline void DynamicMxQuantNotTailAxis<T, U, ISTAIL>::ComputeCuBLAS(in
785 785 
786 // And获取尾数位786 // And获取尾数位
787 AscendC::Reg::And(manFP32Reg, xFP32MaxReg, infForFP32Reg, preMaskScale);787 AscendC::Reg::And(manFP32Reg, xFP32MaxReg, infForFP32Reg, preMaskScale);
788- AscendC::Reg::CompareScalar<uint32_t, CMPMODE::GT>(p1, expFP32Reg, FP32_NUMBER_ZERO, preMaskScale);788+ AscendC::Reg::Compares<uint32_t, CMPMODE::GT>(p1, expFP32Reg, FP32_NUMBER_ZERO, preMaskScale);
789- AscendC::Reg::CompareScalar<uint32_t, CMPMODE::LT>(p1, expFP32Reg, FP32_NUMBER_254, p1);789+ AscendC::Reg::Compares<uint32_t, CMPMODE::LT>(p1, expFP32Reg, FP32_NUMBER_254, p1);
790- AscendC::Reg::CompareScalar<uint32_t, CMPMODE::GT>(p1, manFP32Reg, FP32_NUMBER_ZERO, p1);790+ AscendC::Reg::Compares<uint32_t, CMPMODE::GT>(p1, manFP32Reg, FP32_NUMBER_ZERO, p1);
791 AscendC::Reg::Adds(extractExpReg, expFP32Reg, 1, preMaskScale);791 AscendC::Reg::Adds(extractExpReg, expFP32Reg, 1, preMaskScale);
792 // 根据情况选择指数位是否加一792 // 根据情况选择指数位是否加一
793 AscendC::Reg::Select<uint32_t>(expFP32Reg, extractExpReg, expFP32Reg, p1);793 AscendC::Reg::Select<uint32_t>(expFP32Reg, extractExpReg, expFP32Reg, p1);
794 794 
795 AscendC::Reg::Pack(expBF16Reg, expFP32Reg);795 AscendC::Reg::Pack(expBF16Reg, expFP32Reg);
796 AscendC::Reg::Pack(mxScale, expBF16Reg);796 AscendC::Reg::Pack(mxScale, expBF16Reg);
797- AscendC::Reg::DataCopyUnAlign(mxScaleAddr, mxScale, u1, tailVfLen);797+ AscendC::Reg::StoreUnAlign(mxScaleAddr, mxScale, u1, tailVfLen);
798- AscendC::Reg::DataCopyUnAlignPost(mxScaleAddr, u1, 0);798+ AscendC::Reg::StoreUnAlignPost(mxScaleAddr, u1, 0);
799 799 
800 AscendC::Reg::ShiftLefts(expFP32Reg, expFP32Reg, FP32_SHR_NUM, preMaskScale);800 AscendC::Reg::ShiftLefts(expFP32Reg, expFP32Reg, FP32_SHR_NUM, preMaskScale);
801 // 求1/scale801 // 求1/scale
@@ -814,8 +814,8 @@ __aicore__ inline void DynamicMxQuantNotTailAxis<T, U, ISTAIL>::ComputeCuBLAS(in
814 this->template LoadData<calcType>(xAddr, regLoop * vfLen + j * dataLen, xReg, p2);814 this->template LoadData<calcType>(xAddr, regLoop * vfLen + j * dataLen, xReg, p2);
815 CalcElement<roundMode, U, calcType, calcTypeInt>(xReg, scaleReprocal, infForXReg, out, p2);815 CalcElement<roundMode, U, calcType, calcTypeInt>(xReg, scaleReprocal, infForXReg, out, p2);
816 auto addr = yAddr + (regLoop * vfLen + j * outDataLenAlign) / DIGIT_TWO;816 auto addr = yAddr + (regLoop * vfLen + j * outDataLenAlign) / DIGIT_TWO;
817- Reg::DataCopyUnAlign(addr, out, u1, tailVfLen / DIGIT_TWO);817+ Reg::StoreUnAlign(addr, out, u1, tailVfLen / DIGIT_TWO);
818- Reg::DataCopyUnAlignPost(addr, u1, 0);818+ Reg::StoreUnAlignPost(addr, u1, 0);
819 }819 }
820 }820 }
821 }821 }
Mquant/dynamic_mx_quant/op_kernel/arch35/dynamic_mx_quant_not_tail_axis_base.h+6-6
@@ -214,20 +214,20 @@ template <typename ComType>
214__aicore__ inline void DynamicMxQuantBase<T, U, ISTAIL>::LoadData(__ubuf__ T* xAddr, uint64_t offset,214__aicore__ inline void DynamicMxQuantBase<T, U, ISTAIL>::LoadData(__ubuf__ T* xAddr, uint64_t offset,
215 Reg::RegTensor<ComType>& x, Reg::MaskReg& mask)215 Reg::RegTensor<ComType>& x, Reg::MaskReg& mask)
216{216{
217- Reg::UnalignReg uReg;217+ Reg::UnalignRegForLoad uReg;
218 if constexpr (IsSame<T, half>::value) {218 if constexpr (IsSame<T, half>::value) {
219 Reg::RegTensor<T> xFP16;219 Reg::RegTensor<T> xFP16;
220- Reg::DataCopyUnAlignPre(uReg, xAddr + offset);220+ Reg::LoadUnAlignPre(uReg, xAddr + offset);
221- Reg::DataCopyUnAlign(xFP16, uReg, xAddr + offset);221+ Reg::LoadUnAlign(xFP16, uReg, xAddr + offset);
222 static constexpr Reg::CastTrait castTraitHalf2Fp32 = {Reg::RegLayout::ZERO, Reg::SatMode::UNKNOWN,222 static constexpr Reg::CastTrait castTraitHalf2Fp32 = {Reg::RegLayout::ZERO, Reg::SatMode::UNKNOWN,
223 Reg::MaskMergeMode::ZEROING, AscendC::RoundMode::UNKNOWN};223 Reg::MaskMergeMode::ZEROING, AscendC::RoundMode::UNKNOWN};
224 Reg::UnPack((Reg::RegTensor<uint32_t>&)xFP16, (Reg::RegTensor<uint16_t>&)xFP16);224 Reg::UnPack((Reg::RegTensor<uint32_t>&)xFP16, (Reg::RegTensor<uint16_t>&)xFP16);
225 Reg::Cast<ComType, T, castTraitHalf2Fp32>(x, xFP16, mask);225 Reg::Cast<ComType, T, castTraitHalf2Fp32>(x, xFP16, mask);
226 } else {226 } else {
227- Reg::DataCopyUnAlignPre(uReg, xAddr + offset);227+ Reg::LoadUnAlignPre(uReg, xAddr + offset);
228- Reg::DataCopyUnAlign(x, uReg, xAddr + offset);228+ Reg::LoadUnAlign(x, uReg, xAddr + offset);
229 }229 }
230}230}
231 231 
232} // namespace DynamicMxQuant232} // namespace DynamicMxQuant
233-#endif // DYNAMIC_MX_QUANT_NOT_TAIL_AXIS_BASE_H233+#endif // DYNAMIC_MX_QUANT_NOT_TAIL_AXIS_BASE_H
Mquant/dynamic_mx_quant/op_kernel/arch35/dynamic_mx_quant_not_tail_axis_base_fp8.h+4-4
@@ -204,10 +204,10 @@ template <AscendC::RoundMode toBf16RoundMode, AscendC::RoundMode roundMode, cons
204__aicore__ inline void DynamicMxQuantBaseFP8<T, U, isTail>::LoadData(__ubuf__ T* xAddr, uint64_t offset,204__aicore__ inline void DynamicMxQuantBaseFP8<T, U, isTail>::LoadData(__ubuf__ T* xAddr, uint64_t offset,
205 Reg::RegTensor<T>& x, Reg::MaskReg& mask)205 Reg::RegTensor<T>& x, Reg::MaskReg& mask)
206{206{
207- Reg::UnalignReg uReg;207+ Reg::UnalignRegForLoad uReg;
208- Reg::DataCopyUnAlignPre(uReg, xAddr + offset);208+ Reg::LoadUnAlignPre(uReg, xAddr + offset);
209- Reg::DataCopyUnAlign(x, uReg, xAddr + offset);209+ Reg::LoadUnAlign(x, uReg, xAddr + offset);
210}210}
211 211 
212} // namespace DynamicMxQuant212} // namespace DynamicMxQuant
213-#endif // DYNAMIC_MX_QUANT_NOT_TAIL_AXIS_BASE_FP8_H213+#endif // DYNAMIC_MX_QUANT_NOT_TAIL_AXIS_BASE_FP8_H
Mquant/dynamic_mx_quant/op_kernel/arch35/dynamic_mx_quant_not_tail_axis_fp8.h+43-49
@@ -279,8 +279,8 @@ __aicore__ inline void DynamicMxQuantNotTailAxisFP8<T, U, isTail>::ComputeOCP(in
279 Reg::RegTensor<U> yZeroFP8;279 Reg::RegTensor<U> yZeroFP8;
280 Reg::RegTensor<U> yOneFP8;280 Reg::RegTensor<U> yOneFP8;
281 281 
282- Reg::UnalignReg u0;282+ Reg::UnalignRegForLoad u0;
283- Reg::UnalignReg u1;283+ Reg::UnalignRegForStore u1;
284 Reg::MaskReg p0;284 Reg::MaskReg p0;
285 Reg::MaskReg p1;285 Reg::MaskReg p1;
286 Reg::MaskReg infMask;286 Reg::MaskReg infMask;
@@ -340,8 +340,8 @@ __aicore__ inline void DynamicMxQuantNotTailAxisFP8<T, U, isTail>::ComputeOCP(in
340 Reg::ShiftRights(mxScaleInt, expMax, BF16_SHR_NUM, p0);340 Reg::ShiftRights(mxScaleInt, expMax, BF16_SHR_NUM, p0);
341 Reg::Select<uint16_t>(mxScaleInt, mxScaleInt, fp8Nan, infMask);341 Reg::Select<uint16_t>(mxScaleInt, mxScaleInt, fp8Nan, infMask);
342 Reg::Pack(mxScale, mxScaleInt);342 Reg::Pack(mxScale, mxScaleInt);
343- Reg::DataCopyUnAlign(mxScaleAddr, mxScale, u1, vfNum16);343+ Reg::StoreUnAlign(mxScaleAddr, mxScale, u1, vfNum16);
344- Reg::DataCopyUnAlignPost(mxScaleAddr, u1, 0);344+ Reg::StoreUnAlignPost(mxScaleAddr, u1, 0);
345 // 求1/scale345 // 求1/scale
346 Reg::Compare<uint16_t, CMPMODE::NE>(zeroMask, expMax, zero, p0);346 Reg::Compare<uint16_t, CMPMODE::NE>(zeroMask, expMax, zero, p0);
347 Reg::Compare<uint16_t, CMPMODE::EQ>(specialDataMask, expMax, bias, p0);347 Reg::Compare<uint16_t, CMPMODE::EQ>(specialDataMask, expMax, bias, p0);
@@ -375,11 +375,11 @@ __aicore__ inline void DynamicMxQuantNotTailAxisFP8<T, U, isTail>::ComputeOCP(in
375 Reg::Pack(y1, (Reg::RegTensor<uint32_t>&)yOneFP8);375 Reg::Pack(y1, (Reg::RegTensor<uint32_t>&)yOneFP8);
376 Reg::Pack(outOne, y1);376 Reg::Pack(outOne, y1);
377 auto addr0 = yAddr + (j * outDataLenAlign + i * vfNum16);377 auto addr0 = yAddr + (j * outDataLenAlign + i * vfNum16);
378- Reg::DataCopyUnAlign(addr0, outZero, u1, loopNum0);378+ Reg::StoreUnAlign(addr0, outZero, u1, loopNum0);
379- Reg::DataCopyUnAlignPost(addr0, u1, 0);379+ Reg::StoreUnAlignPost(addr0, u1, 0);
380 auto addr1 = yAddr + (j * outDataLenAlign + i * vfNum16) + loopNum0;380 auto addr1 = yAddr + (j * outDataLenAlign + i * vfNum16) + loopNum0;
381- Reg::DataCopyUnAlign(addr1, outOne, u1, loopNum1);381+ Reg::StoreUnAlign(addr1, outOne, u1, loopNum1);
382- Reg::DataCopyUnAlignPost(addr1, u1, 0);382+ Reg::StoreUnAlignPost(addr1, u1, 0);
383 }383 }
384 }384 }
385 385 
@@ -417,8 +417,8 @@ __aicore__ inline void DynamicMxQuantNotTailAxisFP8<T, U, isTail>::ComputeOCP(in
417 Reg::ShiftRights(mxScaleInt, expMax, BF16_SHR_NUM, p1);417 Reg::ShiftRights(mxScaleInt, expMax, BF16_SHR_NUM, p1);
418 Reg::Select<uint16_t>(mxScaleInt, mxScaleInt, fp8Nan, infMask);418 Reg::Select<uint16_t>(mxScaleInt, mxScaleInt, fp8Nan, infMask);
419 Reg::Pack(mxScale, mxScaleInt);419 Reg::Pack(mxScale, mxScaleInt);
420- Reg::DataCopyUnAlign(mxScaleAddr, mxScale, u1, tailVfLen);420+ Reg::StoreUnAlign(mxScaleAddr, mxScale, u1, tailVfLen);
421- Reg::DataCopyUnAlignPost(mxScaleAddr, u1, 0);421+ Reg::StoreUnAlignPost(mxScaleAddr, u1, 0);
422 // 求1/scale422 // 求1/scale
423 Reg::Compare<uint16_t, CMPMODE::NE>(zeroMask, expMax, zero, p1);423 Reg::Compare<uint16_t, CMPMODE::NE>(zeroMask, expMax, zero, p1);
424 Reg::Compare<uint16_t, CMPMODE::EQ>(specialDataMask, expMax, bias, p1);424 Reg::Compare<uint16_t, CMPMODE::EQ>(specialDataMask, expMax, bias, p1);
@@ -455,11 +455,11 @@ __aicore__ inline void DynamicMxQuantNotTailAxisFP8<T, U, isTail>::ComputeOCP(in
455 Reg::Pack(y1, (Reg::RegTensor<uint32_t>&)yOneFP8);455 Reg::Pack(y1, (Reg::RegTensor<uint32_t>&)yOneFP8);
456 Reg::Pack(outOne, y1);456 Reg::Pack(outOne, y1);
457 auto addr0 = yAddr + (regLoop * vfNum16 + j * outDataLenAlign);457 auto addr0 = yAddr + (regLoop * vfNum16 + j * outDataLenAlign);
458- Reg::DataCopyUnAlign(addr0, outZero, u1, tailLoopNum0);458+ Reg::StoreUnAlign(addr0, outZero, u1, tailLoopNum0);
459- Reg::DataCopyUnAlignPost(addr0, u1, 0);459+ Reg::StoreUnAlignPost(addr0, u1, 0);
460 auto addr1 = yAddr + (regLoop * vfNum16 + j * outDataLenAlign) + tailLoopNum0;460 auto addr1 = yAddr + (regLoop * vfNum16 + j * outDataLenAlign) + tailLoopNum0;
461- Reg::DataCopyUnAlign(addr1, outOne, u1, tailLoopNum1);461+ Reg::StoreUnAlign(addr1, outOne, u1, tailLoopNum1);
462- Reg::DataCopyUnAlignPost(addr1, u1, 0);462+ Reg::StoreUnAlignPost(addr1, u1, 0);
463 }463 }
464 }464 }
465 }465 }
@@ -518,7 +518,7 @@ __aicore__ inline void DynamicMxQuantNotTailAxisFP8<T, U, isTail>::ComputecuBLAS
518 Reg::RegTensor<U> yZeroFP8;518 Reg::RegTensor<U> yZeroFP8;
519 Reg::RegTensor<uint8_t> outZeromxScaleFp8;519 Reg::RegTensor<uint8_t> outZeromxScaleFp8;
520 520 
521- Reg::UnalignReg u1;521+ Reg::UnalignRegForStore u1;
522 Reg::MaskReg infMask;522 Reg::MaskReg infMask;
523 Reg::MaskReg zeroMask;523 Reg::MaskReg zeroMask;
524 Reg::MaskReg p0;524 Reg::MaskReg p0;
@@ -564,8 +564,7 @@ __aicore__ inline void DynamicMxQuantNotTailAxisFP8<T, U, isTail>::ComputecuBLAS
564 (Reg::RegTensor<T>&)expMaxreShareExpFP16YOneRegTensor,564 (Reg::RegTensor<T>&)expMaxreShareExpFP16YOneRegTensor,
565 preMaskScale);565 preMaskScale);
566 // 校验566 // 校验
567- Reg::CompareScalar<uint32_t, CMPMODE::LT>(infMask, expMaxAndAddOneFP32RegTensor, FP32_MX_MAX_EXP,567+ Reg::Compares<uint32_t, CMPMODE::LT>(infMask, expMaxAndAddOneFP32RegTensor, FP32_MX_MAX_EXP, preMaskScale);
568- preMaskScale);
569 Reg::Compare<uint32_t, CMPMODE::NE>(zeroMask, expMaxAndAddOneFP32RegTensor, zeroRegTensor32, preMaskScale);568 Reg::Compare<uint32_t, CMPMODE::NE>(zeroMask, expMaxAndAddOneFP32RegTensor, zeroRegTensor32, preMaskScale);
570 Reg::Maxs((Reg::RegTensor<float>&)expMaxAndAddOneFP32RegTensor,569 Reg::Maxs((Reg::RegTensor<float>&)expMaxAndAddOneFP32RegTensor,
571 (Reg::RegTensor<float>&)expMaxAndAddOneFP32RegTensor, this->maxLowBound_, preMaskScale);570 (Reg::RegTensor<float>&)expMaxAndAddOneFP32RegTensor, this->maxLowBound_, preMaskScale);
@@ -575,17 +574,15 @@ __aicore__ inline void DynamicMxQuantNotTailAxisFP8<T, U, isTail>::ComputecuBLAS
575 preMaskScale); // Exp574 preMaskScale); // Exp
576 Reg::And(manAndmxScaleFP32RegTensor, expMaxAndAddOneFP32RegTensor, manMaskFP32, preMaskScale); // Man575 Reg::And(manAndmxScaleFP32RegTensor, expMaxAndAddOneFP32RegTensor, manMaskFP32, preMaskScale); // Man
577 // 条件576 // 条件
578- Reg::CompareScalar<uint32_t, CMPMODE::GT>(p1, expAndreShareExpFP32RegTensor, FP32_NUMBER_ZERO,577+ Reg::Compares<uint32_t, CMPMODE::GT>(p1, expAndreShareExpFP32RegTensor, FP32_NUMBER_ZERO, preMaskScale);
579- preMaskScale);578+ Reg::Compares<uint32_t, CMPMODE::LT>(p2, expAndreShareExpFP32RegTensor, FP32_NUMBER_254, preMaskScale);
580- Reg::CompareScalar<uint32_t, CMPMODE::LT>(p2, expAndreShareExpFP32RegTensor, FP32_NUMBER_254, preMaskScale);579+ Reg::Compares<uint32_t, CMPMODE::GT>(p3, manAndmxScaleFP32RegTensor, FP32_NUMBER_ZERO, preMaskScale);
581- Reg::CompareScalar<uint32_t, CMPMODE::GT>(p3, manAndmxScaleFP32RegTensor, FP32_NUMBER_ZERO, preMaskScale);580+ Reg::And(p1, p1, p2, preMaskScale);
582- Reg::MaskAnd(p1, p1, p2, preMaskScale);581+ Reg::And(p1, p1, p3, preMaskScale);
583- Reg::MaskAnd(p1, p1, p3, preMaskScale);582+ Reg::Compares<uint32_t, CMPMODE::EQ>(p2, expAndreShareExpFP32RegTensor, FP32_NUMBER_ZERO, preMaskScale);
584- Reg::CompareScalar<uint32_t, CMPMODE::EQ>(p2, expAndreShareExpFP32RegTensor, FP32_NUMBER_ZERO,583+ Reg::Compares<uint32_t, CMPMODE::GT>(p3, manAndmxScaleFP32RegTensor, FP32_NUMBER_HALF, preMaskScale);
585- preMaskScale);584+ Reg::And(p2, p2, p3, preMaskScale);
586- Reg::CompareScalar<uint32_t, CMPMODE::GT>(p3, manAndmxScaleFP32RegTensor, FP32_NUMBER_HALF, preMaskScale);585+ Reg::Xor(p1, p1, p2, preMaskScale);
587- Reg::MaskAnd(p2, p2, p3, preMaskScale);
588- Reg::MaskXor(p1, p1, p2, preMaskScale);
589 // 向上取整586 // 向上取整
590 Reg::Adds(expMaxAndAddOneFP32RegTensor, expAndreShareExpFP32RegTensor, 1, preMaskScale);587 Reg::Adds(expMaxAndAddOneFP32RegTensor, expAndreShareExpFP32RegTensor, 1, preMaskScale);
591 Reg::Select(manAndmxScaleFP32RegTensor, expMaxAndAddOneFP32RegTensor, expAndreShareExpFP32RegTensor, p1);588 Reg::Select(manAndmxScaleFP32RegTensor, expMaxAndAddOneFP32RegTensor, expAndreShareExpFP32RegTensor, p1);
@@ -595,8 +592,8 @@ __aicore__ inline void DynamicMxQuantNotTailAxisFP8<T, U, isTail>::ComputecuBLAS
595 Reg::Pack<uint16_t, uint32_t, Reg::HighLowPart::LOWEST>(expScaleMxYZeroRegTensor,592 Reg::Pack<uint16_t, uint32_t, Reg::HighLowPart::LOWEST>(expScaleMxYZeroRegTensor,
596 manAndmxScaleFP32RegTensor);593 manAndmxScaleFP32RegTensor);
597 Reg::Pack<uint8_t, uint16_t, Reg::HighLowPart::LOWEST>(outZeromxScaleFp8, expScaleMxYZeroRegTensor);594 Reg::Pack<uint8_t, uint16_t, Reg::HighLowPart::LOWEST>(outZeromxScaleFp8, expScaleMxYZeroRegTensor);
598- Reg::DataCopyUnAlign(mxScaleAddr, outZeromxScaleFp8, u1, vfNum32); // 从寄存器搬到UB595+ Reg::StoreUnAlign(mxScaleAddr, outZeromxScaleFp8, u1, vfNum32); // 从寄存器搬到UB
599- Reg::DataCopyUnAlignPost(mxScaleAddr, u1, 0);596+ Reg::StoreUnAlignPost(mxScaleAddr, u1, 0);
600 // 求1/scale597 // 求1/scale
601 Reg::ShiftLefts(manAndmxScaleFP32RegTensor, manAndmxScaleFP32RegTensor, BF16_SHR_NUM, preMaskScale);598 Reg::ShiftLefts(manAndmxScaleFP32RegTensor, manAndmxScaleFP32RegTensor, BF16_SHR_NUM, preMaskScale);
602 Reg::Sub(expAndreShareExpFP32RegTensor, bias, manAndmxScaleFP32RegTensor, preMaskScale);599 Reg::Sub(expAndreShareExpFP32RegTensor, bias, manAndmxScaleFP32RegTensor, preMaskScale);
@@ -630,8 +627,8 @@ __aicore__ inline void DynamicMxQuantNotTailAxisFP8<T, U, isTail>::ComputecuBLAS
630 Reg::Pack(expScaleMxYZeroRegTensor, (Reg::RegTensor<uint32_t>&)yZeroFP8);627 Reg::Pack(expScaleMxYZeroRegTensor, (Reg::RegTensor<uint32_t>&)yZeroFP8);
631 Reg::Pack(outZeromxScaleFp8, expScaleMxYZeroRegTensor);628 Reg::Pack(outZeromxScaleFp8, expScaleMxYZeroRegTensor);
632 auto addr = yAddr + (j * outDataLenAlign + i * vfNum32);629 auto addr = yAddr + (j * outDataLenAlign + i * vfNum32);
633- Reg::DataCopyUnAlign(addr, outZeromxScaleFp8, u1, singleLoopNum);630+ Reg::StoreUnAlign(addr, outZeromxScaleFp8, u1, singleLoopNum);
634- Reg::DataCopyUnAlignPost(addr, u1, 0);631+ Reg::StoreUnAlignPost(addr, u1, 0);
635 }632 }
636 }633 }
637 634 
@@ -655,8 +652,7 @@ __aicore__ inline void DynamicMxQuantNotTailAxisFP8<T, U, isTail>::ComputecuBLAS
655 (Reg::RegTensor<T>&)expMaxreShareExpFP16YOneRegTensor,652 (Reg::RegTensor<T>&)expMaxreShareExpFP16YOneRegTensor,
656 preMaskScale);653 preMaskScale);
657 // 校验654 // 校验
658- Reg::CompareScalar<uint32_t, CMPMODE::LT>(infMask, expMaxAndAddOneFP32RegTensor, FP32_MX_MAX_EXP,655+ Reg::Compares<uint32_t, CMPMODE::LT>(infMask, expMaxAndAddOneFP32RegTensor, FP32_MX_MAX_EXP, preMaskScale);
659- preMaskScale);
660 Reg::Compare<uint32_t, CMPMODE::NE>(zeroMask, expMaxAndAddOneFP32RegTensor, zeroRegTensor32, preMaskScale);656 Reg::Compare<uint32_t, CMPMODE::NE>(zeroMask, expMaxAndAddOneFP32RegTensor, zeroRegTensor32, preMaskScale);
661 Reg::Maxs((Reg::RegTensor<float>&)expMaxAndAddOneFP32RegTensor,657 Reg::Maxs((Reg::RegTensor<float>&)expMaxAndAddOneFP32RegTensor,
662 (Reg::RegTensor<float>&)expMaxAndAddOneFP32RegTensor, this->maxLowBound_, preMaskScale);658 (Reg::RegTensor<float>&)expMaxAndAddOneFP32RegTensor, this->maxLowBound_, preMaskScale);
@@ -666,17 +662,15 @@ __aicore__ inline void DynamicMxQuantNotTailAxisFP8<T, U, isTail>::ComputecuBLAS
666 preMaskScale); // Exp662 preMaskScale); // Exp
667 Reg::And(manAndmxScaleFP32RegTensor, expMaxAndAddOneFP32RegTensor, manMaskFP32, preMaskScale); // Man663 Reg::And(manAndmxScaleFP32RegTensor, expMaxAndAddOneFP32RegTensor, manMaskFP32, preMaskScale); // Man
668 // 条件664 // 条件
669- Reg::CompareScalar<uint32_t, CMPMODE::GT>(p1, expAndreShareExpFP32RegTensor, FP32_NUMBER_ZERO,665+ Reg::Compares<uint32_t, CMPMODE::GT>(p1, expAndreShareExpFP32RegTensor, FP32_NUMBER_ZERO, preMaskScale);
670- preMaskScale);666+ Reg::Compares<uint32_t, CMPMODE::LT>(p2, expAndreShareExpFP32RegTensor, FP32_NUMBER_254, preMaskScale);
671- Reg::CompareScalar<uint32_t, CMPMODE::LT>(p2, expAndreShareExpFP32RegTensor, FP32_NUMBER_254, preMaskScale);667+ Reg::Compares<uint32_t, CMPMODE::GT>(p3, manAndmxScaleFP32RegTensor, FP32_NUMBER_ZERO, preMaskScale);
672- Reg::CompareScalar<uint32_t, CMPMODE::GT>(p3, manAndmxScaleFP32RegTensor, FP32_NUMBER_ZERO, preMaskScale);668+ Reg::And(p1, p1, p2, preMaskScale);
673- Reg::MaskAnd(p1, p1, p2, preMaskScale);669+ Reg::And(p1, p1, p3, preMaskScale);
674- Reg::MaskAnd(p1, p1, p3, preMaskScale);670+ Reg::Compares<uint32_t, CMPMODE::EQ>(p2, expAndreShareExpFP32RegTensor, FP32_NUMBER_ZERO, preMaskScale);
675- Reg::CompareScalar<uint32_t, CMPMODE::EQ>(p2, expAndreShareExpFP32RegTensor, FP32_NUMBER_ZERO,671+ Reg::Compares<uint32_t, CMPMODE::GT>(p3, manAndmxScaleFP32RegTensor, FP32_NUMBER_HALF, preMaskScale);
676- preMaskScale);672+ Reg::And(p2, p2, p3, preMaskScale);
677- Reg::CompareScalar<uint32_t, CMPMODE::GT>(p3, manAndmxScaleFP32RegTensor, FP32_NUMBER_HALF, preMaskScale);673+ Reg::Xor(p1, p1, p2, preMaskScale);
678- Reg::MaskAnd(p2, p2, p3, preMaskScale);
679- Reg::MaskXor(p1, p1, p2, preMaskScale);
680 // 向上取整674 // 向上取整
681 Reg::Adds(expMaxAndAddOneFP32RegTensor, expAndreShareExpFP32RegTensor, 1, preMaskScale);675 Reg::Adds(expMaxAndAddOneFP32RegTensor, expAndreShareExpFP32RegTensor, 1, preMaskScale);
682 Reg::Select(manAndmxScaleFP32RegTensor, expMaxAndAddOneFP32RegTensor, expAndreShareExpFP32RegTensor, p1);676 Reg::Select(manAndmxScaleFP32RegTensor, expMaxAndAddOneFP32RegTensor, expAndreShareExpFP32RegTensor, p1);
@@ -686,8 +680,8 @@ __aicore__ inline void DynamicMxQuantNotTailAxisFP8<T, U, isTail>::ComputecuBLAS
686 Reg::Pack<uint16_t, uint32_t, Reg::HighLowPart::LOWEST>(expScaleMxYZeroRegTensor,680 Reg::Pack<uint16_t, uint32_t, Reg::HighLowPart::LOWEST>(expScaleMxYZeroRegTensor,
687 manAndmxScaleFP32RegTensor);681 manAndmxScaleFP32RegTensor);
688 Reg::Pack<uint8_t, uint16_t, Reg::HighLowPart::LOWEST>(outZeromxScaleFp8, expScaleMxYZeroRegTensor);682 Reg::Pack<uint8_t, uint16_t, Reg::HighLowPart::LOWEST>(outZeromxScaleFp8, expScaleMxYZeroRegTensor);
689- Reg::DataCopyUnAlign(mxScaleAddr, outZeromxScaleFp8, u1, tailVfLen); // 从寄存器搬到UB683+ Reg::StoreUnAlign(mxScaleAddr, outZeromxScaleFp8, u1, tailVfLen); // 从寄存器搬到UB
690- Reg::DataCopyUnAlignPost(mxScaleAddr, u1, 0);684+ Reg::StoreUnAlignPost(mxScaleAddr, u1, 0);
691 // 求1/scale685 // 求1/scale
692 Reg::ShiftLefts(manAndmxScaleFP32RegTensor, manAndmxScaleFP32RegTensor, BF16_SHR_NUM, preMaskScale);686 Reg::ShiftLefts(manAndmxScaleFP32RegTensor, manAndmxScaleFP32RegTensor, BF16_SHR_NUM, preMaskScale);
693 Reg::Sub(expAndreShareExpFP32RegTensor, bias, manAndmxScaleFP32RegTensor, preMaskScale);687 Reg::Sub(expAndreShareExpFP32RegTensor, bias, manAndmxScaleFP32RegTensor, preMaskScale);
@@ -725,8 +719,8 @@ __aicore__ inline void DynamicMxQuantNotTailAxisFP8<T, U, isTail>::ComputecuBLAS
725 Reg::Pack(expScaleMxYZeroRegTensor, (Reg::RegTensor<uint32_t>&)yZeroFP8);719 Reg::Pack(expScaleMxYZeroRegTensor, (Reg::RegTensor<uint32_t>&)yZeroFP8);
726 Reg::Pack(outZeromxScaleFp8, expScaleMxYZeroRegTensor);720 Reg::Pack(outZeromxScaleFp8, expScaleMxYZeroRegTensor);
727 auto addr = yAddr + (regLoop * vfNum32 + j * outDataLenAlign);721 auto addr = yAddr + (regLoop * vfNum32 + j * outDataLenAlign);
728- Reg::DataCopyUnAlign(addr, outZeromxScaleFp8, u1, singleTailLoopNum);722+ Reg::StoreUnAlign(addr, outZeromxScaleFp8, u1, singleTailLoopNum);
729- Reg::DataCopyUnAlignPost(addr, u1, 0);723+ Reg::StoreUnAlignPost(addr, u1, 0);
730 }724 }
731 }725 }
732 }726 }
Mquant/dynamic_mx_quant/op_kernel/arch35/dynamic_mx_quant_not_tail_axis_optimize.h+63-63
@@ -188,7 +188,7 @@ __aicore__ inline void DynamicMxQuantNotTailAxisOptimize<T, U, ISTAIL>::ComputeC
188 rowsSingleLoop = static_cast<uint16_t>(blockCount);188 rowsSingleLoop = static_cast<uint16_t>(blockCount);
189 } else {189 } else {
190 rowsSingleLoop = static_cast<uint16_t>(static_cast<int64_t>(vfLen) / dataLen);190 rowsSingleLoop = static_cast<uint16_t>(static_cast<int64_t>(vfLen) / dataLen);
191- } // 单次处理能处理的行数191+ } // 单次处理能处理的行数
192 uint16_t dataLenSingleLoop = rowsSingleLoop * static_cast<uint16_t>(dataLen); // 单次处理长度192 uint16_t dataLenSingleLoop = rowsSingleLoop * static_cast<uint16_t>(dataLen); // 单次处理长度
193 uint16_t regLoop = Ceil(static_cast<uint16_t>(blockCount), rowsSingleLoop); // 循环数193 uint16_t regLoop = Ceil(static_cast<uint16_t>(blockCount), rowsSingleLoop); // 循环数
194 uint16_t rowsTailLoop = static_cast<uint16_t>(blockCount) % rowsSingleLoop; // 尾循环处理的行数194 uint16_t rowsTailLoop = static_cast<uint16_t>(blockCount) % rowsSingleLoop; // 尾循环处理的行数
@@ -229,8 +229,8 @@ __aicore__ inline void DynamicMxQuantNotTailAxisOptimize<T, U, ISTAIL>::ComputeC
229 Reg::RegTensor<calcTypeInt> nan;229 Reg::RegTensor<calcTypeInt> nan;
230 Reg::RegTensor<calcTypeInt> specialExp;230 Reg::RegTensor<calcTypeInt> specialExp;
231 231 
232- Reg::UnalignReg u0;232+ Reg::UnalignRegForLoad u0;
233- Reg::UnalignReg u1;233+ Reg::UnalignRegForStore u1;
234 Reg::MaskReg zeroMask;234 Reg::MaskReg zeroMask;
235 Reg::MaskReg infMask;235 Reg::MaskReg infMask;
236 Reg::MaskReg specialDataMask;236 Reg::MaskReg specialDataMask;
@@ -259,28 +259,28 @@ __aicore__ inline void DynamicMxQuantNotTailAxisOptimize<T, U, ISTAIL>::ComputeC
259 this->template LoadData<calcType>(xAddr, (regLoop - 1) * dataLenSingleLoop, xReg, tailPnumMask);259 this->template LoadData<calcType>(xAddr, (regLoop - 1) * dataLenSingleLoop, xReg, tailPnumMask);
260 Reg::And(absReg, (Reg::RegTensor<calcTypeInt>&)xReg, absForXReg, tailPnumMask);260 Reg::And(absReg, (Reg::RegTensor<calcTypeInt>&)xReg, absForXReg, tailPnumMask);
261 Reg::Max(absReg, xMaxReg, absReg, tailPnumMask);261 Reg::Max(absReg, xMaxReg, absReg, tailPnumMask);
262- Reg::Copy<calcTypeInt, Reg::MaskMergeMode::MERGING>(xMaxReg, absReg, tailPnumMask);262+ Reg::Move<calcTypeInt, Reg::MaskMergeMode::MERGING>(xMaxReg, absReg, tailPnumMask);
263 // 二分法求rowsSingleLoop行中的最大行263 // 二分法求rowsSingleLoop行中的最大行
264- Reg::DataCopy(maxExpAddr, xMaxReg, pnumMask);264+ Reg::StoreAlign(maxExpAddr, xMaxReg, pnumMask);
265 Reg::LocalMemBar<Reg::MemType::VEC_STORE, Reg::MemType::VEC_LOAD>();265 Reg::LocalMemBar<Reg::MemType::VEC_STORE, Reg::MemType::VEC_LOAD>();
266 uint32_t maskNum = dataLenSingleLoop - expOffset;266 uint32_t maskNum = dataLenSingleLoop - expOffset;
267 Reg::MaskReg mask = Reg::UpdateMask<calcTypeInt>(maskNum);267 Reg::MaskReg mask = Reg::UpdateMask<calcTypeInt>(maskNum);
268- Reg::DataCopyUnAlignPre(u0, maxExpAddr);268+ Reg::LoadUnAlignPre(u0, maxExpAddr);
269- Reg::DataCopyUnAlign(xMaxReg, u0, maxExpAddr);269+ Reg::LoadUnAlign(xMaxReg, u0, maxExpAddr);
270- Reg::DataCopyUnAlignPre(u0, maxExpAddr + expOffset);270+ Reg::LoadUnAlignPre(u0, maxExpAddr + expOffset);
271- Reg::DataCopyUnAlign(absReg, u0, maxExpAddr + expOffset);271+ Reg::LoadUnAlign(absReg, u0, maxExpAddr + expOffset);
272 Reg::Max(absReg, xMaxReg, absReg, mask);272 Reg::Max(absReg, xMaxReg, absReg, mask);
273- Reg::Copy<calcTypeInt, Reg::MaskMergeMode::MERGING>(xMaxReg, absReg, mask);273+ Reg::Move<calcTypeInt, Reg::MaskMergeMode::MERGING>(xMaxReg, absReg, mask);
274 for (uint16_t i = 0; i < loopSize; i++) {274 for (uint16_t i = 0; i < loopSize; i++) {
275- Reg::DataCopy(maxExpAddr, xMaxReg, pnumMask);275+ Reg::StoreAlign(maxExpAddr, xMaxReg, pnumMask);
276 Reg::LocalMemBar<Reg::MemType::VEC_STORE, Reg::MemType::VEC_LOAD>();276 Reg::LocalMemBar<Reg::MemType::VEC_STORE, Reg::MemType::VEC_LOAD>();
277 expOffset /= DIGIT_TWO;277 expOffset /= DIGIT_TWO;
278 maskNum = expOffset;278 maskNum = expOffset;
279 mask = Reg::UpdateMask<calcTypeInt>(maskNum);279 mask = Reg::UpdateMask<calcTypeInt>(maskNum);
280- Reg::DataCopyUnAlignPre(u0, maxExpAddr);280+ Reg::LoadUnAlignPre(u0, maxExpAddr);
281- Reg::DataCopyUnAlign(xMaxReg, u0, maxExpAddr);281+ Reg::LoadUnAlign(xMaxReg, u0, maxExpAddr);
282- Reg::DataCopyUnAlignPre(u0, maxExpAddr + expOffset);282+ Reg::LoadUnAlignPre(u0, maxExpAddr + expOffset);
283- Reg::DataCopyUnAlign(absReg, u0, maxExpAddr + expOffset);283+ Reg::LoadUnAlign(absReg, u0, maxExpAddr + expOffset);
284 Reg::Max(xMaxReg, xMaxReg, absReg, mask);284 Reg::Max(xMaxReg, xMaxReg, absReg, mask);
285 }285 }
286 // 求scale286 // 求scale
@@ -298,17 +298,17 @@ __aicore__ inline void DynamicMxQuantNotTailAxisOptimize<T, U, ISTAIL>::ComputeC
298 Reg::ShiftRights(expFP32Reg, xFP32MaxReg, FP32_SHR_NUM, mask32);298 Reg::ShiftRights(expFP32Reg, xFP32MaxReg, FP32_SHR_NUM, mask32);
299 // And获取尾数位299 // And获取尾数位
300 Reg::And(manFP32Reg, xFP32MaxReg, manForFP32Reg, mask32);300 Reg::And(manFP32Reg, xFP32MaxReg, manForFP32Reg, mask32);
301- Reg::CompareScalar<uint32_t, CMPMODE::GT>(p0, expFP32Reg, FP32_NUMBER_ZERO, mask32);301+ Reg::Compares<uint32_t, CMPMODE::GT>(p0, expFP32Reg, FP32_NUMBER_ZERO, mask32);
302- Reg::CompareScalar<uint32_t, CMPMODE::LT>(p0, expFP32Reg, FP32_NUMBER_254, p0);302+ Reg::Compares<uint32_t, CMPMODE::LT>(p0, expFP32Reg, FP32_NUMBER_254, p0);
303- Reg::CompareScalar<uint32_t, CMPMODE::GT>(p0, manFP32Reg, FP32_NUMBER_ZERO, p0);303+ Reg::Compares<uint32_t, CMPMODE::GT>(p0, manFP32Reg, FP32_NUMBER_ZERO, p0);
304 Reg::Adds(extractExpReg, expFP32Reg, 1, mask32);304 Reg::Adds(extractExpReg, expFP32Reg, 1, mask32);
305 // 根据情况选择指数位是否加一305 // 根据情况选择指数位是否加一
306 Reg::Select<uint32_t>(expFP32Reg, extractExpReg, expFP32Reg, p0);306 Reg::Select<uint32_t>(expFP32Reg, extractExpReg, expFP32Reg, p0);
307 307 
308 Reg::Pack(expBF16Reg, expFP32Reg);308 Reg::Pack(expBF16Reg, expFP32Reg);
309 Reg::Pack(mxScale, expBF16Reg);309 Reg::Pack(mxScale, expBF16Reg);
310- Reg::DataCopyUnAlign(mxScaleAddr, mxScale, u1, dataLen);310+ Reg::StoreUnAlign(mxScaleAddr, mxScale, u1, dataLen);
311- Reg::DataCopyUnAlignPost(mxScaleAddr, u1, 0);311+ Reg::StoreUnAlignPost(mxScaleAddr, u1, 0);
312 312 
313 Reg::ShiftLefts(expBF16Reg, expBF16Reg, BF16_SHR_NUM, mask16);313 Reg::ShiftLefts(expBF16Reg, expBF16Reg, BF16_SHR_NUM, mask16);
314 Reg::ShiftLefts(expFP32Reg, expFP32Reg, FP32_SHR_NUM, mask16);314 Reg::ShiftLefts(expFP32Reg, expFP32Reg, FP32_SHR_NUM, mask16);
@@ -328,25 +328,25 @@ __aicore__ inline void DynamicMxQuantNotTailAxisOptimize<T, U, ISTAIL>::ComputeC
328 328 
329 auto scaleAddr = maxExpAddr;329 auto scaleAddr = maxExpAddr;
330 for (uint16_t i = 0; i < rowsSingleLoop; i++) {330 for (uint16_t i = 0; i < rowsSingleLoop; i++) {
331- Reg::DataCopyUnAlign(scaleAddr, scaleReprocal, u1, dataLen);331+ Reg::StoreUnAlign(scaleAddr, scaleReprocal, u1, dataLen);
332- Reg::DataCopyUnAlignPost(scaleAddr, u1, 0);332+ Reg::StoreUnAlignPost(scaleAddr, u1, 0);
333 }333 }
334 Reg::LocalMemBar<Reg::MemType::VEC_STORE, Reg::MemType::VEC_LOAD>();334 Reg::LocalMemBar<Reg::MemType::VEC_STORE, Reg::MemType::VEC_LOAD>();
335- Reg::DataCopy(scaleReprocal, maxExpAddr);335+ Reg::LoadAlign(scaleReprocal, maxExpAddr);
336 336 
337 // 求data value337 // 求data value
338 for (uint16_t i = 0; i < static_cast<uint16_t>(regLoop - 1); i++) {338 for (uint16_t i = 0; i < static_cast<uint16_t>(regLoop - 1); i++) {
339 this->template LoadData<calcType>(xAddr, i * dataLenSingleLoop, xReg, pnumMask);339 this->template LoadData<calcType>(xAddr, i * dataLenSingleLoop, xReg, pnumMask);
340 CalcElement<roundMode, U, calcType, calcTypeInt>(xReg, scaleReprocal, infForXReg, out, pnumMask);340 CalcElement<roundMode, U, calcType, calcTypeInt>(xReg, scaleReprocal, infForXReg, out, pnumMask);
341 auto addr = yAddr + (i * dataLenSingleLoop) / DIGIT_TWO;341 auto addr = yAddr + (i * dataLenSingleLoop) / DIGIT_TWO;
342- Reg::DataCopyUnAlign(addr, out, u1, dataLenSingleLoop / DIGIT_TWO);342+ Reg::StoreUnAlign(addr, out, u1, dataLenSingleLoop / DIGIT_TWO);
343- Reg::DataCopyUnAlignPost(addr, u1, 0);343+ Reg::StoreUnAlignPost(addr, u1, 0);
344 }344 }
345 this->template LoadData<calcType>(xAddr, (regLoop - 1) * dataLenSingleLoop, xReg, tailPnumMask);345 this->template LoadData<calcType>(xAddr, (regLoop - 1) * dataLenSingleLoop, xReg, tailPnumMask);
346 CalcElement<roundMode, U, calcType, calcTypeInt>(xReg, scaleReprocal, infForXReg, out, tailPnumMask);346 CalcElement<roundMode, U, calcType, calcTypeInt>(xReg, scaleReprocal, infForXReg, out, tailPnumMask);
347 auto addr = yAddr + ((regLoop - 1) * dataLenSingleLoop) / DIGIT_TWO;347 auto addr = yAddr + ((regLoop - 1) * dataLenSingleLoop) / DIGIT_TWO;
348- Reg::DataCopyUnAlign(addr, out, u1, dataLenTailLoop / DIGIT_TWO);348+ Reg::StoreUnAlign(addr, out, u1, dataLenTailLoop / DIGIT_TWO);
349- Reg::DataCopyUnAlignPost(addr, u1, 0);349+ Reg::StoreUnAlignPost(addr, u1, 0);
350 }350 }
351}351}
352template <typename T, typename U, const bool ISTAIL>352template <typename T, typename U, const bool ISTAIL>
@@ -362,7 +362,7 @@ __aicore__ inline void DynamicMxQuantNotTailAxisOptimize<T, U, ISTAIL>::ComputeO
362 rowsSingleLoop = static_cast<uint16_t>(blockCount);362 rowsSingleLoop = static_cast<uint16_t>(blockCount);
363 } else {363 } else {
364 rowsSingleLoop = static_cast<uint16_t>(static_cast<int64_t>(vfLen) / dataLen);364 rowsSingleLoop = static_cast<uint16_t>(static_cast<int64_t>(vfLen) / dataLen);
365- } // 单次处理能处理的行数365+ } // 单次处理能处理的行数
366 uint16_t dataLenSingleLoop = rowsSingleLoop * static_cast<uint16_t>(dataLen); // 单次处理长度366 uint16_t dataLenSingleLoop = rowsSingleLoop * static_cast<uint16_t>(dataLen); // 单次处理长度
367 uint16_t regLoop = Ceil(static_cast<uint16_t>(blockCount), rowsSingleLoop); // 循环数367 uint16_t regLoop = Ceil(static_cast<uint16_t>(blockCount), rowsSingleLoop); // 循环数
368 uint16_t rowsTailLoop = static_cast<uint16_t>(blockCount) % rowsSingleLoop; // 尾循环处理的行数368 uint16_t rowsTailLoop = static_cast<uint16_t>(blockCount) % rowsSingleLoop; // 尾循环处理的行数
@@ -403,8 +403,8 @@ __aicore__ inline void DynamicMxQuantNotTailAxisOptimize<T, U, ISTAIL>::ComputeO
403 Reg::RegTensor<calcTypeInt> fp4MaxExpReg;403 Reg::RegTensor<calcTypeInt> fp4MaxExpReg;
404 Reg::RegTensor<calcTypeInt> zeroReg;404 Reg::RegTensor<calcTypeInt> zeroReg;
405 405 
406- Reg::UnalignReg u0;406+ Reg::UnalignRegForLoad u0;
407- Reg::UnalignReg u1;407+ Reg::UnalignRegForStore u1;
408 Reg::MaskReg zeroMask;408 Reg::MaskReg zeroMask;
409 Reg::MaskReg infMask;409 Reg::MaskReg infMask;
410 Reg::MaskReg invalidDataMask;410 Reg::MaskReg invalidDataMask;
@@ -441,26 +441,26 @@ __aicore__ inline void DynamicMxQuantNotTailAxisOptimize<T, U, ISTAIL>::ComputeO
441 this->template LoadData<calcType>(xAddr, (regLoop - 1) * dataLenSingleLoop, xReg, tailPnumMask);441 this->template LoadData<calcType>(xAddr, (regLoop - 1) * dataLenSingleLoop, xReg, tailPnumMask);
442 Reg::And(absReg, (Reg::RegTensor<calcTypeInt>&)xReg, absForXReg, tailPnumMask);442 Reg::And(absReg, (Reg::RegTensor<calcTypeInt>&)xReg, absForXReg, tailPnumMask);
443 Reg::Max(absReg, xMaxReg, absReg, tailPnumMask);443 Reg::Max(absReg, xMaxReg, absReg, tailPnumMask);
444- Reg::Copy<calcTypeInt, Reg::MaskMergeMode::MERGING>(xMaxReg, absReg, tailPnumMask);444+ Reg::Move<calcTypeInt, Reg::MaskMergeMode::MERGING>(xMaxReg, absReg, tailPnumMask);
445 // 二分法求rowsSingleLoop行中的最大行445 // 二分法求rowsSingleLoop行中的最大行
446- Reg::DataCopy(maxExpAddr, xMaxReg, pnumMask);446+ Reg::StoreAlign(maxExpAddr, xMaxReg, pnumMask);
447 Reg::LocalMemBar<Reg::MemType::VEC_STORE, Reg::MemType::VEC_LOAD>();447 Reg::LocalMemBar<Reg::MemType::VEC_STORE, Reg::MemType::VEC_LOAD>();
448- Reg::DataCopyUnAlignPre(u0, maxExpAddr);448+ Reg::LoadUnAlignPre(u0, maxExpAddr);
449- Reg::DataCopyUnAlign(xMaxReg, u0, maxExpAddr);449+ Reg::LoadUnAlign(xMaxReg, u0, maxExpAddr);
450- Reg::DataCopyUnAlignPre(u0, maxExpAddr + expOffset);450+ Reg::LoadUnAlignPre(u0, maxExpAddr + expOffset);
451- Reg::DataCopyUnAlign(absReg, u0, maxExpAddr + expOffset);451+ Reg::LoadUnAlign(absReg, u0, maxExpAddr + expOffset);
452 Reg::Max(absReg, xMaxReg, absReg, mask);452 Reg::Max(absReg, xMaxReg, absReg, mask);
453- Reg::Copy<calcTypeInt, Reg::MaskMergeMode::MERGING>(xMaxReg, absReg, mask);453+ Reg::Move<calcTypeInt, Reg::MaskMergeMode::MERGING>(xMaxReg, absReg, mask);
454 for (uint16_t i = 0; i < loopSize; i++) {454 for (uint16_t i = 0; i < loopSize; i++) {
455- Reg::DataCopy(maxExpAddr, xMaxReg, pnumMask);455+ Reg::StoreAlign(maxExpAddr, xMaxReg, pnumMask);
456 Reg::LocalMemBar<Reg::MemType::VEC_STORE, Reg::MemType::VEC_LOAD>();456 Reg::LocalMemBar<Reg::MemType::VEC_STORE, Reg::MemType::VEC_LOAD>();
457 expOffset /= DIGIT_TWO;457 expOffset /= DIGIT_TWO;
458 maskNum = expOffset;458 maskNum = expOffset;
459 mask = Reg::UpdateMask<calcTypeInt>(maskNum);459 mask = Reg::UpdateMask<calcTypeInt>(maskNum);
460- Reg::DataCopyUnAlignPre(u0, maxExpAddr);460+ Reg::LoadUnAlignPre(u0, maxExpAddr);
461- Reg::DataCopyUnAlign(xMaxReg, u0, maxExpAddr);461+ Reg::LoadUnAlign(xMaxReg, u0, maxExpAddr);
462- Reg::DataCopyUnAlignPre(u0, maxExpAddr + expOffset);462+ Reg::LoadUnAlignPre(u0, maxExpAddr + expOffset);
463- Reg::DataCopyUnAlign(absReg, u0, maxExpAddr + expOffset);463+ Reg::LoadUnAlign(absReg, u0, maxExpAddr + expOffset);
464 Reg::Max(xMaxReg, xMaxReg, absReg, mask);464 Reg::Max(xMaxReg, xMaxReg, absReg, mask);
465 }465 }
466 maskNum = static_cast<uint32_t>(dataLen);466 maskNum = static_cast<uint32_t>(dataLen);
@@ -485,26 +485,26 @@ __aicore__ inline void DynamicMxQuantNotTailAxisOptimize<T, U, ISTAIL>::ComputeO
485 this->template LoadData<calcType>(xAddr, (regLoop - 1) * dataLenSingleLoop, xReg, tailPnumMask);485 this->template LoadData<calcType>(xAddr, (regLoop - 1) * dataLenSingleLoop, xReg, tailPnumMask);
486 Reg::And(expReg, (Reg::RegTensor<calcTypeInt>&)xReg, infForXReg, tailPnumMask);486 Reg::And(expReg, (Reg::RegTensor<calcTypeInt>&)xReg, infForXReg, tailPnumMask);
487 Reg::Max(expReg, expMaxReg, expReg, tailPnumMask);487 Reg::Max(expReg, expMaxReg, expReg, tailPnumMask);
488- Reg::Copy<calcTypeInt, Reg::MaskMergeMode::MERGING>(expMaxReg, expReg, tailPnumMask);488+ Reg::Move<calcTypeInt, Reg::MaskMergeMode::MERGING>(expMaxReg, expReg, tailPnumMask);
489 // 二分法求rowsSingleLoop行中的最大行489 // 二分法求rowsSingleLoop行中的最大行
490- Reg::DataCopy(maxExpAddr, expMaxReg, pnumMask);490+ Reg::StoreAlign(maxExpAddr, expMaxReg, pnumMask);
491 Reg::LocalMemBar<Reg::MemType::VEC_STORE, Reg::MemType::VEC_LOAD>();491 Reg::LocalMemBar<Reg::MemType::VEC_STORE, Reg::MemType::VEC_LOAD>();
492- Reg::DataCopyUnAlignPre(u0, maxExpAddr);492+ Reg::LoadUnAlignPre(u0, maxExpAddr);
493- Reg::DataCopyUnAlign(expMaxReg, u0, maxExpAddr);493+ Reg::LoadUnAlign(expMaxReg, u0, maxExpAddr);
494- Reg::DataCopyUnAlignPre(u0, maxExpAddr + expOffset);494+ Reg::LoadUnAlignPre(u0, maxExpAddr + expOffset);
495- Reg::DataCopyUnAlign(expReg, u0, maxExpAddr + expOffset);495+ Reg::LoadUnAlign(expReg, u0, maxExpAddr + expOffset);
496 Reg::Max(expReg, expMaxReg, expReg, mask);496 Reg::Max(expReg, expMaxReg, expReg, mask);
497- Reg::Copy<calcTypeInt, Reg::MaskMergeMode::MERGING>(expMaxReg, expReg, mask);497+ Reg::Move<calcTypeInt, Reg::MaskMergeMode::MERGING>(expMaxReg, expReg, mask);
498 for (uint16_t i = 0; i < loopSize; i++) {498 for (uint16_t i = 0; i < loopSize; i++) {
499- Reg::DataCopy(maxExpAddr, expMaxReg, pnumMask);499+ Reg::StoreAlign(maxExpAddr, expMaxReg, pnumMask);
500 Reg::LocalMemBar<Reg::MemType::VEC_STORE, Reg::MemType::VEC_LOAD>();500 Reg::LocalMemBar<Reg::MemType::VEC_STORE, Reg::MemType::VEC_LOAD>();
501 expOffset /= DIGIT_TWO;501 expOffset /= DIGIT_TWO;
502 maskNum = expOffset;502 maskNum = expOffset;
503 mask = Reg::UpdateMask<calcTypeInt>(maskNum);503 mask = Reg::UpdateMask<calcTypeInt>(maskNum);
504- Reg::DataCopyUnAlignPre(u0, maxExpAddr);504+ Reg::LoadUnAlignPre(u0, maxExpAddr);
505- Reg::DataCopyUnAlign(expMaxReg, u0, maxExpAddr);505+ Reg::LoadUnAlign(expMaxReg, u0, maxExpAddr);
506- Reg::DataCopyUnAlignPre(u0, maxExpAddr + expOffset);506+ Reg::LoadUnAlignPre(u0, maxExpAddr + expOffset);
507- Reg::DataCopyUnAlign(expReg, u0, maxExpAddr + expOffset);507+ Reg::LoadUnAlign(expReg, u0, maxExpAddr + expOffset);
508 Reg::Max(expMaxReg, expMaxReg, expReg, mask);508 Reg::Max(expMaxReg, expMaxReg, expReg, mask);
509 }509 }
510 maskNum = static_cast<uint32_t>(dataLen);510 maskNum = static_cast<uint32_t>(dataLen);
@@ -524,8 +524,8 @@ __aicore__ inline void DynamicMxQuantNotTailAxisOptimize<T, U, ISTAIL>::ComputeO
524 } else {524 } else {
525 Reg::Pack(mxScale, mxScaleReg);525 Reg::Pack(mxScale, mxScaleReg);
526 }526 }
527- Reg::DataCopyUnAlign(mxScaleAddr, mxScale, u1, dataLen);527+ Reg::StoreUnAlign(mxScaleAddr, mxScale, u1, dataLen);
528- Reg::DataCopyUnAlignPost(mxScaleAddr, u1, 0);528+ Reg::StoreUnAlignPost(mxScaleAddr, u1, 0);
529 529 
530 // 求1/scale530 // 求1/scale
531 Reg::Compare<calcTypeInt, CMPMODE::NE>(zeroMask, expMaxReg, zeroReg, mask);531 Reg::Compare<calcTypeInt, CMPMODE::NE>(zeroMask, expMaxReg, zeroReg, mask);
@@ -537,25 +537,25 @@ __aicore__ inline void DynamicMxQuantNotTailAxisOptimize<T, U, ISTAIL>::ComputeO
537 537 
538 auto scaleAddr = maxExpAddr;538 auto scaleAddr = maxExpAddr;
539 for (uint16_t i = 0; i < rowsSingleLoop; i++) {539 for (uint16_t i = 0; i < rowsSingleLoop; i++) {
540- Reg::DataCopyUnAlign(scaleAddr, scaleReprocal, u1, dataLen);540+ Reg::StoreUnAlign(scaleAddr, scaleReprocal, u1, dataLen);
541- Reg::DataCopyUnAlignPost(scaleAddr, u1, 0);541+ Reg::StoreUnAlignPost(scaleAddr, u1, 0);
542 }542 }
543 Reg::LocalMemBar<Reg::MemType::VEC_STORE, Reg::MemType::VEC_LOAD>();543 Reg::LocalMemBar<Reg::MemType::VEC_STORE, Reg::MemType::VEC_LOAD>();
544- Reg::DataCopy(scaleReprocal, maxExpAddr);544+ Reg::LoadAlign(scaleReprocal, maxExpAddr);
545 545 
546 // 求data value546 // 求data value
547 for (uint16_t i = 0; i < static_cast<uint16_t>(regLoop - 1); i++) {547 for (uint16_t i = 0; i < static_cast<uint16_t>(regLoop - 1); i++) {
548 this->template LoadData<calcType>(xAddr, i * dataLenSingleLoop, xReg, pnumMask);548 this->template LoadData<calcType>(xAddr, i * dataLenSingleLoop, xReg, pnumMask);
549 CalcElement<roundMode, U, calcType, calcTypeInt>(xReg, scaleReprocal, infForXReg, out, pnumMask);549 CalcElement<roundMode, U, calcType, calcTypeInt>(xReg, scaleReprocal, infForXReg, out, pnumMask);
550 auto addr = yAddr + (i * dataLenSingleLoop) / DIGIT_TWO;550 auto addr = yAddr + (i * dataLenSingleLoop) / DIGIT_TWO;
551- Reg::DataCopyUnAlign(addr, out, u1, dataLenSingleLoop / DIGIT_TWO);551+ Reg::StoreUnAlign(addr, out, u1, dataLenSingleLoop / DIGIT_TWO);
552- Reg::DataCopyUnAlignPost(addr, u1, 0);552+ Reg::StoreUnAlignPost(addr, u1, 0);
553 }553 }
554 this->template LoadData<calcType>(xAddr, (regLoop - 1) * dataLenSingleLoop, xReg, tailPnumMask);554 this->template LoadData<calcType>(xAddr, (regLoop - 1) * dataLenSingleLoop, xReg, tailPnumMask);
555 CalcElement<roundMode, U, calcType, calcTypeInt>(xReg, scaleReprocal, infForXReg, out, tailPnumMask);555 CalcElement<roundMode, U, calcType, calcTypeInt>(xReg, scaleReprocal, infForXReg, out, tailPnumMask);
556 auto addr = yAddr + ((regLoop - 1) * dataLenSingleLoop) / DIGIT_TWO;556 auto addr = yAddr + ((regLoop - 1) * dataLenSingleLoop) / DIGIT_TWO;
557- Reg::DataCopyUnAlign(addr, out, u1, dataLenTailLoop / DIGIT_TWO);557+ Reg::StoreUnAlign(addr, out, u1, dataLenTailLoop / DIGIT_TWO);
558- Reg::DataCopyUnAlignPost(addr, u1, 0);558+ Reg::StoreUnAlignPost(addr, u1, 0);
559 }559 }
560}560}
561 561 
Mquant/dynamic_mx_quant/op_kernel/arch35/dynamic_mx_quant_not_tail_axis_optimize_fp8.h+50-50
@@ -204,8 +204,8 @@ __aicore__ inline void DynamicMxQuantNotTailAxisOptimizeFP8<T, U, isTail>::Compu
204 Reg::RegTensor<bfloat16_t> valueRegTensor;204 Reg::RegTensor<bfloat16_t> valueRegTensor;
205 Reg::RegTensor<uint16_t> invalidMaskFp16;205 Reg::RegTensor<uint16_t> invalidMaskFp16;
206 Reg::RegTensor<uint16_t> xSelectRegTensor;206 Reg::RegTensor<uint16_t> xSelectRegTensor;
207- Reg::UnalignReg u0;207+ Reg::UnalignRegForLoad u0;
208- Reg::UnalignReg u1;208+ Reg::UnalignRegForStore u1;
209 Reg::MaskReg zeroMask;209 Reg::MaskReg zeroMask;
210 Reg::MaskReg infMask;210 Reg::MaskReg infMask;
211 Reg::MaskReg invalidDataMask;211 Reg::MaskReg invalidDataMask;
@@ -258,28 +258,28 @@ __aicore__ inline void DynamicMxQuantNotTailAxisOptimizeFP8<T, U, isTail>::Compu
258 Reg::And(exp, (Reg::RegTensor<uint16_t>&)x, maxEle, tailPnumMask);258 Reg::And(exp, (Reg::RegTensor<uint16_t>&)x, maxEle, tailPnumMask);
259 }259 }
260 Reg::Max(exp, expMax, exp, tailPnumMask);260 Reg::Max(exp, expMax, exp, tailPnumMask);
261- Reg::Copy<uint16_t, Reg::MaskMergeMode::MERGING>(expMax, exp, tailPnumMask);261+ Reg::Move<uint16_t, Reg::MaskMergeMode::MERGING>(expMax, exp, tailPnumMask);
262 // 二分法求rowsSingleLoop行中的最大行262 // 二分法求rowsSingleLoop行中的最大行
263- Reg::DataCopy(maxExpAddr, expMax, pnumMask);263+ Reg::StoreAlign(maxExpAddr, expMax, pnumMask);
264 Reg::LocalMemBar<Reg::MemType::VEC_STORE, Reg::MemType::VEC_LOAD>();264 Reg::LocalMemBar<Reg::MemType::VEC_STORE, Reg::MemType::VEC_LOAD>();
265 uint32_t maskNum = dataLenSingleLoop - expOffset;265 uint32_t maskNum = dataLenSingleLoop - expOffset;
266 Reg::MaskReg mask = Reg::UpdateMask<uint16_t>(maskNum);266 Reg::MaskReg mask = Reg::UpdateMask<uint16_t>(maskNum);
267- Reg::DataCopyUnAlignPre(u0, maxExpAddr);267+ Reg::LoadUnAlignPre(u0, maxExpAddr);
268- Reg::DataCopyUnAlign(expMax, u0, maxExpAddr);268+ Reg::LoadUnAlign(expMax, u0, maxExpAddr);
269- Reg::DataCopyUnAlignPre(u0, maxExpAddr + expOffset);269+ Reg::LoadUnAlignPre(u0, maxExpAddr + expOffset);
270- Reg::DataCopyUnAlign(exp, u0, maxExpAddr + expOffset);270+ Reg::LoadUnAlign(exp, u0, maxExpAddr + expOffset);
271 Reg::Max(exp, expMax, exp, mask);271 Reg::Max(exp, expMax, exp, mask);
272- Reg::Copy<uint16_t, Reg::MaskMergeMode::MERGING>(expMax, exp, mask);272+ Reg::Move<uint16_t, Reg::MaskMergeMode::MERGING>(expMax, exp, mask);
273 for (uint16_t i = 0; i < loopSize; i++) {273 for (uint16_t i = 0; i < loopSize; i++) {
274- Reg::DataCopy(maxExpAddr, expMax, pnumMask);274+ Reg::StoreAlign(maxExpAddr, expMax, pnumMask);
275 Reg::LocalMemBar<Reg::MemType::VEC_STORE, Reg::MemType::VEC_LOAD>();275 Reg::LocalMemBar<Reg::MemType::VEC_STORE, Reg::MemType::VEC_LOAD>();
276 expOffset /= DIGIT_TWO;276 expOffset /= DIGIT_TWO;
277 maskNum = expOffset;277 maskNum = expOffset;
278 mask = Reg::UpdateMask<uint16_t>(maskNum);278 mask = Reg::UpdateMask<uint16_t>(maskNum);
279- Reg::DataCopyUnAlignPre(u0, maxExpAddr);279+ Reg::LoadUnAlignPre(u0, maxExpAddr);
280- Reg::DataCopyUnAlign(expMax, u0, maxExpAddr);280+ Reg::LoadUnAlign(expMax, u0, maxExpAddr);
281- Reg::DataCopyUnAlignPre(u0, maxExpAddr + expOffset);281+ Reg::LoadUnAlignPre(u0, maxExpAddr + expOffset);
282- Reg::DataCopyUnAlign(exp, u0, maxExpAddr + expOffset);282+ Reg::LoadUnAlign(exp, u0, maxExpAddr + expOffset);
283 Reg::Max(expMax, expMax, exp, mask);283 Reg::Max(expMax, expMax, exp, mask);
284 }284 }
285 maskNum = static_cast<uint32_t>(dataLen);285 maskNum = static_cast<uint32_t>(dataLen);
@@ -291,8 +291,8 @@ __aicore__ inline void DynamicMxQuantNotTailAxisOptimizeFP8<T, U, isTail>::Compu
291 Reg::ShiftRights(mxScaleInt, expMax, BF16_SHR_NUM, mask);291 Reg::ShiftRights(mxScaleInt, expMax, BF16_SHR_NUM, mask);
292 Reg::Select<uint16_t>(mxScaleInt, mxScaleInt, fp8Nan, infMask);292 Reg::Select<uint16_t>(mxScaleInt, mxScaleInt, fp8Nan, infMask);
293 Reg::Pack(mxScale, mxScaleInt);293 Reg::Pack(mxScale, mxScaleInt);
294- Reg::DataCopyUnAlign(mxScaleAddr, mxScale, u1, dataLen);294+ Reg::StoreUnAlign(mxScaleAddr, mxScale, u1, dataLen);
295- Reg::DataCopyUnAlignPost(mxScaleAddr, u1, 0);295+ Reg::StoreUnAlignPost(mxScaleAddr, u1, 0);
296 296 
297 // 求1/scale297 // 求1/scale
298 Reg::Compare<uint16_t, CMPMODE::NE>(zeroMask, expMax, zero, mask);298 Reg::Compare<uint16_t, CMPMODE::NE>(zeroMask, expMax, zero, mask);
@@ -304,11 +304,11 @@ __aicore__ inline void DynamicMxQuantNotTailAxisOptimizeFP8<T, U, isTail>::Compu
304 304 
305 auto scaleAddr = maxExpAddr;305 auto scaleAddr = maxExpAddr;
306 for (uint16_t i = 0; i < rowsSingleLoop; i++) {306 for (uint16_t i = 0; i < rowsSingleLoop; i++) {
307- Reg::DataCopyUnAlign(scaleAddr, reversedShareExpRegTensor, u1, dataLen);307+ Reg::StoreUnAlign(scaleAddr, reversedShareExpRegTensor, u1, dataLen);
308- Reg::DataCopyUnAlignPost(scaleAddr, u1, 0);308+ Reg::StoreUnAlignPost(scaleAddr, u1, 0);
309 }309 }
310 Reg::LocalMemBar<Reg::MemType::VEC_STORE, Reg::MemType::VEC_LOAD>();310 Reg::LocalMemBar<Reg::MemType::VEC_STORE, Reg::MemType::VEC_LOAD>();
311- Reg::DataCopy(reversedShareExpRegTensor, maxExpAddr);311+ Reg::LoadAlign(reversedShareExpRegTensor, maxExpAddr);
312 // 求data value312 // 求data value
313 for (uint16_t i = 0; i < static_cast<uint16_t>(regLoop - 1); i++) {313 for (uint16_t i = 0; i < static_cast<uint16_t>(regLoop - 1); i++) {
314 this->template LoadData<toBf16RoundMode, roundMode, true>(xAddr, i * dataLenSingleLoop, x, pnumMask);314 this->template LoadData<toBf16RoundMode, roundMode, true>(xAddr, i * dataLenSingleLoop, x, pnumMask);
@@ -335,11 +335,11 @@ __aicore__ inline void DynamicMxQuantNotTailAxisOptimizeFP8<T, U, isTail>::Compu
335 Reg::Pack(yRegTensorOne, (Reg::RegTensor<uint32_t>&)yOneFP8);335 Reg::Pack(yRegTensorOne, (Reg::RegTensor<uint32_t>&)yOneFP8);
336 Reg::Pack(outOne, yRegTensorOne);336 Reg::Pack(outOne, yRegTensorOne);
337 auto addr0 = yAddr + i * dataLenSingleLoop;337 auto addr0 = yAddr + i * dataLenSingleLoop;
338- Reg::DataCopyUnAlign(addr0, outZero, u1, loopNum0);338+ Reg::StoreUnAlign(addr0, outZero, u1, loopNum0);
339- Reg::DataCopyUnAlignPost(addr0, u1, 0);339+ Reg::StoreUnAlignPost(addr0, u1, 0);
340 auto addr1 = yAddr + i * dataLenSingleLoop + loopNum0;340 auto addr1 = yAddr + i * dataLenSingleLoop + loopNum0;
341- Reg::DataCopyUnAlign(addr1, outOne, u1, loopNum1);341+ Reg::StoreUnAlign(addr1, outOne, u1, loopNum1);
342- Reg::DataCopyUnAlignPost(addr1, u1, 0);342+ Reg::StoreUnAlignPost(addr1, u1, 0);
343 }343 }
344 this->template LoadData<toBf16RoundMode, roundMode, true>(xAddr, (regLoop - 1) * dataLenSingleLoop, x,344 this->template LoadData<toBf16RoundMode, roundMode, true>(xAddr, (regLoop - 1) * dataLenSingleLoop, x,
345 tailPnumMask);345 tailPnumMask);
@@ -366,11 +366,11 @@ __aicore__ inline void DynamicMxQuantNotTailAxisOptimizeFP8<T, U, isTail>::Compu
366 Reg::Pack(yRegTensorOne, (Reg::RegTensor<uint32_t>&)yOneFP8);366 Reg::Pack(yRegTensorOne, (Reg::RegTensor<uint32_t>&)yOneFP8);
367 Reg::Pack(outOne, yRegTensorOne);367 Reg::Pack(outOne, yRegTensorOne);
368 auto addr0 = yAddr + (regLoop - 1) * dataLenSingleLoop;368 auto addr0 = yAddr + (regLoop - 1) * dataLenSingleLoop;
369- Reg::DataCopyUnAlign(addr0, outZero, u1, tailLoopNum0);369+ Reg::StoreUnAlign(addr0, outZero, u1, tailLoopNum0);
370- Reg::DataCopyUnAlignPost(addr0, u1, 0);370+ Reg::StoreUnAlignPost(addr0, u1, 0);
371 auto addr1 = yAddr + (regLoop - 1) * dataLenSingleLoop + tailLoopNum0;371 auto addr1 = yAddr + (regLoop - 1) * dataLenSingleLoop + tailLoopNum0;
372- Reg::DataCopyUnAlign(addr1, outOne, u1, tailLoopNum1);372+ Reg::StoreUnAlign(addr1, outOne, u1, tailLoopNum1);
373- Reg::DataCopyUnAlignPost(addr1, u1, 0);373+ Reg::StoreUnAlignPost(addr1, u1, 0);
374 }374 }
375}375}
376 376 
@@ -437,8 +437,8 @@ __aicore__ inline void DynamicMxQuantNotTailAxisOptimizeFP8<T, U, isTail>::Compu
437 Reg::RegTensor<float> yOne;437 Reg::RegTensor<float> yOne;
438 Reg::RegTensor<U> yZeroFP8;438 Reg::RegTensor<U> yZeroFP8;
439 Reg::RegTensor<U> yOneFP8;439 Reg::RegTensor<U> yOneFP8;
440- Reg::UnalignReg u0;440+ Reg::UnalignRegForLoad u0;
441- Reg::UnalignReg u1;441+ Reg::UnalignRegForStore u1;
442 442 
443 static constexpr Reg::CastTrait castTraitZero = {Reg::RegLayout::ZERO, Reg::SatMode::UNKNOWN,443 static constexpr Reg::CastTrait castTraitZero = {Reg::RegLayout::ZERO, Reg::SatMode::UNKNOWN,
444 Reg::MaskMergeMode::ZEROING, RoundMode::UNKNOWN};444 Reg::MaskMergeMode::ZEROING, RoundMode::UNKNOWN};
@@ -472,25 +472,25 @@ __aicore__ inline void DynamicMxQuantNotTailAxisOptimizeFP8<T, U, isTail>::Compu
472 tailPnumMask);472 tailPnumMask);
473 Reg::And(exp, (Reg::RegTensor<uint16_t>&)x, tmp16RegTensor, tailPnumMask);473 Reg::And(exp, (Reg::RegTensor<uint16_t>&)x, tmp16RegTensor, tailPnumMask);
474 Reg::Max(exp, expMax, exp, tailPnumMask);474 Reg::Max(exp, expMax, exp, tailPnumMask);
475- Reg::Copy<uint16_t, Reg::MaskMergeMode::MERGING>(expMax, exp, tailPnumMask);475+ Reg::Move<uint16_t, Reg::MaskMergeMode::MERGING>(expMax, exp, tailPnumMask);
476 476 
477 // 二分法求rowsSingleLoop行中的最大行477 // 二分法求rowsSingleLoop行中的最大行
478- Reg::DataCopy(maxExpAddr, expMax, pnumMask);478+ Reg::StoreAlign(maxExpAddr, expMax, pnumMask);
479 Reg::LocalMemBar<Reg::MemType::VEC_STORE, Reg::MemType::VEC_LOAD>();479 Reg::LocalMemBar<Reg::MemType::VEC_STORE, Reg::MemType::VEC_LOAD>();
480 uint32_t maskNum = dataLenSingleLoop - expOffset;480 uint32_t maskNum = dataLenSingleLoop - expOffset;
481 Reg::MaskReg mask = Reg::UpdateMask<uint16_t>(maskNum);481 Reg::MaskReg mask = Reg::UpdateMask<uint16_t>(maskNum);
482- Reg::DataCopyUnAlignPre(u0, maxExpAddr + expOffset);482+ Reg::LoadUnAlignPre(u0, maxExpAddr + expOffset);
483- Reg::DataCopyUnAlign(exp, u0, maxExpAddr + expOffset);483+ Reg::LoadUnAlign(exp, u0, maxExpAddr + expOffset);
484 Reg::Max(exp, expMax, exp, mask);484 Reg::Max(exp, expMax, exp, mask);
485- Reg::Copy<uint16_t, Reg::MaskMergeMode::MERGING>(expMax, exp, mask);485+ Reg::Move<uint16_t, Reg::MaskMergeMode::MERGING>(expMax, exp, mask);
486 for (uint16_t i = 0; i < loopSize; i++) {486 for (uint16_t i = 0; i < loopSize; i++) {
487- Reg::DataCopy(maxExpAddr, expMax, pnumMask);487+ Reg::StoreAlign(maxExpAddr, expMax, pnumMask);
488 Reg::LocalMemBar<Reg::MemType::VEC_STORE, Reg::MemType::VEC_LOAD>();488 Reg::LocalMemBar<Reg::MemType::VEC_STORE, Reg::MemType::VEC_LOAD>();
489 expOffset /= DIGIT_TWO;489 expOffset /= DIGIT_TWO;
490 maskNum = expOffset;490 maskNum = expOffset;
491 mask = Reg::UpdateMask<uint16_t>(maskNum);491 mask = Reg::UpdateMask<uint16_t>(maskNum);
492- Reg::DataCopyUnAlignPre(u0, maxExpAddr + expOffset);492+ Reg::LoadUnAlignPre(u0, maxExpAddr + expOffset);
493- Reg::DataCopyUnAlign(exp, u0, maxExpAddr + expOffset);493+ Reg::LoadUnAlign(exp, u0, maxExpAddr + expOffset);
494 Reg::Max(expMax, expMax, exp, mask);494 Reg::Max(expMax, expMax, exp, mask);
495 }495 }
496 maskNum = static_cast<uint32_t>(dataLen);496 maskNum = static_cast<uint32_t>(dataLen);
@@ -514,14 +514,14 @@ __aicore__ inline void DynamicMxQuantNotTailAxisOptimizeFP8<T, U, isTail>::Compu
514 // And获取尾数位514 // And获取尾数位
515 Reg::And(manFP32RegTensor, maxFP32RegTensor, manFP32RegTensor, mask);515 Reg::And(manFP32RegTensor, maxFP32RegTensor, manFP32RegTensor, mask);
516 516 
517- Reg::CompareScalar<uint32_t, CMPMODE::GT>(p0, expFP32RegTensor, FP32_NUMBER_ZERO, mask);517+ Reg::Compares<uint32_t, CMPMODE::GT>(p0, expFP32RegTensor, FP32_NUMBER_ZERO, mask);
518- Reg::CompareScalar<uint32_t, CMPMODE::LT>(p0, expFP32RegTensor, FP32_NUMBER_254, p0);518+ Reg::Compares<uint32_t, CMPMODE::LT>(p0, expFP32RegTensor, FP32_NUMBER_254, p0);
519- Reg::CompareScalar<uint32_t, CMPMODE::GT>(p0, manFP32RegTensor, FP32_NUMBER_ZERO, p0);519+ Reg::Compares<uint32_t, CMPMODE::GT>(p0, manFP32RegTensor, FP32_NUMBER_ZERO, p0);
520 520 
521- Reg::CompareScalar<uint32_t, CMPMODE::EQ>(p1, expFP32RegTensor, FP32_NUMBER_ZERO, mask);521+ Reg::Compares<uint32_t, CMPMODE::EQ>(p1, expFP32RegTensor, FP32_NUMBER_ZERO, mask);
522- Reg::CompareScalar<uint32_t, CMPMODE::GT>(p1, manFP32RegTensor, FP32_NUMBER_HALF, p1);522+ Reg::Compares<uint32_t, CMPMODE::GT>(p1, manFP32RegTensor, FP32_NUMBER_HALF, p1);
523 523 
524- Reg::MaskOr(p0, p0, p1, mask);524+ Reg::Or(p0, p0, p1, mask);
525 Reg::Adds(extractExpRegTensor, expFP32RegTensor, 1, mask);525 Reg::Adds(extractExpRegTensor, expFP32RegTensor, 1, mask);
526 // 根据情况选择指数位是否加一526 // 根据情况选择指数位是否加一
527 Reg::Select<uint32_t>(extractExpRegTensor, extractExpRegTensor, expFP32RegTensor, p0);527 Reg::Select<uint32_t>(extractExpRegTensor, extractExpRegTensor, expFP32RegTensor, p0);
@@ -531,8 +531,8 @@ __aicore__ inline void DynamicMxQuantNotTailAxisOptimizeFP8<T, U, isTail>::Compu
531 Reg::Pack(tmp16RegTensor, extractExpRegTensor);531 Reg::Pack(tmp16RegTensor, extractExpRegTensor);
532 Reg::Pack(outZero, tmp16RegTensor);532 Reg::Pack(outZero, tmp16RegTensor);
533 // 搬出mxScale533 // 搬出mxScale
534- Reg::DataCopyUnAlign(mxScaleAddr, outZero, u1, dataLen);534+ Reg::StoreUnAlign(mxScaleAddr, outZero, u1, dataLen);
535- Reg::DataCopyUnAlignPost(mxScaleAddr, u1, 0);535+ Reg::StoreUnAlignPost(mxScaleAddr, u1, 0);
536 536 
537 // 求1/scale537 // 求1/scale
538 Reg::ShiftLefts(extractExpRegTensor, extractExpRegTensor, FP32_SHR_NUM, mask);538 Reg::ShiftLefts(extractExpRegTensor, extractExpRegTensor, FP32_SHR_NUM, mask);
@@ -545,11 +545,11 @@ __aicore__ inline void DynamicMxQuantNotTailAxisOptimizeFP8<T, U, isTail>::Compu
545 Reg::DeInterleave(reversedShareExpRegTensor, tmp16RegTensor, reversedShareExpRegTensor, tmp16RegTensor);545 Reg::DeInterleave(reversedShareExpRegTensor, tmp16RegTensor, reversedShareExpRegTensor, tmp16RegTensor);
546 auto scaleAddr = maxExpAddr;546 auto scaleAddr = maxExpAddr;
547 for (uint16_t i = 0; i < rowsSingleLoop; i++) {547 for (uint16_t i = 0; i < rowsSingleLoop; i++) {
548- Reg::DataCopyUnAlign(scaleAddr, reversedShareExpRegTensor, u1, dataLen);548+ Reg::StoreUnAlign(scaleAddr, reversedShareExpRegTensor, u1, dataLen);
549- Reg::DataCopyUnAlignPost(scaleAddr, u1, 0);549+ Reg::StoreUnAlignPost(scaleAddr, u1, 0);
550 }550 }
551 Reg::LocalMemBar<Reg::MemType::VEC_STORE, Reg::MemType::VEC_LOAD>();551 Reg::LocalMemBar<Reg::MemType::VEC_STORE, Reg::MemType::VEC_LOAD>();
552- Reg::DataCopy(reversedShareExpRegTensor, maxExpAddr);552+ Reg::LoadAlign(reversedShareExpRegTensor, maxExpAddr);
553 mask = Reg::CreateMask<uint16_t, Reg::MaskPattern::ALL>();553 mask = Reg::CreateMask<uint16_t, Reg::MaskPattern::ALL>();
554 if constexpr (IsSame<T, half>::value) {554 if constexpr (IsSame<T, half>::value) {
555 Reg::Cast<float, bfloat16_t, castTraitZero>(555 Reg::Cast<float, bfloat16_t, castTraitZero>(
@@ -578,8 +578,8 @@ __aicore__ inline void DynamicMxQuantNotTailAxisOptimizeFP8<T, U, isTail>::Compu
578 Reg::Interleave(tmp16RegTensor, exp, tmp16RegTensor, exp);578 Reg::Interleave(tmp16RegTensor, exp, tmp16RegTensor, exp);
579 Reg::Pack(outZero, tmp16RegTensor);579 Reg::Pack(outZero, tmp16RegTensor);
580 auto addr0 = yAddr + i * dataLenSingleLoop;580 auto addr0 = yAddr + i * dataLenSingleLoop;
581- Reg::DataCopyUnAlign(addr0, outZero, u1, dataLenSingleLoop);581+ Reg::StoreUnAlign(addr0, outZero, u1, dataLenSingleLoop);
582- Reg::DataCopyUnAlignPost(addr0, u1, 0);582+ Reg::StoreUnAlignPost(addr0, u1, 0);
583 }583 }
584 }584 }
585}585}
Mquant/dynamic_mx_quant/op_kernel/arch35/dynamic_mx_quant_not_tail_axis_optimize_high_perf_large_tail.h+57-55
@@ -416,8 +416,8 @@ DynamicMxQuantNotTailAxisOptimizeLargeTail<xDtype, yDtype, roundMode, calcMode>:
416 if constexpr (IsSame<xDtype, float>::value) {416 if constexpr (IsSame<xDtype, float>::value) {
417 Reg::Duplicate(manAbs0FP32, 0);417 Reg::Duplicate(manAbs0FP32, 0);
418 for (uint16_t j = 0; j <= loopNum; j++) {418 for (uint16_t j = 0; j <= loopNum; j++) {
419- DataCopy((Reg::RegTensor<xDtype>&)x0U32, xAddr + j * dataLen8Align_);419+ Reg::LoadAlign((Reg::RegTensor<xDtype>&)x0U32, xAddr + j * dataLen8Align_);
420- DataCopy((Reg::RegTensor<xDtype>&)x1U32, xAddr + (blockCount - j - 1) * dataLen8Align_);420+ Reg::LoadAlign((Reg::RegTensor<xDtype>&)x1U32, xAddr + (blockCount - j - 1) * dataLen8Align_);
421 Reg::And(x0AbsU32, x0U32, absForXU32, pregAll32);421 Reg::And(x0AbsU32, x0U32, absForXU32, pregAll32);
422 Reg::And(x1AbsU32, x1U32, absForXU32, pregAll32);422 Reg::And(x1AbsU32, x1U32, absForXU32, pregAll32);
423 Reg::Max(x0AbsU32, x1AbsU32, x0AbsU32, pregAll32);423 Reg::Max(x0AbsU32, x1AbsU32, x0AbsU32, pregAll32);
@@ -425,8 +425,8 @@ DynamicMxQuantNotTailAxisOptimizeLargeTail<xDtype, yDtype, roundMode, calcMode>:
425 }425 }
426 } else {426 } else {
427 for (uint16_t j = 0; j <= loopNum; j++) {427 for (uint16_t j = 0; j <= loopNum; j++) {
428- DataCopy((Reg::RegTensor<xDtype>&)x0U16, xAddr + j * dataLen16Align_);428+ Reg::LoadAlign((Reg::RegTensor<xDtype>&)x0U16, xAddr + j * dataLen16Align_);
429- DataCopy((Reg::RegTensor<xDtype>&)x1U16, xAddr + (blockCount - j - 1) * dataLen16Align_);429+ Reg::LoadAlign((Reg::RegTensor<xDtype>&)x1U16, xAddr + (blockCount - j - 1) * dataLen16Align_);
430 Reg::And(x0AbsU16, x0U16, absForXU16, pregAll16);430 Reg::And(x0AbsU16, x0U16, absForXU16, pregAll16);
431 Reg::And(x1AbsU16, x1U16, absForXU16, pregAll16);431 Reg::And(x1AbsU16, x1U16, absForXU16, pregAll16);
432 Reg::Max(x0AbsU16, x1AbsU16, x0AbsU16, pregAll16);432 Reg::Max(x0AbsU16, x1AbsU16, x0AbsU16, pregAll16);
@@ -440,8 +440,8 @@ DynamicMxQuantNotTailAxisOptimizeLargeTail<xDtype, yDtype, roundMode, calcMode>:
440 }440 }
441 441 
442 if constexpr (calcMode == MODE_ONE) {442 if constexpr (calcMode == MODE_ONE) {
443- Reg::CompareScalar<uint32_t, CMPMODE::NE>(pZeroBlock, (Reg::RegTensor<uint32_t>&)manAbs0FP32,443+ Reg::Compares<uint32_t, CMPMODE::NE>(pZeroBlock, (Reg::RegTensor<uint32_t>&)manAbs0FP32, FP32_NUMBER_ZERO,
444- FP32_NUMBER_ZERO, pregAll32);444+ pregAll32);
445 Reg::Maxs((Reg::RegTensor<float>&)manAbs0FP32, (Reg::RegTensor<float>&)manAbs0FP32, maxLowBound_,445 Reg::Maxs((Reg::RegTensor<float>&)manAbs0FP32, (Reg::RegTensor<float>&)manAbs0FP32, maxLowBound_,
446 pregAll32);446 pregAll32);
447 Reg::Mul((Reg::RegTensor<float>&)manAbs0FP32, (Reg::RegTensor<float>&)manAbs0FP32,447 Reg::Mul((Reg::RegTensor<float>&)manAbs0FP32, (Reg::RegTensor<float>&)manAbs0FP32,
@@ -454,14 +454,14 @@ DynamicMxQuantNotTailAxisOptimizeLargeTail<xDtype, yDtype, roundMode, calcMode>:
454 Reg::ShiftRights(mxScale0FP32, manAbs0FP32, FP32_SHR_NUM, pregAll32);454 Reg::ShiftRights(mxScale0FP32, manAbs0FP32, FP32_SHR_NUM, pregAll32);
455 // 提取尾数455 // 提取尾数
456 Reg::And(manAbs0FP32, manAbs0FP32, manForFP32, pregAll32);456 Reg::And(manAbs0FP32, manAbs0FP32, manForFP32, pregAll32);
457- Reg::CompareScalar<uint32_t, CMPMODE::GT>(p0, mxScale0FP32, FP32_NUMBER_ZERO, pregAll32);457+ Reg::Compares<uint32_t, CMPMODE::GT>(p0, mxScale0FP32, FP32_NUMBER_ZERO, pregAll32);
458- Reg::CompareScalar<uint32_t, CMPMODE::LT>(p0, mxScale0FP32, FP32_NUMBER_254, p0);458+ Reg::Compares<uint32_t, CMPMODE::LT>(p0, mxScale0FP32, FP32_NUMBER_254, p0);
459- Reg::CompareScalar<uint32_t, CMPMODE::GT>(p0, manAbs0FP32, FP32_NUMBER_ZERO, p0);459+ Reg::Compares<uint32_t, CMPMODE::GT>(p0, manAbs0FP32, FP32_NUMBER_ZERO, p0);
460 460 
461 if constexpr (calcMode == MODE_ONE) {461 if constexpr (calcMode == MODE_ONE) {
462- Reg::CompareScalar<uint32_t, CMPMODE::EQ>(p1, mxScale0FP32, FP32_NUMBER_ZERO, pregAll32);462+ Reg::Compares<uint32_t, CMPMODE::EQ>(p1, mxScale0FP32, FP32_NUMBER_ZERO, pregAll32);
463- Reg::CompareScalar<uint32_t, CMPMODE::GT>(p1, manAbs0FP32, FP32_NUMBER_HALF, p1);463+ Reg::Compares<uint32_t, CMPMODE::GT>(p1, manAbs0FP32, FP32_NUMBER_HALF, p1);
464- Reg::MaskXor(p0, p0, p1, pregAll32);464+ Reg::Xor(p0, p0, p1, pregAll32);
465 }465 }
466 466 
467 Reg::Adds(manAbs0FP32, mxScale0FP32, 1, p0);467 Reg::Adds(manAbs0FP32, mxScale0FP32, 1, p0);
@@ -475,7 +475,7 @@ DynamicMxQuantNotTailAxisOptimizeLargeTail<xDtype, yDtype, roundMode, calcMode>:
475 475 
476 if constexpr (!IsSame<xDtype, float>::value) {476 if constexpr (!IsSame<xDtype, float>::value) {
477 if constexpr (calcMode == MODE_ONE) {477 if constexpr (calcMode == MODE_ONE) {
478- Reg::CompareScalar<uint32_t, CMPMODE::NE>(pZeroBlock, manAbs1FP32, FP32_NUMBER_ZERO, pregAll32);478+ Reg::Compares<uint32_t, CMPMODE::NE>(pZeroBlock, manAbs1FP32, FP32_NUMBER_ZERO, pregAll32);
479 Reg::Maxs((Reg::RegTensor<float>&)manAbs1FP32, (Reg::RegTensor<float>&)manAbs1FP32, maxLowBound_,479 Reg::Maxs((Reg::RegTensor<float>&)manAbs1FP32, (Reg::RegTensor<float>&)manAbs1FP32, maxLowBound_,
480 pregAll32);480 pregAll32);
481 Reg::Mul((Reg::RegTensor<float>&)manAbs1FP32, (Reg::RegTensor<float>&)manAbs1FP32,481 Reg::Mul((Reg::RegTensor<float>&)manAbs1FP32, (Reg::RegTensor<float>&)manAbs1FP32,
@@ -488,14 +488,14 @@ DynamicMxQuantNotTailAxisOptimizeLargeTail<xDtype, yDtype, roundMode, calcMode>:
488 Reg::ShiftRights(mxScale1FP32, manAbs1FP32, FP32_SHR_NUM, pregAll32);488 Reg::ShiftRights(mxScale1FP32, manAbs1FP32, FP32_SHR_NUM, pregAll32);
489 // 提取尾数489 // 提取尾数
490 Reg::And(manAbs1FP32, manAbs1FP32, manForFP32, pregAll32);490 Reg::And(manAbs1FP32, manAbs1FP32, manForFP32, pregAll32);
491- Reg::CompareScalar<uint32_t, CMPMODE::GT>(p2, mxScale1FP32, FP32_NUMBER_ZERO, pregAll32);491+ Reg::Compares<uint32_t, CMPMODE::GT>(p2, mxScale1FP32, FP32_NUMBER_ZERO, pregAll32);
492- Reg::CompareScalar<uint32_t, CMPMODE::LT>(p2, mxScale1FP32, FP32_NUMBER_254, p2);492+ Reg::Compares<uint32_t, CMPMODE::LT>(p2, mxScale1FP32, FP32_NUMBER_254, p2);
493- Reg::CompareScalar<uint32_t, CMPMODE::GT>(p2, manAbs1FP32, FP32_NUMBER_ZERO, p2);493+ Reg::Compares<uint32_t, CMPMODE::GT>(p2, manAbs1FP32, FP32_NUMBER_ZERO, p2);
494 494 
495 if constexpr (calcMode == MODE_ONE) {495 if constexpr (calcMode == MODE_ONE) {
496- Reg::CompareScalar<uint32_t, CMPMODE::EQ>(p3, mxScale1FP32, FP32_NUMBER_ZERO, pregAll32);496+ Reg::Compares<uint32_t, CMPMODE::EQ>(p3, mxScale1FP32, FP32_NUMBER_ZERO, pregAll32);
497- Reg::CompareScalar<uint32_t, CMPMODE::GT>(p3, manAbs1FP32, FP32_NUMBER_HALF, p3);497+ Reg::Compares<uint32_t, CMPMODE::GT>(p3, manAbs1FP32, FP32_NUMBER_HALF, p3);
498- Reg::MaskXor(p2, p3, p2, pregAll32);498+ Reg::Xor(p2, p3, p2, pregAll32);
499 }499 }
500 500 
501 Reg::Adds(manAbs1FP32, mxScale1FP32, 1, p2);501 Reg::Adds(manAbs1FP32, mxScale1FP32, 1, p2);
@@ -514,7 +514,7 @@ DynamicMxQuantNotTailAxisOptimizeLargeTail<xDtype, yDtype, roundMode, calcMode>:
514 Reg::ShiftLefts(mxScaleBF16, mxScale0BF16, BF16_SHR_NUM, pregAll16);514 Reg::ShiftLefts(mxScaleBF16, mxScale0BF16, BF16_SHR_NUM, pregAll16);
515 Reg::Pack<uint8_t, uint16_t, Reg::HighLowPart::LOWEST>((Reg::RegTensor<uint8_t>&)mxScale, mxScale0BF16);515 Reg::Pack<uint8_t, uint16_t, Reg::HighLowPart::LOWEST>((Reg::RegTensor<uint8_t>&)mxScale, mxScale0BF16);
516 516 
517- DataCopy(mxScaleAddr, mxScale, pregAll8);517+ Reg::StoreAlign(mxScaleAddr, mxScale, pregAll8);
518 518 
519 // 求1/scale519 // 求1/scale
520 Reg::Compare<uint16_t, CMPMODE::NE>(p0, mxScaleBF16, maxEleU16, pregAll16);520 Reg::Compare<uint16_t, CMPMODE::NE>(p0, mxScaleBF16, maxEleU16, pregAll16);
@@ -522,7 +522,7 @@ DynamicMxQuantNotTailAxisOptimizeLargeTail<xDtype, yDtype, roundMode, calcMode>:
522 Reg::Sub(reversedShareExpBF16, biasU16, mxScaleBF16, pregAll16);522 Reg::Sub(reversedShareExpBF16, biasU16, mxScaleBF16, pregAll16);
523 Reg::Select<uint16_t>(reversedShareExpBF16, reversedShareExpBF16, nanU16, p0);523 Reg::Select<uint16_t>(reversedShareExpBF16, reversedShareExpBF16, nanU16, p0);
524 Reg::Select<uint16_t>(reversedShareExpBF16, specialExpU16, reversedShareExpBF16, p1);524 Reg::Select<uint16_t>(reversedShareExpBF16, specialExpU16, reversedShareExpBF16, p1);
525- DataCopy(tmpAddr, reversedShareExpBF16, pregAll16);525+ Reg::StoreAlign(tmpAddr, reversedShareExpBF16, pregAll16);
526 }526 }
527}527}
528 528 
@@ -602,15 +602,15 @@ DynamicMxQuantNotTailAxisOptimizeLargeTail<xDtype, yDtype, roundMode, calcMode>:
602 602 
603 for (uint16_t j = 0; j <= loopNum; j++) {603 for (uint16_t j = 0; j <= loopNum; j++) {
604 if constexpr (IsSame<xDtype, float>::value) {604 if constexpr (IsSame<xDtype, float>::value) {
605- DataCopy((Reg::RegTensor<xDtype>&)x0U32, xAddr + j * dataLen8Align_);605+ Reg::LoadAlign((Reg::RegTensor<xDtype>&)x0U32, xAddr + j * dataLen8Align_);
606- DataCopy((Reg::RegTensor<xDtype>&)x1U32, xAddr + (blockCount - j - 1) * dataLen8Align_);606+ Reg::LoadAlign((Reg::RegTensor<xDtype>&)x1U32, xAddr + (blockCount - j - 1) * dataLen8Align_);
607 Reg::And(x0AbsU32, x0U32, absForXU32, pregAll32);607 Reg::And(x0AbsU32, x0U32, absForXU32, pregAll32);
608 Reg::And(x1AbsU32, x1U32, absForXU32, pregAll32);608 Reg::And(x1AbsU32, x1U32, absForXU32, pregAll32);
609 Reg::Max(x0AbsU32, x0AbsU32, x1AbsU32, pregAll32);609 Reg::Max(x0AbsU32, x0AbsU32, x1AbsU32, pregAll32);
610 Reg::Max(maxU32, maxU32, x0AbsU32, pregAll32);610 Reg::Max(maxU32, maxU32, x0AbsU32, pregAll32);
611 } else {611 } else {
612- DataCopy((Reg::RegTensor<xDtype>&)x0U16, xAddr + j * dataLen16Align_);612+ Reg::LoadAlign((Reg::RegTensor<xDtype>&)x0U16, xAddr + j * dataLen16Align_);
613- DataCopy((Reg::RegTensor<xDtype>&)x1U16, xAddr + (blockCount - j - 1) * dataLen16Align_);613+ Reg::LoadAlign((Reg::RegTensor<xDtype>&)x1U16, xAddr + (blockCount - j - 1) * dataLen16Align_);
614 Reg::And(x0AbsU16, x0U16, absForXU16, pregAll16);614 Reg::And(x0AbsU16, x0U16, absForXU16, pregAll16);
615 Reg::And(x1AbsU16, x1U16, absForXU16, pregAll16);615 Reg::And(x1AbsU16, x1U16, absForXU16, pregAll16);
616 if constexpr (IsSame<xDtype, half>::value) {616 if constexpr (IsSame<xDtype, half>::value) {
@@ -646,7 +646,7 @@ DynamicMxQuantNotTailAxisOptimizeLargeTail<xDtype, yDtype, roundMode, calcMode>:
646 Reg::Select<uint16_t>(mxScaleBF16, mxScaleBF16, fp8NanU16, infMask);646 Reg::Select<uint16_t>(mxScaleBF16, mxScaleBF16, fp8NanU16, infMask);
647 647 
648 Reg::Pack<uint8_t, uint16_t, Reg::HighLowPart::LOWEST>((Reg::RegTensor<uint8_t>&)mxScale, mxScaleBF16);648 Reg::Pack<uint8_t, uint16_t, Reg::HighLowPart::LOWEST>((Reg::RegTensor<uint8_t>&)mxScale, mxScaleBF16);
649- DataCopy(mxScaleAddr, mxScale, pregAll8);649+ Reg::StoreAlign(mxScaleAddr, mxScale, pregAll8);
650 // 求1/scale650 // 求1/scale
651 Reg::And(expMaxU16, expMaxU16, maxEleU16, pregAll16);651 Reg::And(expMaxU16, expMaxU16, maxEleU16, pregAll16);
652 Reg::Compare<uint16_t, CMPMODE::NE>(zeroMask, expMaxU16, zeroU16, pregAll16);652 Reg::Compare<uint16_t, CMPMODE::NE>(zeroMask, expMaxU16, zeroU16, pregAll16);
@@ -655,7 +655,7 @@ DynamicMxQuantNotTailAxisOptimizeLargeTail<xDtype, yDtype, roundMode, calcMode>:
655 Reg::Select<uint16_t>(reversedShareExpBF16, reversedShareExpBF16, nanU16, infMask);655 Reg::Select<uint16_t>(reversedShareExpBF16, reversedShareExpBF16, nanU16, infMask);
656 Reg::Select<uint16_t>(reversedShareExpBF16, reversedShareExpBF16, zeroU16, zeroMask);656 Reg::Select<uint16_t>(reversedShareExpBF16, reversedShareExpBF16, zeroU16, zeroMask);
657 Reg::Select<uint16_t>(reversedShareExpBF16, specialExpU16, reversedShareExpBF16, invalidDataMask);657 Reg::Select<uint16_t>(reversedShareExpBF16, specialExpU16, reversedShareExpBF16, invalidDataMask);
658- DataCopy(tmpAddr, reversedShareExpBF16, pregAll16);658+ Reg::StoreAlign(tmpAddr, reversedShareExpBF16, pregAll16);
659 }659 }
660}660}
661 661 
@@ -727,15 +727,15 @@ __aicore__ inline void DynamicMxQuantNotTailAxisOptimizeLargeTail<xDtype, yDtype
727 727 
728 for (uint16_t j = 0; j <= loopNum; j++) {728 for (uint16_t j = 0; j <= loopNum; j++) {
729 if constexpr (IsSame<xDtype, float>::value) {729 if constexpr (IsSame<xDtype, float>::value) {
730- DataCopy((Reg::RegTensor<xDtype>&)x0U32, xAddr + j * dataLen8Align_);730+ Reg::LoadAlign((Reg::RegTensor<xDtype>&)x0U32, xAddr + j * dataLen8Align_);
731- DataCopy((Reg::RegTensor<xDtype>&)x1U32, xAddr + (blockCount - j - 1) * dataLen8Align_);731+ Reg::LoadAlign((Reg::RegTensor<xDtype>&)x1U32, xAddr + (blockCount - j - 1) * dataLen8Align_);
732 Reg::And(exp0U32, x0U32, maxEleU32, pregAll32);732 Reg::And(exp0U32, x0U32, maxEleU32, pregAll32);
733 Reg::And(exp1U32, x1U32, maxEleU32, pregAll32);733 Reg::And(exp1U32, x1U32, maxEleU32, pregAll32);
734 Reg::Max(exp0U32, exp0U32, exp1U32, pregAll32);734 Reg::Max(exp0U32, exp0U32, exp1U32, pregAll32);
735 Reg::Max(expMaxU32, expMaxU32, exp0U32, pregAll32);735 Reg::Max(expMaxU32, expMaxU32, exp0U32, pregAll32);
736 } else {736 } else {
737- DataCopy((Reg::RegTensor<xDtype>&)x0U16, xAddr + j * dataLen16Align_);737+ Reg::LoadAlign((Reg::RegTensor<xDtype>&)x0U16, xAddr + j * dataLen16Align_);
738- DataCopy((Reg::RegTensor<xDtype>&)x1U16, xAddr + (blockCount - j - 1) * dataLen16Align_);738+ Reg::LoadAlign((Reg::RegTensor<xDtype>&)x1U16, xAddr + (blockCount - j - 1) * dataLen16Align_);
739 if constexpr (IsSame<xDtype, half>::value) {739 if constexpr (IsSame<xDtype, half>::value) {
740 Reg::Cast<bfloat16_t, xDtype, castTraitOcpHalf2Bf16>((Reg::RegTensor<bfloat16_t>&)exp0U16,740 Reg::Cast<bfloat16_t, xDtype, castTraitOcpHalf2Bf16>((Reg::RegTensor<bfloat16_t>&)exp0U16,
741 (Reg::RegTensor<float16_t>&)x0U16, pregAll16);741 (Reg::RegTensor<float16_t>&)x0U16, pregAll16);
@@ -765,7 +765,8 @@ __aicore__ inline void DynamicMxQuantNotTailAxisOptimizeLargeTail<xDtype, yDtype
765 Reg::ShiftRights(mxScaleBF16, expMaxU16, BF16_SHR_NUM, pregAll16);765 Reg::ShiftRights(mxScaleBF16, expMaxU16, BF16_SHR_NUM, pregAll16);
766 Reg::Select<uint16_t>(mxScaleBF16, mxScaleBF16, fp8NanU16, infMask);766 Reg::Select<uint16_t>(mxScaleBF16, mxScaleBF16, fp8NanU16, infMask);
767 767 
768- DataCopy<uint8_t, Reg::StoreDist::DIST_PACK_B16>(mxScaleAddr, (Reg::RegTensor<uint8_t>&)mxScaleBF16, pregAll8);768+ Reg::StoreAlign<uint8_t, Reg::StoreDist::DIST_PACK_B16>(mxScaleAddr, (Reg::RegTensor<uint8_t>&)mxScaleBF16,
769+ pregAll8);
769 770 
770 // 求1/scale771 // 求1/scale
771 Reg::Compare<uint16_t, CMPMODE::NE>(zeroMask, expMaxU16, zeroU16, pregAll16);772 Reg::Compare<uint16_t, CMPMODE::NE>(zeroMask, expMaxU16, zeroU16, pregAll16);
@@ -774,7 +775,7 @@ __aicore__ inline void DynamicMxQuantNotTailAxisOptimizeLargeTail<xDtype, yDtype
774 Reg::Select<uint16_t>(reversedShareExpBF16, reversedShareExpBF16, nanU16, infMask);775 Reg::Select<uint16_t>(reversedShareExpBF16, reversedShareExpBF16, nanU16, infMask);
775 Reg::Select<uint16_t>(reversedShareExpBF16, reversedShareExpBF16, zeroU16, zeroMask);776 Reg::Select<uint16_t>(reversedShareExpBF16, reversedShareExpBF16, zeroU16, zeroMask);
776 Reg::Select<uint16_t>(reversedShareExpBF16, specialExpU16, reversedShareExpBF16, invalidDataMask);777 Reg::Select<uint16_t>(reversedShareExpBF16, specialExpU16, reversedShareExpBF16, invalidDataMask);
777- DataCopy(tmpAddr, reversedShareExpBF16, pregAll16);778+ Reg::StoreAlign(tmpAddr, reversedShareExpBF16, pregAll16);
778 }779 }
779}780}
780 781 
@@ -826,7 +827,7 @@ DynamicMxQuantNotTailAxisOptimizeLargeTail<xDtype, yDtype, roundMode, calcMode>:
826 static constexpr Reg::CastTrait castTraitFp32toYdtype = {Reg::RegLayout::ZERO, Reg::SatMode::SAT,827 static constexpr Reg::CastTrait castTraitFp32toYdtype = {Reg::RegLayout::ZERO, Reg::SatMode::SAT,
827 Reg::MaskMergeMode::ZEROING, RoundMode::CAST_RINT};828 Reg::MaskMergeMode::ZEROING, RoundMode::CAST_RINT};
828 829 
829- Reg::DataCopy<uint16_t, Reg::LoadDist::DIST_NORM>(reversedShareExpBF16, tmpAddr);830+ Reg::LoadAlign<uint16_t, Reg::LoadDist::DIST_NORM>(reversedShareExpBF16, tmpAddr);
830 831 
831 Reg::Cast<float, bfloat16_t, castTraitXdtypetoFp32Zero>(832 Reg::Cast<float, bfloat16_t, castTraitXdtypetoFp32Zero>(
832 reversedShareExp0FP32, (Reg::RegTensor<bfloat16_t>&)reversedShareExpBF16, pregAll16);833 reversedShareExp0FP32, (Reg::RegTensor<bfloat16_t>&)reversedShareExpBF16, pregAll16);
@@ -834,7 +835,7 @@ DynamicMxQuantNotTailAxisOptimizeLargeTail<xDtype, yDtype, roundMode, calcMode>:
834 reversedShareExp1FP32, (Reg::RegTensor<bfloat16_t>&)reversedShareExpBF16, pregAll16);835 reversedShareExp1FP32, (Reg::RegTensor<bfloat16_t>&)reversedShareExpBF16, pregAll16);
835 836 
836 for (uint16_t j = 0; j < blockCount; j++) {837 for (uint16_t j = 0; j < blockCount; j++) {
837- Reg::DataCopy<xDtype, Reg::LoadDist::DIST_NORM>(x, xAddr + j * dataLen16Align_);838+ Reg::LoadAlign<xDtype, Reg::LoadDist::DIST_NORM>(x, xAddr + j * dataLen16Align_);
838 839 
839 Reg::Cast<float, xDtype, castTraitXdtypetoFp32Zero>(x0FP32, x, pregAll16);840 Reg::Cast<float, xDtype, castTraitXdtypetoFp32Zero>(x0FP32, x, pregAll16);
840 Reg::Cast<float, xDtype, castTraitXdtypetoFp32One>(x1FP32, x, pregAll16);841 Reg::Cast<float, xDtype, castTraitXdtypetoFp32One>(x1FP32, x, pregAll16);
@@ -860,8 +861,8 @@ DynamicMxQuantNotTailAxisOptimizeLargeTail<xDtype, yDtype, roundMode, calcMode>:
860 Reg::Cast<yDtype, bfloat16_t, castTraitBf16toFp4>(yZeroFP4, (Reg::RegTensor<bfloat16_t>&)x0BF16,861 Reg::Cast<yDtype, bfloat16_t, castTraitBf16toFp4>(yZeroFP4, (Reg::RegTensor<bfloat16_t>&)x0BF16,
861 pregAll16);862 pregAll16);
862 863 
863- DataCopy<uint8_t, Reg::StoreDist::DIST_PACK4_B32>(yAddr + (j * dataLen64Align_ / DIGIT_TWO),864+ Reg::StoreAlign<uint8_t, Reg::StoreDist::DIST_PACK4_B32>(yAddr + (j * dataLen64Align_ / DIGIT_TWO),
864- (Reg::RegTensor<uint8_t>&)yZeroFP4, pregAll8);865+ (Reg::RegTensor<uint8_t>&)yZeroFP4, pregAll8);
865 } else {866 } else {
866 Reg::Cast<yDtype, float, castTraitFp32toYdtype>(yZeroFP8, (Reg::RegTensor<float>&)x0FP32, pregAll32);867 Reg::Cast<yDtype, float, castTraitFp32toYdtype>(yZeroFP8, (Reg::RegTensor<float>&)x0FP32, pregAll32);
867 Reg::Pack<uint16_t, uint32_t, Reg::HighLowPart::LOWEST>((Reg::RegTensor<uint16_t>&)yZeroFP8,868 Reg::Pack<uint16_t, uint32_t, Reg::HighLowPart::LOWEST>((Reg::RegTensor<uint16_t>&)yZeroFP8,
@@ -874,8 +875,8 @@ DynamicMxQuantNotTailAxisOptimizeLargeTail<xDtype, yDtype, roundMode, calcMode>:
874 Reg::Interleave((Reg::RegTensor<uint16_t>&)yZeroFP8, (Reg::RegTensor<uint16_t>&)yOneFP8,875 Reg::Interleave((Reg::RegTensor<uint16_t>&)yZeroFP8, (Reg::RegTensor<uint16_t>&)yOneFP8,
875 (Reg::RegTensor<uint16_t>&)yZeroFP8, (Reg::RegTensor<uint16_t>&)yOneFP8);876 (Reg::RegTensor<uint16_t>&)yZeroFP8, (Reg::RegTensor<uint16_t>&)yOneFP8);
876 877 
877- DataCopy<uint8_t, Reg::StoreDist::DIST_PACK_B16>(yAddr + (j * dataLen32Align_),878+ Reg::StoreAlign<uint8_t, Reg::StoreDist::DIST_PACK_B16>(yAddr + (j * dataLen32Align_),
878- (Reg::RegTensor<uint8_t>&)yZeroFP8, pregAll8);879+ (Reg::RegTensor<uint8_t>&)yZeroFP8, pregAll8);
879 }880 }
880 }881 }
881 }882 }
@@ -902,7 +903,7 @@ DynamicMxQuantNotTailAxisOptimizeLargeTail<xDtype, yDtype, roundMode, calcMode>:
902 /*negzero*/ negInfMask, (Reg::RegTensor<int32_t>&)Reg, negZeroI32, pregAll32);903 /*negzero*/ negInfMask, (Reg::RegTensor<int32_t>&)Reg, negZeroI32, pregAll32);
903 if constexpr (IsSame<yDtype, fp4x2_e1m2_t>::value) {904 if constexpr (IsSame<yDtype, fp4x2_e1m2_t>::value) {
904 Reg::Muls(Reg, Reg, FP4_SCALE_FACTOR, pregAll32);905 Reg::Muls(Reg, Reg, FP4_SCALE_FACTOR, pregAll32);
905- Reg::CompareScalar<float, CMPMODE::LT>(/*negvalue*/ specialMask, Reg, 0, pregAll32);906+ Reg::Compares<float, CMPMODE::LT>(/*negvalue*/ specialMask, Reg, 0, pregAll32);
906 Reg::Truncate<float, roundMode>(Reg, Reg, pregAll32);907 Reg::Truncate<float, roundMode>(Reg, Reg, pregAll32);
907 Reg::Muls(Reg, Reg, FP4_INV_SCALE_FACTOR, pregAll32);908 Reg::Muls(Reg, Reg, FP4_INV_SCALE_FACTOR, pregAll32);
908 } else { // fp4x2_e2m1909 } else { // fp4x2_e2m1
@@ -919,13 +920,13 @@ DynamicMxQuantNotTailAxisOptimizeLargeTail<xDtype, yDtype, roundMode, calcMode>:
919 Reg::Mul(Reg, Reg, (Reg::RegTensor<float>&)exp1FP32, pregAll32);920 Reg::Mul(Reg, Reg, (Reg::RegTensor<float>&)exp1FP32, pregAll32);
920 Reg::Adds(exp0FP32, exp0FP32, FP32_BIAS_VALUE, pregAll32);921 Reg::Adds(exp0FP32, exp0FP32, FP32_BIAS_VALUE, pregAll32);
921 Reg::ShiftLefts(exp0FP32, exp0FP32, FP32_SHR_NUM, pregAll32);922 Reg::ShiftLefts(exp0FP32, exp0FP32, FP32_SHR_NUM, pregAll32);
922- Reg::CompareScalar<float, CMPMODE::LT>(/*negvalue*/ specialMask, Reg, 0, pregAll32);923+ Reg::Compares<float, CMPMODE::LT>(/*negvalue*/ specialMask, Reg, 0, pregAll32);
923 Reg::Truncate<float, roundMode>(Reg, Reg, pregAll32);924 Reg::Truncate<float, roundMode>(Reg, Reg, pregAll32);
924 Reg::Mul(Reg, Reg, (Reg::RegTensor<float>&)exp0FP32, pregAll32);925 Reg::Mul(Reg, Reg, (Reg::RegTensor<float>&)exp0FP32, pregAll32);
925 }926 }
926- Reg::CompareScalar<float, CMPMODE::EQ>(zeroMask, Reg, 0, pregAll32);927+ Reg::Compares<float, CMPMODE::EQ>(zeroMask, Reg, 0, pregAll32);
927- Reg::MaskAnd(zeroMask, specialMask, zeroMask, pregAll32);928+ Reg::And(zeroMask, specialMask, zeroMask, pregAll32);
928- Reg::MaskOr(zeroMask, negInfMask, zeroMask, pregAll32);929+ Reg::Or(zeroMask, negInfMask, zeroMask, pregAll32);
929 Reg::Select<int32_t>((Reg::RegTensor<int32_t>&)Reg, negZeroI32, (Reg::RegTensor<int32_t>&)Reg, zeroMask);930 Reg::Select<int32_t>((Reg::RegTensor<int32_t>&)Reg, negZeroI32, (Reg::RegTensor<int32_t>&)Reg, zeroMask);
930}931}
931 932 
@@ -960,7 +961,7 @@ DynamicMxQuantNotTailAxisOptimizeLargeTail<xDtype, yDtype, roundMode, calcMode>:
960 static constexpr Reg::CastTrait castTraitFp32toYdtype = {Reg::RegLayout::ZERO, Reg::SatMode::SAT,961 static constexpr Reg::CastTrait castTraitFp32toYdtype = {Reg::RegLayout::ZERO, Reg::SatMode::SAT,
961 Reg::MaskMergeMode::ZEROING, RoundMode::CAST_RINT};962 Reg::MaskMergeMode::ZEROING, RoundMode::CAST_RINT};
962 963 
963- Reg::DataCopy<uint16_t, Reg::LoadDist::DIST_NORM>(reversedShareExpBF16, tmpAddr);964+ Reg::LoadAlign<uint16_t, Reg::LoadDist::DIST_NORM>(reversedShareExpBF16, tmpAddr);
964 965 
965 Reg::Cast<float, bfloat16_t, castTraitXdtypetoFp32Zero>(966 Reg::Cast<float, bfloat16_t, castTraitXdtypetoFp32Zero>(
966 reversedShareExp0FP32, (Reg::RegTensor<bfloat16_t>&)reversedShareExpBF16, pregAll16);967 reversedShareExp0FP32, (Reg::RegTensor<bfloat16_t>&)reversedShareExpBF16, pregAll16);
@@ -968,14 +969,14 @@ DynamicMxQuantNotTailAxisOptimizeLargeTail<xDtype, yDtype, roundMode, calcMode>:
968 reversedShareExp1FP32, (Reg::RegTensor<bfloat16_t>&)reversedShareExpBF16, pregAll16);969 reversedShareExp1FP32, (Reg::RegTensor<bfloat16_t>&)reversedShareExpBF16, pregAll16);
969 970 
970 for (uint16_t j = 0; j < blockCount; j++) {971 for (uint16_t j = 0; j < blockCount; j++) {
971- Reg::DataCopy<xDtype, Reg::LoadDist::DIST_NORM>(x, xAddr + j * dataLen16Align_);972+ Reg::LoadAlign<xDtype, Reg::LoadDist::DIST_NORM>(x, xAddr + j * dataLen16Align_);
972 973 
973 if constexpr (IsSame<yDtype, fp4x2_e2m1_t>::value || IsSame<yDtype, fp4x2_e1m2_t>::value) {974 if constexpr (IsSame<yDtype, fp4x2_e2m1_t>::value || IsSame<yDtype, fp4x2_e1m2_t>::value) {
974 Reg::Mul(valueBF16, x, (Reg::RegTensor<bfloat16_t>&)reversedShareExpBF16, pregAll16);975 Reg::Mul(valueBF16, x, (Reg::RegTensor<bfloat16_t>&)reversedShareExpBF16, pregAll16);
975 Reg::Cast<yDtype, bfloat16_t, castTraitBf16toFp4>(yZeroFP4, valueBF16, pregAll16);976 Reg::Cast<yDtype, bfloat16_t, castTraitBf16toFp4>(yZeroFP4, valueBF16, pregAll16);
976 977 
977- DataCopy<uint8_t, Reg::StoreDist::DIST_PACK4_B32>(yAddr + (j * dataLen64Align_ / DIGIT_TWO),978+ Reg::StoreAlign<uint8_t, Reg::StoreDist::DIST_PACK4_B32>(yAddr + (j * dataLen64Align_ / DIGIT_TWO),
978- (Reg::RegTensor<uint8_t>&)yZeroFP4, pregAll8);979+ (Reg::RegTensor<uint8_t>&)yZeroFP4, pregAll8);
979 } else {980 } else {
980 Reg::Cast<float, xDtype, castTraitXdtypetoFp32Zero>(x0FP32, x, pregAll16);981 Reg::Cast<float, xDtype, castTraitXdtypetoFp32Zero>(x0FP32, x, pregAll16);
981 Reg::Cast<float, xDtype, castTraitXdtypetoFp32One>(x1FP32, x, pregAll16);982 Reg::Cast<float, xDtype, castTraitXdtypetoFp32One>(x1FP32, x, pregAll16);
@@ -993,8 +994,8 @@ DynamicMxQuantNotTailAxisOptimizeLargeTail<xDtype, yDtype, roundMode, calcMode>:
993 Reg::Interleave((Reg::RegTensor<uint16_t>&)yZeroFP8, (Reg::RegTensor<uint16_t>&)yOneFP8,994 Reg::Interleave((Reg::RegTensor<uint16_t>&)yZeroFP8, (Reg::RegTensor<uint16_t>&)yOneFP8,
994 (Reg::RegTensor<uint16_t>&)yZeroFP8, (Reg::RegTensor<uint16_t>&)yOneFP8);995 (Reg::RegTensor<uint16_t>&)yZeroFP8, (Reg::RegTensor<uint16_t>&)yOneFP8);
995 996 
996- DataCopy<uint8_t, Reg::StoreDist::DIST_PACK_B16>(yAddr + (j * dataLen32Align_),997+ Reg::StoreAlign<uint8_t, Reg::StoreDist::DIST_PACK_B16>(yAddr + (j * dataLen32Align_),
997- (Reg::RegTensor<uint8_t>&)yZeroFP8, pregAll8);998+ (Reg::RegTensor<uint8_t>&)yZeroFP8, pregAll8);
998 }999 }
999 }1000 }
1000 }1001 }
@@ -1040,12 +1041,13 @@ DynamicMxQuantNotTailAxisOptimizeLargeTail<xDtype, yDtype, roundMode, calcMode>:
1040 static constexpr Reg::CastTrait castTraitFp32toBF16One = {Reg::RegLayout::ONE, Reg::SatMode::NO_SAT,1041 static constexpr Reg::CastTrait castTraitFp32toBF16One = {Reg::RegLayout::ONE, Reg::SatMode::NO_SAT,
1041 Reg::MaskMergeMode::ZEROING, roundMode};1042 Reg::MaskMergeMode::ZEROING, roundMode};
1042 1043 
1043- DataCopy<uint16_t, Reg::LoadDist::DIST_UNPACK_B16>((Reg::RegTensor<uint16_t>&)reversedShareExp0FP32, tmpAddr);1044+ Reg::LoadAlign<uint16_t, Reg::LoadDist::DIST_UNPACK_B16>((Reg::RegTensor<uint16_t>&)reversedShareExp0FP32,
1045+ tmpAddr);
1044 Reg::Cast<float, bfloat16_t, castTraitXdtypetoFp32Zero>(1046 Reg::Cast<float, bfloat16_t, castTraitXdtypetoFp32Zero>(
1045 reversedShareExp0FP32, (Reg::RegTensor<bfloat16_t>&)reversedShareExp0FP32, pregAll16);1047 reversedShareExp0FP32, (Reg::RegTensor<bfloat16_t>&)reversedShareExp0FP32, pregAll16);
1046 1048 
1047 for (uint16_t j = 0; j < blockCount; j++) {1049 for (uint16_t j = 0; j < blockCount; j++) {
1048- DataCopy<xDtype, Reg::LoadDist::DIST_NORM>(x0FP32, xAddr + j * dataLen8Align_);1050+ Reg::LoadAlign<xDtype, Reg::LoadDist::DIST_NORM>(x0FP32, xAddr + j * dataLen8Align_);
1049 if constexpr (IsSame<yDtype, fp4x2_e2m1_t>::value || IsSame<yDtype, fp4x2_e1m2_t>::value) {1051 if constexpr (IsSame<yDtype, fp4x2_e2m1_t>::value || IsSame<yDtype, fp4x2_e1m2_t>::value) {
1050 maskLen = dataLen64Align_ * DIGIT_TWO;1052 maskLen = dataLen64Align_ * DIGIT_TWO;
1051 pregAll8 = Reg::UpdateMask<uint8_t>(maskLen);1053 pregAll8 = Reg::UpdateMask<uint8_t>(maskLen);
@@ -1057,8 +1059,8 @@ DynamicMxQuantNotTailAxisOptimizeLargeTail<xDtype, yDtype, roundMode, calcMode>:
1057 1059 
1058 Reg::Cast<yDtype, bfloat16_t, castTraitBf16toFp4>(yZeroFP4, (Reg::RegTensor<bfloat16_t>&)x0BF16,1060 Reg::Cast<yDtype, bfloat16_t, castTraitBf16toFp4>(yZeroFP4, (Reg::RegTensor<bfloat16_t>&)x0BF16,
1059 pregAll16);1061 pregAll16);
1060- DataCopy<uint8_t, Reg::StoreDist::DIST_PACK4_B32>(yAddr + (j * dataLen64Align_ / DIGIT_TWO),1062+ Reg::StoreAlign<uint8_t, Reg::StoreDist::DIST_PACK4_B32>(yAddr + (j * dataLen64Align_ / DIGIT_TWO),
1061- (Reg::RegTensor<uint8_t>&)yZeroFP4, pregAll8);1063+ (Reg::RegTensor<uint8_t>&)yZeroFP4, pregAll8);
1062 } else {1064 } else {
1063 maskLen = dataLen32Align_;1065 maskLen = dataLen32Align_;
1064 pregAll8 = Reg::UpdateMask<uint8_t>(maskLen);1066 pregAll8 = Reg::UpdateMask<uint8_t>(maskLen);
@@ -1066,7 +1068,7 @@ DynamicMxQuantNotTailAxisOptimizeLargeTail<xDtype, yDtype, roundMode, calcMode>:
1066 Reg::Cast<yDtype, float, castTraitFp32toYdtype>((Reg::RegTensor<yDtype>&)yZeroU32, x0FP32, pregAll32);1068 Reg::Cast<yDtype, float, castTraitFp32toYdtype>((Reg::RegTensor<yDtype>&)yZeroU32, x0FP32, pregAll32);
1067 Reg::Pack<uint16_t, uint32_t, Reg::HighLowPart::LOWEST>(yZeroU16, yZeroU32);1069 Reg::Pack<uint16_t, uint32_t, Reg::HighLowPart::LOWEST>(yZeroU16, yZeroU32);
1068 Reg::Pack<uint8_t, uint16_t, Reg::HighLowPart::LOWEST>(yZeroU8, yZeroU16);1070 Reg::Pack<uint8_t, uint16_t, Reg::HighLowPart::LOWEST>(yZeroU8, yZeroU16);
1069- DataCopy<uint8_t, Reg::PostLiteral::POST_MODE_UPDATE, Reg::StoreDist::DIST_NORM>(1071+ Reg::StoreAlign<uint8_t, Reg::PostLiteral::POST_MODE_UPDATE, Reg::StoreDist::DIST_NORM>(
1070 yAddr, yZeroU8, dataLen32Align_, pregAll8);1072 yAddr, yZeroU8, dataLen32Align_, pregAll8);
1071 }1073 }
1072 }1074 }
Mquant/dynamic_mx_quant/op_kernel/arch35/dynamic_mx_quant_not_tail_axis_optimize_high_perf_small_tail.h+36-31
@@ -484,19 +484,19 @@ __aicore__ inline void DynamicMxQuantNotTailAxisOptimizeSmallTail<xDtype, yDtype
484 rhsRegs.mxScale);484 rhsRegs.mxScale);
485 Reg::Or(lhsRegs.mxScale, lhsRegs.mxScale, rhsRegs.mxScale, auxRegs.p0);485 Reg::Or(lhsRegs.mxScale, lhsRegs.mxScale, rhsRegs.mxScale, auxRegs.p0);
486 }486 }
487- Reg::DataCopy(mxScaleAddr, (Reg::RegTensor<uint8_t>&)lhsRegs.mxScale, auxRegs.p2);487+ Reg::StoreAlign(mxScaleAddr, (Reg::RegTensor<uint8_t>&)lhsRegs.mxScale, auxRegs.p2);
488 } else {488 } else {
489 // Compute MX scales and shared exponents489 // Compute MX scales and shared exponents
490 ComputeScalesAndSharedExp<canMaxLowBound>(lhsRegs, auxRegs);490 ComputeScalesAndSharedExp<canMaxLowBound>(lhsRegs, auxRegs);
491 Reg::Pack<uint8_t, uint16_t, Reg::HighLowPart::LOWEST>((Reg::RegTensor<uint8_t>&)lhsRegs.mxScale,491 Reg::Pack<uint8_t, uint16_t, Reg::HighLowPart::LOWEST>((Reg::RegTensor<uint8_t>&)lhsRegs.mxScale,
492 lhsRegs.mxScale);492 lhsRegs.mxScale);
493- Reg::DataCopy(mxScaleAddr, (Reg::RegTensor<uint8_t>&)lhsRegs.mxScale, auxRegs.p2);493+ Reg::StoreAlign(mxScaleAddr, (Reg::RegTensor<uint8_t>&)lhsRegs.mxScale, auxRegs.p2);
494 if constexpr (!canInterleave) {494 if constexpr (!canInterleave) {
495 ComputeScalesAndSharedExp<canMaxLowBound>(rhsRegs, auxRegs);495 ComputeScalesAndSharedExp<canMaxLowBound>(rhsRegs, auxRegs);
496 Reg::Pack<uint8_t, uint16_t, Reg::HighLowPart::LOWEST>((Reg::RegTensor<uint8_t>&)rhsRegs.mxScale,496 Reg::Pack<uint8_t, uint16_t, Reg::HighLowPart::LOWEST>((Reg::RegTensor<uint8_t>&)rhsRegs.mxScale,
497 rhsRegs.mxScale);497 rhsRegs.mxScale);
498- Reg::DataCopy(mxScaleAddr + alignedPostAxisSize_, (Reg::RegTensor<uint8_t>&)rhsRegs.mxScale,498+ Reg::StoreAlign(mxScaleAddr + alignedPostAxisSize_, (Reg::RegTensor<uint8_t>&)rhsRegs.mxScale,
499- auxRegs.p2);499+ auxRegs.p2);
500 }500 }
501 }501 }
502 502 
@@ -612,9 +612,9 @@ __aicore__ inline void DynamicMxQuantNotTailAxisOptimizeSmallTail<xDtype, yDtype
612 ComputeRegisters& lhsRegs, ComputeRegisters& rhsRegs, AuxRegisters& auxRegs, __ubuf__ DTYPE_X* lhsXAddr,612 ComputeRegisters& lhsRegs, ComputeRegisters& rhsRegs, AuxRegisters& auxRegs, __ubuf__ DTYPE_X* lhsXAddr,
613 __ubuf__ DTYPE_X* rhsXAddr)613 __ubuf__ DTYPE_X* rhsXAddr)
614{614{
615- DataCopy(lhsRegs.x, lhsXAddr);615+ Reg::LoadAlign(lhsRegs.x, lhsXAddr);
616 if constexpr (initRHS == 1) {616 if constexpr (initRHS == 1) {
617- DataCopy(rhsRegs.x, rhsXAddr);617+ Reg::LoadAlign(rhsRegs.x, rhsXAddr);
618 } else if constexpr (initRHS == 2) {618 } else if constexpr (initRHS == 2) {
619 Reg::Duplicate(rhsRegs.x, 0);619 Reg::Duplicate(rhsRegs.x, 0);
620 }620 }
@@ -655,8 +655,8 @@ __aicore__ inline void DynamicMxQuantNotTailAxisOptimizeSmallTail<xDtype, yDtype
655 Reg::MaskMergeMode::ZEROING, RoundMode::CAST_TRUNC};655 Reg::MaskMergeMode::ZEROING, RoundMode::CAST_TRUNC};
656 if constexpr (IsSame<DTYPE_X, half>::value) {656 if constexpr (IsSame<DTYPE_X, half>::value) {
657 Reg::And(reg.expU16, (Reg::RegTensor<uint16_t>&)reg.x, auxRegs.fp16MaxEleU16, auxRegs.p0);657 Reg::And(reg.expU16, (Reg::RegTensor<uint16_t>&)reg.x, auxRegs.fp16MaxEleU16, auxRegs.p0);
658- Reg::CompareScalar<uint16_t, CMPMODE::EQ>(auxRegs.infMask, (Reg::RegTensor<uint16_t>&)reg.expU16, FP16_INF,658+ Reg::Compares<uint16_t, CMPMODE::EQ>(auxRegs.infMask, (Reg::RegTensor<uint16_t>&)reg.expU16, FP16_INF,
659- auxRegs.p0);659+ auxRegs.p0);
660 Reg::Cast<bfloat16_t, DTYPE_X, castTraitHalf2Bf16>((Reg::RegTensor<bfloat16_t>&)reg.expU16,660 Reg::Cast<bfloat16_t, DTYPE_X, castTraitHalf2Bf16>((Reg::RegTensor<bfloat16_t>&)reg.expU16,
661 (Reg::RegTensor<DTYPE_X>&)reg.x, auxRegs.p0);661 (Reg::RegTensor<DTYPE_X>&)reg.x, auxRegs.p0);
662 Reg::Select<uint16_t>((Reg::RegTensor<uint16_t>&)reg.expU16, auxRegs.maxEleU16,662 Reg::Select<uint16_t>((Reg::RegTensor<uint16_t>&)reg.expU16, auxRegs.maxEleU16,
@@ -805,7 +805,7 @@ DynamicMxQuantNotTailAxisOptimizeSmallTail<xDtype, yDtype, roundMode, calcMode>:
805 Reg::RegTensor<uint32_t>& mxScaleAdd1U32, Reg::RegTensor<uint16_t>& mxScale)805 Reg::RegTensor<uint32_t>& mxScaleAdd1U32, Reg::RegTensor<uint16_t>& mxScale)
806{806{
807 if constexpr (calcMode == MODE_ONE) {807 if constexpr (calcMode == MODE_ONE) {
808- Reg::CompareScalar<uint32_t, CMPMODE::NE>(auxRegs.maxLowBoundMask, absMaxU32, FP32_NUMBER_ZERO, auxRegs.p1);808+ Reg::Compares<uint32_t, CMPMODE::NE>(auxRegs.maxLowBoundMask, absMaxU32, FP32_NUMBER_ZERO, auxRegs.p1);
809 if constexpr (IsSame<xDtype, float>::value) {809 if constexpr (IsSame<xDtype, float>::value) {
810 if constexpr (canMaxLowBound) {810 if constexpr (canMaxLowBound) {
811 Reg::MaskReg maskAll = Reg::CreateMask<float, Reg::MaskPattern::ALL>();811 Reg::MaskReg maskAll = Reg::CreateMask<float, Reg::MaskPattern::ALL>();
@@ -833,17 +833,17 @@ DynamicMxQuantNotTailAxisOptimizeSmallTail<xDtype, yDtype, roundMode, calcMode>:
833 Reg::ShiftRights(mxScaleU32, absMaxU32, FP32_SHR_NUM, auxRegs.p1);833 Reg::ShiftRights(mxScaleU32, absMaxU32, FP32_SHR_NUM, auxRegs.p1);
834 Reg::And(absMaxU32, absMaxU32, auxRegs.manForFP32, auxRegs.p1);834 Reg::And(absMaxU32, absMaxU32, auxRegs.manForFP32, auxRegs.p1);
835 835 
836- Reg::CompareScalar<uint32_t, CMPMODE::GT>(auxRegs.p4, mxScaleU32, FP32_NUMBER_ZERO, auxRegs.p1);836+ Reg::Compares<uint32_t, CMPMODE::GT>(auxRegs.p4, mxScaleU32, FP32_NUMBER_ZERO, auxRegs.p1);
837- Reg::CompareScalar<uint32_t, CMPMODE::LT>(auxRegs.p5, mxScaleU32, FP32_NUMBER_254, auxRegs.p1);837+ Reg::Compares<uint32_t, CMPMODE::LT>(auxRegs.p5, mxScaleU32, FP32_NUMBER_254, auxRegs.p1);
838- Reg::CompareScalar<uint32_t, CMPMODE::GT>(auxRegs.p6, absMaxU32, FP32_NUMBER_ZERO, auxRegs.p1);838+ Reg::Compares<uint32_t, CMPMODE::GT>(auxRegs.p6, absMaxU32, FP32_NUMBER_ZERO, auxRegs.p1);
839- Reg::MaskAnd(auxRegs.p4, auxRegs.p4, auxRegs.p5, auxRegs.p1);839+ Reg::And(auxRegs.p4, auxRegs.p4, auxRegs.p5, auxRegs.p1);
840- Reg::MaskAnd(auxRegs.p4, auxRegs.p4, auxRegs.p6, auxRegs.p1);840+ Reg::And(auxRegs.p4, auxRegs.p4, auxRegs.p6, auxRegs.p1);
841 841 
842 if constexpr (calcMode == MODE_ONE) {842 if constexpr (calcMode == MODE_ONE) {
843- Reg::CompareScalar<uint32_t, CMPMODE::EQ>(auxRegs.p5, mxScaleU32, FP32_NUMBER_ZERO, auxRegs.p1);843+ Reg::Compares<uint32_t, CMPMODE::EQ>(auxRegs.p5, mxScaleU32, FP32_NUMBER_ZERO, auxRegs.p1);
844- Reg::CompareScalar<uint32_t, CMPMODE::GT>(auxRegs.p6, absMaxU32, FP32_NUMBER_HALF, auxRegs.p1);844+ Reg::Compares<uint32_t, CMPMODE::GT>(auxRegs.p6, absMaxU32, FP32_NUMBER_HALF, auxRegs.p1);
845- Reg::MaskAnd(auxRegs.p5, auxRegs.p5, auxRegs.p6, auxRegs.p1);845+ Reg::And(auxRegs.p5, auxRegs.p5, auxRegs.p6, auxRegs.p1);
846- Reg::MaskXor(auxRegs.p4, auxRegs.p4, auxRegs.p5, auxRegs.p1);846+ Reg::Xor(auxRegs.p4, auxRegs.p4, auxRegs.p5, auxRegs.p1);
847 }847 }
848 848 
849 Reg::Adds(mxScaleAdd1U32, mxScaleU32, 1, auxRegs.p4);849 Reg::Adds(mxScaleAdd1U32, mxScaleU32, 1, auxRegs.p4);
@@ -999,7 +999,8 @@ DynamicMxQuantNotTailAxisOptimizeSmallTail<xDtype, yDtype, roundMode, calcMode>:
999 Reg::Pack<uint16_t, uint32_t, Reg::HighLowPart::LOWEST>(lhsRegs.yU16, (Reg::RegTensor<uint32_t>&)lhsRegs.yU16);999 Reg::Pack<uint16_t, uint32_t, Reg::HighLowPart::LOWEST>(lhsRegs.yU16, (Reg::RegTensor<uint32_t>&)lhsRegs.yU16);
1000 Reg::Cast<DTYPE_Y, bfloat16_t, castTraitDownZero>(lhsRegs.y, (Reg::RegTensor<bfloat16_t>&)lhsRegs.yU16,1000 Reg::Cast<DTYPE_Y, bfloat16_t, castTraitDownZero>(lhsRegs.y, (Reg::RegTensor<bfloat16_t>&)lhsRegs.yU16,
1001 auxRegs.p0);1001 auxRegs.p0);
1002- DataCopy<uint8_t, Reg::StoreDist::DIST_PACK4_B32>(lhsYAddr, (Reg::RegTensor<uint8_t>&)lhsRegs.y, auxRegs.p3);1002+ Reg::StoreAlign<uint8_t, Reg::StoreDist::DIST_PACK4_B32>(lhsYAddr, (Reg::RegTensor<uint8_t>&)lhsRegs.y,
1003+ auxRegs.p3);
1003 1004 
1004 // handle rhsregs1005 // handle rhsregs
1005 PreProcessFP32(rhsRegs.x0FP32, auxRegs);1006 PreProcessFP32(rhsRegs.x0FP32, auxRegs);
@@ -1008,19 +1009,22 @@ DynamicMxQuantNotTailAxisOptimizeSmallTail<xDtype, yDtype, roundMode, calcMode>:
1008 Reg::Pack<uint16_t, uint32_t, Reg::HighLowPart::LOWEST>(rhsRegs.yU16, (Reg::RegTensor<uint32_t>&)rhsRegs.yU16);1009 Reg::Pack<uint16_t, uint32_t, Reg::HighLowPart::LOWEST>(rhsRegs.yU16, (Reg::RegTensor<uint32_t>&)rhsRegs.yU16);
1009 Reg::Cast<DTYPE_Y, bfloat16_t, castTraitDownZero>(rhsRegs.y, (Reg::RegTensor<bfloat16_t>&)rhsRegs.yU16,1010 Reg::Cast<DTYPE_Y, bfloat16_t, castTraitDownZero>(rhsRegs.y, (Reg::RegTensor<bfloat16_t>&)rhsRegs.yU16,
1010 auxRegs.p0);1011 auxRegs.p0);
1011- DataCopy<uint8_t, Reg::StoreDist::DIST_PACK4_B32>(rhsYAddr, (Reg::RegTensor<uint8_t>&)rhsRegs.y, auxRegs.p3);1012+ Reg::StoreAlign<uint8_t, Reg::StoreDist::DIST_PACK4_B32>(rhsYAddr, (Reg::RegTensor<uint8_t>&)rhsRegs.y,
1013+ auxRegs.p3);
1012 } else {1014 } else {
1013 static constexpr Reg::CastTrait castTraitDownZero = {Reg::RegLayout::ZERO, Reg::SatMode::SAT,1015 static constexpr Reg::CastTrait castTraitDownZero = {Reg::RegLayout::ZERO, Reg::SatMode::SAT,
1014 Reg::MaskMergeMode::ZEROING, RoundMode::CAST_RINT};1016 Reg::MaskMergeMode::ZEROING, RoundMode::CAST_RINT};
1015 Reg::Cast<DTYPE_Y, float, castTraitDownZero>(lhsRegs.y, lhsRegs.x0FP32, auxRegs.p1);1017 Reg::Cast<DTYPE_Y, float, castTraitDownZero>(lhsRegs.y, lhsRegs.x0FP32, auxRegs.p1);
1016 Reg::Pack<uint16_t, uint32_t, Reg::HighLowPart::LOWEST>((Reg::RegTensor<uint16_t>&)lhsRegs.y,1018 Reg::Pack<uint16_t, uint32_t, Reg::HighLowPart::LOWEST>((Reg::RegTensor<uint16_t>&)lhsRegs.y,
1017 (Reg::RegTensor<uint32_t>&)lhsRegs.y);1019 (Reg::RegTensor<uint32_t>&)lhsRegs.y);
1018- DataCopy<uint8_t, Reg::StoreDist::DIST_PACK_B16>(lhsYAddr, (Reg::RegTensor<uint8_t>&)lhsRegs.y, auxRegs.p3);1020+ Reg::StoreAlign<uint8_t, Reg::StoreDist::DIST_PACK_B16>(lhsYAddr, (Reg::RegTensor<uint8_t>&)lhsRegs.y,
1021+ auxRegs.p3);
1019 1022 
1020 Reg::Cast<DTYPE_Y, float, castTraitDownZero>(rhsRegs.y, rhsRegs.x0FP32, auxRegs.p1);1023 Reg::Cast<DTYPE_Y, float, castTraitDownZero>(rhsRegs.y, rhsRegs.x0FP32, auxRegs.p1);
1021 Reg::Pack<uint16_t, uint32_t, Reg::HighLowPart::LOWEST>((Reg::RegTensor<uint16_t>&)rhsRegs.y,1024 Reg::Pack<uint16_t, uint32_t, Reg::HighLowPart::LOWEST>((Reg::RegTensor<uint16_t>&)rhsRegs.y,
1022 (Reg::RegTensor<uint32_t>&)rhsRegs.y);1025 (Reg::RegTensor<uint32_t>&)rhsRegs.y);
1023- DataCopy<uint8_t, Reg::StoreDist::DIST_PACK_B16>(rhsYAddr, (Reg::RegTensor<uint8_t>&)rhsRegs.y, auxRegs.p3);1026+ Reg::StoreAlign<uint8_t, Reg::StoreDist::DIST_PACK_B16>(rhsYAddr, (Reg::RegTensor<uint8_t>&)rhsRegs.y,
1027+ auxRegs.p3);
1024 }1028 }
1025}1029}
1026 1030 
@@ -1091,11 +1095,12 @@ DynamicMxQuantNotTailAxisOptimizeSmallTail<xDtype, yDtype, roundMode, calcMode>:
1091 }1095 }
1092 Reg::DeInterleave(lhsRegs.x, rhsRegs.x, lhsRegs.x, rhsRegs.x);1096 Reg::DeInterleave(lhsRegs.x, rhsRegs.x, lhsRegs.x, rhsRegs.x);
1093 Reg::Cast<DTYPE_Y, bfloat16_t, castTraitDownZero>(lhsRegs.y, (Reg::RegTensor<bfloat16_t>&)lhsRegs.x, auxRegs.p0);1097 Reg::Cast<DTYPE_Y, bfloat16_t, castTraitDownZero>(lhsRegs.y, (Reg::RegTensor<bfloat16_t>&)lhsRegs.x, auxRegs.p0);
1094- DataCopy<uint8_t, Reg::StoreDist::DIST_PACK4_B32>(lhsYAddr, (Reg::RegTensor<uint8_t>&)lhsRegs.y, auxRegs.p3);1098+ Reg::StoreAlign<uint8_t, Reg::StoreDist::DIST_PACK4_B32>(lhsYAddr, (Reg::RegTensor<uint8_t>&)lhsRegs.y, auxRegs.p3);
1095 if constexpr (withRHS) {1099 if constexpr (withRHS) {
1096 Reg::Cast<DTYPE_Y, bfloat16_t, castTraitDownZero>(rhsRegs.y, (Reg::RegTensor<bfloat16_t>&)rhsRegs.x,1100 Reg::Cast<DTYPE_Y, bfloat16_t, castTraitDownZero>(rhsRegs.y, (Reg::RegTensor<bfloat16_t>&)rhsRegs.x,
1097 auxRegs.p0);1101 auxRegs.p0);
1098- DataCopy<uint8_t, Reg::StoreDist::DIST_PACK4_B32>(rhsYAddr, (Reg::RegTensor<uint8_t>&)rhsRegs.y, auxRegs.p3);1102+ Reg::StoreAlign<uint8_t, Reg::StoreDist::DIST_PACK4_B32>(rhsYAddr, (Reg::RegTensor<uint8_t>&)rhsRegs.y,
1103+ auxRegs.p3);
1099 }1104 }
1100}1105}
1101 1106 
@@ -1133,7 +1138,7 @@ DynamicMxQuantNotTailAxisOptimizeSmallTail<xDtype, yDtype, roundMode, calcMode>:
1133 }1138 }
1134 1139 
1135 Reg::Cast<DTYPE_Y, bfloat16_t, castTraitDownZero>(regs.y, (Reg::RegTensor<bfloat16_t>&)regs.yU16, auxRegs.p0);1140 Reg::Cast<DTYPE_Y, bfloat16_t, castTraitDownZero>(regs.y, (Reg::RegTensor<bfloat16_t>&)regs.yU16, auxRegs.p0);
1136- DataCopy<uint8_t, Reg::StoreDist::DIST_PACK4_B32>(yAddr, (Reg::RegTensor<uint8_t>&)regs.y, auxRegs.p3);1141+ Reg::StoreAlign<uint8_t, Reg::StoreDist::DIST_PACK4_B32>(yAddr, (Reg::RegTensor<uint8_t>&)regs.y, auxRegs.p3);
1137}1142}
1138 1143 
1139template <typename xDtype, typename yDtype, RoundMode roundMode, const int64_t calcMode>1144template <typename xDtype, typename yDtype, RoundMode roundMode, const int64_t calcMode>
@@ -1156,7 +1161,7 @@ DynamicMxQuantNotTailAxisOptimizeSmallTail<xDtype, yDtype, roundMode, calcMode>:
1156 1161 
1157 Reg::Or<uint16_t>((Reg::RegTensor<uint16_t>&)regs.y, regs.yU16, (Reg::RegTensor<uint16_t>&)regs.y, auxRegs.p0);1162 Reg::Or<uint16_t>((Reg::RegTensor<uint16_t>&)regs.y, regs.yU16, (Reg::RegTensor<uint16_t>&)regs.y, auxRegs.p0);
1158 }1163 }
1159- DataCopy<uint8_t, Reg::StoreDist::DIST_PACK_B16>(yAddr, (Reg::RegTensor<uint8_t>&)regs.y, auxRegs.p3);1164+ Reg::StoreAlign<uint8_t, Reg::StoreDist::DIST_PACK_B16>(yAddr, (Reg::RegTensor<uint8_t>&)regs.y, auxRegs.p3);
1160}1165}
1161 1166 
1162template <typename xDtype, typename yDtype, RoundMode roundMode, const int64_t calcMode>1167template <typename xDtype, typename yDtype, RoundMode roundMode, const int64_t calcMode>
@@ -1177,7 +1182,7 @@ __aicore__ inline void DynamicMxQuantNotTailAxisOptimizeSmallTail<xDtype, yDtype
1177 Reg::Compare<int32_t, CMPMODE::EQ>(negInfMask, (Reg::RegTensor<int32_t>&)in, negZero, pregAll32);1182 Reg::Compare<int32_t, CMPMODE::EQ>(negInfMask, (Reg::RegTensor<int32_t>&)in, negZero, pregAll32);
1178 if constexpr (IsSame<DTYPE_Y, fp4x2_e1m2_t>::value) {1183 if constexpr (IsSame<DTYPE_Y, fp4x2_e1m2_t>::value) {
1179 Reg::Muls(in, in, FP4_SCALE_FACTOR, pregAll32);1184 Reg::Muls(in, in, FP4_SCALE_FACTOR, pregAll32);
1180- Reg::CompareScalar<float, CMPMODE::LT>(specialMask, in, 0, pregAll32);1185+ Reg::Compares<float, CMPMODE::LT>(specialMask, in, 0, pregAll32);
1181 Reg::Truncate<float, roundMode>(in, in, pregAll32);1186 Reg::Truncate<float, roundMode>(in, in, pregAll32);
1182 Reg::Muls(in, in, FP4_INV_SCALE_FACTOR, pregAll32);1187 Reg::Muls(in, in, FP4_INV_SCALE_FACTOR, pregAll32);
1183 } else { // fp4x2_e2m11188 } else { // fp4x2_e2m1
@@ -1194,13 +1199,13 @@ __aicore__ inline void DynamicMxQuantNotTailAxisOptimizeSmallTail<xDtype, yDtype
1194 Reg::Mul(in, in, (Reg::RegTensor<float>&)exp1FP32, pregAll32);1199 Reg::Mul(in, in, (Reg::RegTensor<float>&)exp1FP32, pregAll32);
1195 Reg::Adds(exp0FP32, exp0FP32, FP32_BIAS_VALUE, pregAll32);1200 Reg::Adds(exp0FP32, exp0FP32, FP32_BIAS_VALUE, pregAll32);
1196 Reg::ShiftLefts(exp0FP32, exp0FP32, FP32_SHR_NUM, pregAll32);1201 Reg::ShiftLefts(exp0FP32, exp0FP32, FP32_SHR_NUM, pregAll32);
1197- Reg::CompareScalar<float, CMPMODE::LT>(specialMask, in, 0, pregAll32);1202+ Reg::Compares<float, CMPMODE::LT>(specialMask, in, 0, pregAll32);
1198 Reg::Truncate<float, roundMode>(in, in, pregAll32);1203 Reg::Truncate<float, roundMode>(in, in, pregAll32);
1199 Reg::Mul(in, in, (Reg::RegTensor<float>&)exp0FP32, pregAll32);1204 Reg::Mul(in, in, (Reg::RegTensor<float>&)exp0FP32, pregAll32);
1200 }1205 }
1201- Reg::CompareScalar<float, CMPMODE::EQ>(zeroMask, in, 0, pregAll32);1206+ Reg::Compares<float, CMPMODE::EQ>(zeroMask, in, 0, pregAll32);
1202- Reg::MaskAnd(zeroMask, specialMask, zeroMask, pregAll32);1207+ Reg::And(zeroMask, specialMask, zeroMask, pregAll32);
1203- Reg::MaskOr(zeroMask, negInfMask, zeroMask, pregAll32);1208+ Reg::Or(zeroMask, negInfMask, zeroMask, pregAll32);
1204 Reg::Select<int32_t>((Reg::RegTensor<int32_t>&)in, negZero, (Reg::RegTensor<int32_t>&)in, zeroMask);1209 Reg::Select<int32_t>((Reg::RegTensor<int32_t>&)in, negZero, (Reg::RegTensor<int32_t>&)in, zeroMask);
1205}1210}
1206 1211 
Mquant/dynamic_mx_quant/op_kernel/arch35/dynamic_mx_quant_post.h+24-24
@@ -40,8 +40,8 @@ public:
40 int64_t axisSize2, int64_t axisSize3);40 int64_t axisSize2, int64_t axisSize3);
41 __aicore__ inline void ComputeInterleave(uint32_t elementNum);41 __aicore__ inline void ComputeInterleave(uint32_t elementNum);
42 __aicore__ inline void CopyOut(int64_t scaleGmOffset, int64_t blockLen, int64_t blockCount);42 __aicore__ inline void CopyOut(int64_t scaleGmOffset, int64_t blockLen, int64_t blockCount);
43- __aicore__ inline void ComputeInterleaveVF(__local_mem__ uint8_t* LocalAddr1, __local_mem__ uint8_t* LocalAddr2,43+ __aicore__ inline void ComputeInterleaveVF(__ubuf__ uint8_t* LocalAddr1, __ubuf__ uint8_t* LocalAddr2,
44- __local_mem__ uint8_t* ScaleAddr, uint32_t elementNum);44+ __ubuf__ uint8_t* ScaleAddr, uint32_t elementNum);
45 45 
46private:46private:
47 __aicore__ inline void TailAxisCompute();47 __aicore__ inline void TailAxisCompute();
@@ -375,13 +375,13 @@ __aicore__ inline void DynamicMxQuantPost::NonTailAxisCompute()
375__aicore__ inline void DynamicMxQuantPost::CopyIn(int64_t wsGmOffset1, int64_t wsGmOffset2, int64_t blockLen,375__aicore__ inline void DynamicMxQuantPost::CopyIn(int64_t wsGmOffset1, int64_t wsGmOffset2, int64_t blockLen,
376 int64_t blockCount)376 int64_t blockCount)
377{377{
378- MultiCopyLoopInfo<COPY_WITH_DIM_2> loopInfo = {{1, static_cast<uint32_t>(COPY_WITH_DIM_2 * blockLen)},378+ NdDmaLoopInfo<COPY_WITH_DIM_2> loopInfo = {{1, static_cast<uint32_t>(COPY_WITH_DIM_2 * blockLen)},
379- {1, static_cast<uint32_t>(blockLen)},379+ {1, static_cast<uint32_t>(blockLen)},
380- {static_cast<uint32_t>(blockLen), static_cast<uint32_t>(blockCount)},380+ {static_cast<uint32_t>(blockLen), static_cast<uint32_t>(blockCount)},
381- {0, 0},381+ {0, 0},
382- {0, 0}};382+ {0, 0}};
383- MultiCopyParams<uint8_t, COPY_WITH_DIM_2> params = {loopInfo, 0};383+ NdDmaParams<uint8_t, COPY_WITH_DIM_2> params = {loopInfo, 0};
384- static constexpr MultiCopyConfig config = {false};384+ static constexpr NdDmaConfig config = {false};
385 auto localBuf1 = inQueue1_.AllocTensor<uint8_t>();385 auto localBuf1 = inQueue1_.AllocTensor<uint8_t>();
386 DataCopy<uint8_t, COPY_WITH_DIM_2, config>(localBuf1, workspaceGm_[wsGmOffset1], params);386 DataCopy<uint8_t, COPY_WITH_DIM_2, config>(localBuf1, workspaceGm_[wsGmOffset1], params);
387 inQueue1_.EnQue(localBuf1);387 inQueue1_.EnQue(localBuf1);
@@ -393,26 +393,26 @@ __aicore__ inline void DynamicMxQuantPost::CopyIn(int64_t wsGmOffset1, int64_t w
393__aicore__ inline void DynamicMxQuantPost::CopyInPaddingZero(int64_t wsGmOffset1, int64_t wsGmOffset2,393__aicore__ inline void DynamicMxQuantPost::CopyInPaddingZero(int64_t wsGmOffset1, int64_t wsGmOffset2,
394 int64_t axisSize1, int64_t axisSize2, int64_t axisSize3)394 int64_t axisSize1, int64_t axisSize2, int64_t axisSize3)
395{395{
396- MultiCopyLoopInfo<COPY_WITH_DIM_3> loopInfo1 = {396+ NdDmaLoopInfo<COPY_WITH_DIM_3> loopInfo1 = {
397 {1, static_cast<uint32_t>(DIGIT_TWO * axisSize1),397 {1, static_cast<uint32_t>(DIGIT_TWO * axisSize1),
398 static_cast<uint32_t>((DIGIT_TWO * axisSize2 - 1) * axisSize1)},398 static_cast<uint32_t>((DIGIT_TWO * axisSize2 - 1) * axisSize1)},
399 {1, static_cast<uint32_t>(axisSize1), static_cast<uint32_t>(axisSize1 * axisSize2)},399 {1, static_cast<uint32_t>(axisSize1), static_cast<uint32_t>(axisSize1 * axisSize2)},
400 {static_cast<uint32_t>(axisSize1), static_cast<uint32_t>(axisSize2), static_cast<uint32_t>(axisSize3)},400 {static_cast<uint32_t>(axisSize1), static_cast<uint32_t>(axisSize2), static_cast<uint32_t>(axisSize3)},
401 {0, 0, 0},401 {0, 0, 0},
402 {0, 0, 0}};402 {0, 0, 0}};
403- MultiCopyLoopInfo<COPY_WITH_DIM_3> loopInfo2 = {403+ NdDmaLoopInfo<COPY_WITH_DIM_3> loopInfo2 = {
404 {1, static_cast<uint32_t>(DIGIT_TWO * axisSize1),404 {1, static_cast<uint32_t>(DIGIT_TWO * axisSize1),
405 static_cast<uint32_t>((DIGIT_TWO * axisSize2 - 1) * axisSize1)},405 static_cast<uint32_t>((DIGIT_TWO * axisSize2 - 1) * axisSize1)},
406 {1, static_cast<uint32_t>(axisSize1), static_cast<uint32_t>(axisSize1 * axisSize2)},406 {1, static_cast<uint32_t>(axisSize1), static_cast<uint32_t>(axisSize1 * axisSize2)},
407 {static_cast<uint32_t>(axisSize1), static_cast<uint32_t>(axisSize2 - 1), static_cast<uint32_t>(axisSize3)},407 {static_cast<uint32_t>(axisSize1), static_cast<uint32_t>(axisSize2 - 1), static_cast<uint32_t>(axisSize3)},
408 {0, 0, 0},408 {0, 0, 0},
409 {0, 1, 0}};409 {0, 1, 0}};
410- static constexpr MultiCopyConfig config = {false};410+ static constexpr NdDmaConfig config = {false};
411- MultiCopyParams<uint8_t, COPY_WITH_DIM_3> params1 = {loopInfo1, 0};411+ NdDmaParams<uint8_t, COPY_WITH_DIM_3> params1 = {loopInfo1, 0};
412 auto localBuf1 = inQueue1_.AllocTensor<uint8_t>();412 auto localBuf1 = inQueue1_.AllocTensor<uint8_t>();
413 DataCopy<uint8_t, COPY_WITH_DIM_3, config>(localBuf1, workspaceGm_[wsGmOffset1], params1);413 DataCopy<uint8_t, COPY_WITH_DIM_3, config>(localBuf1, workspaceGm_[wsGmOffset1], params1);
414 inQueue1_.EnQue(localBuf1);414 inQueue1_.EnQue(localBuf1);
415- MultiCopyParams<uint8_t, COPY_WITH_DIM_3> params2 = {loopInfo2, 0};415+ NdDmaParams<uint8_t, COPY_WITH_DIM_3> params2 = {loopInfo2, 0};
416 auto localBuf2 = inQueue2_.AllocTensor<uint8_t>();416 auto localBuf2 = inQueue2_.AllocTensor<uint8_t>();
417 if (axisSize2 > 1) {417 if (axisSize2 > 1) {
418 DataCopy<uint8_t, COPY_WITH_DIM_3, config>(localBuf2, workspaceGm_[wsGmOffset2], params2);418 DataCopy<uint8_t, COPY_WITH_DIM_3, config>(localBuf2, workspaceGm_[wsGmOffset2], params2);
@@ -427,9 +427,9 @@ __aicore__ inline void DynamicMxQuantPost::ComputeInterleave(uint32_t elementNum
427 auto inBuf1 = inQueue1_.DeQue<uint8_t>();427 auto inBuf1 = inQueue1_.DeQue<uint8_t>();
428 auto inBuf2 = inQueue2_.DeQue<uint8_t>();428 auto inBuf2 = inQueue2_.DeQue<uint8_t>();
429 auto outBuf = scaleQueue.DeQue<uint8_t>();429 auto outBuf = scaleQueue.DeQue<uint8_t>();
430- __local_mem__ uint8_t* inAddr1 = (__local_mem__ uint8_t*)inBuf1.GetPhyAddr();430+ __ubuf__ uint8_t* inAddr1 = (__ubuf__ uint8_t*)inBuf1.GetPhyAddr();
431- __local_mem__ uint8_t* inAddr2 = (__local_mem__ uint8_t*)inBuf2.GetPhyAddr();431+ __ubuf__ uint8_t* inAddr2 = (__ubuf__ uint8_t*)inBuf2.GetPhyAddr();
432- __local_mem__ uint8_t* outAddr = (__local_mem__ uint8_t*)outBuf.GetPhyAddr();432+ __ubuf__ uint8_t* outAddr = (__ubuf__ uint8_t*)outBuf.GetPhyAddr();
433 433 
434 ComputeInterleaveVF(inAddr1, inAddr2, outAddr, elementNum);434 ComputeInterleaveVF(inAddr1, inAddr2, outAddr, elementNum);
435 435 
@@ -452,9 +452,9 @@ __aicore__ inline void DynamicMxQuantPost::CopyOut(int64_t scaleGmOffset, int64_
452 scaleQueue.FreeTensor(scale);452 scaleQueue.FreeTensor(scale);
453}453}
454 454 
455-__aicore__ inline void DynamicMxQuantPost::ComputeInterleaveVF(__local_mem__ uint8_t* localAddr1,455+__aicore__ inline void DynamicMxQuantPost::ComputeInterleaveVF(__ubuf__ uint8_t* localAddr1,
456- __local_mem__ uint8_t* localAddr2,456+ __ubuf__ uint8_t* localAddr2,
457- __local_mem__ uint8_t* scaleAddr, uint32_t elementNum)457+ __ubuf__ uint8_t* scaleAddr, uint32_t elementNum)
458{458{
459 uint32_t dtypeSize = sizeof(uint8_t);459 uint32_t dtypeSize = sizeof(uint8_t);
460 uint32_t VL = AscendC::VECTOR_REG_WIDTH / dtypeSize;460 uint32_t VL = AscendC::VECTOR_REG_WIDTH / dtypeSize;
@@ -470,11 +470,11 @@ __aicore__ inline void DynamicMxQuantPost::ComputeInterleaveVF(__local_mem__ uin
470 Reg::MaskReg mask = Reg::CreateMask<uint8_t, Reg::MaskPattern::ALL>();470 Reg::MaskReg mask = Reg::CreateMask<uint8_t, Reg::MaskPattern::ALL>();
471 471 
472 for (uint16_t i = 0; i < vfLoopNum; i++) {472 for (uint16_t i = 0; i < vfLoopNum; i++) {
473- Reg::DataCopy(vreg0, localAddr1 + i * VL);473+ Reg::LoadAlign(vreg0, localAddr1 + i * VL);
474- Reg::DataCopy(vreg1, localAddr2 + i * VL);474+ Reg::LoadAlign(vreg1, localAddr2 + i * VL);
475 Reg::Interleave(vreg3, vreg4, vreg0, vreg1);475 Reg::Interleave(vreg3, vreg4, vreg0, vreg1);
476- Reg::DataCopy(scaleAddr + DIGIT_TWO * i * VL, vreg3, mask);476+ Reg::StoreAlign(scaleAddr + DIGIT_TWO * i * VL, vreg3, mask);
477- Reg::DataCopy(scaleAddr + (DIGIT_TWO * i + 1) * VL, vreg4, mask);477+ Reg::StoreAlign(scaleAddr + (DIGIT_TWO * i + 1) * VL, vreg4, mask);
478 }478 }
479 }479 }
480}480}
Mquant/dynamic_mx_quant/op_kernel/arch35/dynamic_mx_quant_tail_axis.h+31-31
@@ -434,7 +434,7 @@ __aicore__ inline void DynamicMxQuantTailAxis<T, U, SCALE_ALG>::ComputeMaxExpOcp
434 Reg::MaskReg Mask = Reg::CreateMask<uint16_t, Reg::MaskPattern::ALL>();434 Reg::MaskReg Mask = Reg::CreateMask<uint16_t, Reg::MaskPattern::ALL>();
435 Reg::MaskReg invalidDataMask0;435 Reg::MaskReg invalidDataMask0;
436 Reg::MaskReg invalidDataMask1;436 Reg::MaskReg invalidDataMask1;
437- Reg::UnalignReg ureg;437+ Reg::UnalignRegForStore ureg;
438 438 
439 static constexpr Reg::CastTrait castTraitHalf2Bf16 = {Reg::RegLayout::UNKNOWN, Reg::SatMode::UNKNOWN,439 static constexpr Reg::CastTrait castTraitHalf2Bf16 = {Reg::RegLayout::UNKNOWN, Reg::SatMode::UNKNOWN,
440 Reg::MaskMergeMode::ZEROING, RoundMode::CAST_TRUNC};440 Reg::MaskMergeMode::ZEROING, RoundMode::CAST_TRUNC};
@@ -455,7 +455,7 @@ __aicore__ inline void DynamicMxQuantTailAxis<T, U, SCALE_ALG>::ComputeMaxExpOcp
455 Reg::Select<uint16_t>(xExpExtract1, xExpExtract1, expMaskBF16, invalidDataMask1);455 Reg::Select<uint16_t>(xExpExtract1, xExpExtract1, expMaskBF16, invalidDataMask1);
456 456 
457 Reg::Max(xMaxExp, xExpExtract0, xExpExtract1, Mask);457 Reg::Max(xMaxExp, xExpExtract0, xExpExtract1, Mask);
458- Reg::ReduceMaxWithDataBlock(xMaxExp, xMaxExp, Mask);458+ Reg::ReduceDataBlock<ReduceType::MAX>(xMaxExp, xMaxExp, Mask);
459 459 
460 Reg::StoreUnAlign<uint16_t, Reg::PostLiteral::POST_MODE_UPDATE>(maxExpAddr, xMaxExp, ureg,460 Reg::StoreUnAlign<uint16_t, Reg::PostLiteral::POST_MODE_UPDATE>(maxExpAddr, xMaxExp, ureg,
461 ELEMENT_AFTER_REDUCE_);461 ELEMENT_AFTER_REDUCE_);
@@ -482,7 +482,7 @@ __aicore__ inline void DynamicMxQuantTailAxis<T, U, SCALE_ALG>::ComputeMaxExpOcp
482 Reg::Duplicate(expMaskBF16, BF16_MAX_EXP);482 Reg::Duplicate(expMaskBF16, BF16_MAX_EXP);
483 483 
484 Reg::MaskReg Mask = Reg::CreateMask<uint16_t, Reg::MaskPattern::ALL>();484 Reg::MaskReg Mask = Reg::CreateMask<uint16_t, Reg::MaskPattern::ALL>();
485- Reg::UnalignReg ureg;485+ Reg::UnalignRegForStore ureg;
486 486 
487 for (uint16_t i = 0; i < loopNum2VF; i++) {487 for (uint16_t i = 0; i < loopNum2VF; i++) {
488 Reg::LoadAlign<bfloat16_t, Reg::PostLiteral::POST_MODE_UPDATE, Reg::LoadDist::DIST_DINTLV_B16>(488 Reg::LoadAlign<bfloat16_t, Reg::PostLiteral::POST_MODE_UPDATE, Reg::LoadDist::DIST_DINTLV_B16>(
@@ -490,7 +490,7 @@ __aicore__ inline void DynamicMxQuantTailAxis<T, U, SCALE_ALG>::ComputeMaxExpOcp
490 Reg::And(xExpExtract0, (Reg::RegTensor<uint16_t>&)xExp0, expMaskBF16, Mask);490 Reg::And(xExpExtract0, (Reg::RegTensor<uint16_t>&)xExp0, expMaskBF16, Mask);
491 Reg::And(xExpExtract1, (Reg::RegTensor<uint16_t>&)xExp1, expMaskBF16, Mask);491 Reg::And(xExpExtract1, (Reg::RegTensor<uint16_t>&)xExp1, expMaskBF16, Mask);
492 Reg::Max(xMaxExp, xExpExtract0, xExpExtract1, Mask);492 Reg::Max(xMaxExp, xExpExtract0, xExpExtract1, Mask);
493- Reg::ReduceMaxWithDataBlock(xMaxExp, xMaxExp, Mask);493+ Reg::ReduceDataBlock<ReduceType::MAX>(xMaxExp, xMaxExp, Mask);
494 Reg::StoreUnAlign<uint16_t, Reg::PostLiteral::POST_MODE_UPDATE>(maxExpAddr, xMaxExp, ureg,494 Reg::StoreUnAlign<uint16_t, Reg::PostLiteral::POST_MODE_UPDATE>(maxExpAddr, xMaxExp, ureg,
495 ELEMENT_AFTER_REDUCE_);495 ELEMENT_AFTER_REDUCE_);
496 }496 }
@@ -520,7 +520,7 @@ __aicore__ inline void DynamicMxQuantTailAxis<T, U, SCALE_ALG>::ComputeMaxExpOcp
520 520 
521 Reg::MaskReg MaskB32 = Reg::CreateMask<uint32_t, Reg::MaskPattern::ALL>();521 Reg::MaskReg MaskB32 = Reg::CreateMask<uint32_t, Reg::MaskPattern::ALL>();
522 Reg::MaskReg Mask = Reg::CreateMask<uint16_t, Reg::MaskPattern::ALL>();522 Reg::MaskReg Mask = Reg::CreateMask<uint16_t, Reg::MaskPattern::ALL>();
523- Reg::UnalignReg ureg;523+ Reg::UnalignRegForStore ureg;
524 static constexpr Reg::CastTrait castTraitFp32toBF16 = {Reg::RegLayout::ZERO, Reg::SatMode::NO_SAT,524 static constexpr Reg::CastTrait castTraitFp32toBF16 = {Reg::RegLayout::ZERO, Reg::SatMode::NO_SAT,
525 Reg::MaskMergeMode::ZEROING, RoundMode::CAST_TRUNC};525 Reg::MaskMergeMode::ZEROING, RoundMode::CAST_TRUNC};
526 for (uint16_t i = 0; i < loopNum2VF; i++) {526 for (uint16_t i = 0; i < loopNum2VF; i++) {
@@ -543,7 +543,7 @@ __aicore__ inline void DynamicMxQuantTailAxis<T, U, SCALE_ALG>::ComputeMaxExpOcp
543 Reg::Max(x0MaxExpB32, (Reg::RegTensor<uint32_t>&)x0ZeroFP32, (Reg::RegTensor<uint32_t>&)x0OneFP32, MaskB32);543 Reg::Max(x0MaxExpB32, (Reg::RegTensor<uint32_t>&)x0ZeroFP32, (Reg::RegTensor<uint32_t>&)x0OneFP32, MaskB32);
544 Reg::Max(x1MaxExpB32, (Reg::RegTensor<uint32_t>&)x1ZeroFP32, (Reg::RegTensor<uint32_t>&)x1OneFP32, MaskB32);544 Reg::Max(x1MaxExpB32, (Reg::RegTensor<uint32_t>&)x1ZeroFP32, (Reg::RegTensor<uint32_t>&)x1OneFP32, MaskB32);
545 Reg::Max(xMaxExpB32, x0MaxExpB32, x1MaxExpB32, MaskB32);545 Reg::Max(xMaxExpB32, x0MaxExpB32, x1MaxExpB32, MaskB32);
546- Reg::ReduceMaxWithDataBlock(xMaxExpB32, xMaxExpB32, MaskB32);546+ Reg::ReduceDataBlock<ReduceType::MAX>(xMaxExpB32, xMaxExpB32, MaskB32);
547 Reg::ShiftRights(xMaxExpB32, xMaxExpB32, FP32_PACK_SHR_NUM, MaskB32);547 Reg::ShiftRights(xMaxExpB32, xMaxExpB32, FP32_PACK_SHR_NUM, MaskB32);
548 Reg::Pack<uint16_t, uint32_t, Reg::HighLowPart::LOWEST>(xMaxExpB16, xMaxExpB32);548 Reg::Pack<uint16_t, uint32_t, Reg::HighLowPart::LOWEST>(xMaxExpB16, xMaxExpB32);
549 Reg::StoreUnAlign<uint16_t, Reg::PostLiteral::POST_MODE_UPDATE>(maxExpAddr, xMaxExpB16, ureg,549 Reg::StoreUnAlign<uint16_t, Reg::PostLiteral::POST_MODE_UPDATE>(maxExpAddr, xMaxExpB16, ureg,
@@ -580,7 +580,7 @@ __aicore__ inline void DynamicMxQuantTailAxis<T, U, SCALE_ALG>::ComputeMaxExpDyn
580 Reg::MaskReg Mask = Reg::CreateMask<uint16_t, Reg::MaskPattern::ALL>();580 Reg::MaskReg Mask = Reg::CreateMask<uint16_t, Reg::MaskPattern::ALL>();
581 Reg::MaskReg invalidDataMask0;581 Reg::MaskReg invalidDataMask0;
582 Reg::MaskReg invalidDataMask1;582 Reg::MaskReg invalidDataMask1;
583- Reg::UnalignReg ureg;583+ Reg::UnalignRegForStore ureg;
584 584 
585 static constexpr Reg::CastTrait castTraitHalf2Bf16 = {Reg::RegLayout::UNKNOWN, Reg::SatMode::UNKNOWN,585 static constexpr Reg::CastTrait castTraitHalf2Bf16 = {Reg::RegLayout::UNKNOWN, Reg::SatMode::UNKNOWN,
586 Reg::MaskMergeMode::ZEROING, RoundMode::CAST_RINT};586 Reg::MaskMergeMode::ZEROING, RoundMode::CAST_RINT};
@@ -600,7 +600,7 @@ __aicore__ inline void DynamicMxQuantTailAxis<T, U, SCALE_ALG>::ComputeMaxExpDyn
600 Reg::Select<uint16_t>(xExpExtract1, xExpExtract1, expMaskBF16, invalidDataMask1);600 Reg::Select<uint16_t>(xExpExtract1, xExpExtract1, expMaskBF16, invalidDataMask1);
601 601 
602 Reg::Max(xMaxExp, xExpExtract0, xExpExtract1, Mask);602 Reg::Max(xMaxExp, xExpExtract0, xExpExtract1, Mask);
603- Reg::ReduceMaxWithDataBlock(xMaxExp, xMaxExp, Mask);603+ Reg::ReduceDataBlock<ReduceType::MAX>(xMaxExp, xMaxExp, Mask);
604 604 
605 Reg::StoreUnAlign<uint16_t, Reg::PostLiteral::POST_MODE_UPDATE>(maxExpAddr, xMaxExp, ureg,605 Reg::StoreUnAlign<uint16_t, Reg::PostLiteral::POST_MODE_UPDATE>(maxExpAddr, xMaxExp, ureg,
606 ELEMENT_AFTER_REDUCE_);606 ELEMENT_AFTER_REDUCE_);
@@ -626,7 +626,7 @@ __aicore__ inline void DynamicMxQuantTailAxis<T, U, SCALE_ALG>::ComputeMaxExpDyn
626 Reg::Duplicate(absMask16Bit, BF16_ABS_MASK);626 Reg::Duplicate(absMask16Bit, BF16_ABS_MASK);
627 627 
628 Reg::MaskReg Mask = Reg::CreateMask<uint16_t, Reg::MaskPattern::ALL>();628 Reg::MaskReg Mask = Reg::CreateMask<uint16_t, Reg::MaskPattern::ALL>();
629- Reg::UnalignReg ureg;629+ Reg::UnalignRegForStore ureg;
630 630 
631 for (uint16_t i = 0; i < loopNum2VF; i++) {631 for (uint16_t i = 0; i < loopNum2VF; i++) {
632 Reg::LoadAlign<bfloat16_t, Reg::PostLiteral::POST_MODE_UPDATE, Reg::LoadDist::DIST_DINTLV_B16>(632 Reg::LoadAlign<bfloat16_t, Reg::PostLiteral::POST_MODE_UPDATE, Reg::LoadDist::DIST_DINTLV_B16>(
@@ -634,7 +634,7 @@ __aicore__ inline void DynamicMxQuantTailAxis<T, U, SCALE_ALG>::ComputeMaxExpDyn
634 Reg::And(xExpExtract0, (Reg::RegTensor<uint16_t>&)xExp0, absMask16Bit, Mask);634 Reg::And(xExpExtract0, (Reg::RegTensor<uint16_t>&)xExp0, absMask16Bit, Mask);
635 Reg::And(xExpExtract1, (Reg::RegTensor<uint16_t>&)xExp1, absMask16Bit, Mask);635 Reg::And(xExpExtract1, (Reg::RegTensor<uint16_t>&)xExp1, absMask16Bit, Mask);
636 Reg::Max(xMaxExp, xExpExtract0, xExpExtract1, Mask);636 Reg::Max(xMaxExp, xExpExtract0, xExpExtract1, Mask);
637- Reg::ReduceMaxWithDataBlock(xMaxExp, xMaxExp, Mask);637+ Reg::ReduceDataBlock<ReduceType::MAX>(xMaxExp, xMaxExp, Mask);
638 Reg::StoreUnAlign<uint16_t, Reg::PostLiteral::POST_MODE_UPDATE>(maxExpAddr, xMaxExp, ureg,638 Reg::StoreUnAlign<uint16_t, Reg::PostLiteral::POST_MODE_UPDATE>(maxExpAddr, xMaxExp, ureg,
639 ELEMENT_AFTER_REDUCE_);639 ELEMENT_AFTER_REDUCE_);
640 }640 }
@@ -660,7 +660,7 @@ __aicore__ inline void DynamicMxQuantTailAxis<T, U, SCALE_ALG>::ComputeMaxExpDyn
660 660 
661 Reg::MaskReg MaskB32 = Reg::CreateMask<uint32_t, Reg::MaskPattern::ALL>();661 Reg::MaskReg MaskB32 = Reg::CreateMask<uint32_t, Reg::MaskPattern::ALL>();
662 Reg::MaskReg Mask = Reg::CreateMask<uint16_t, Reg::MaskPattern::ALL>();662 Reg::MaskReg Mask = Reg::CreateMask<uint16_t, Reg::MaskPattern::ALL>();
663- Reg::UnalignReg ureg;663+ Reg::UnalignRegForStore ureg;
664 664 
665 for (uint16_t i = 0; i < loopNum2VF; i++) {665 for (uint16_t i = 0; i < loopNum2VF; i++) {
666 Reg::LoadAlign<float, Reg::PostLiteral::POST_MODE_UPDATE, Reg::LoadDist::DIST_DINTLV_B32>(666 Reg::LoadAlign<float, Reg::PostLiteral::POST_MODE_UPDATE, Reg::LoadDist::DIST_DINTLV_B32>(
@@ -676,7 +676,7 @@ __aicore__ inline void DynamicMxQuantTailAxis<T, U, SCALE_ALG>::ComputeMaxExpDyn
676 Reg::Max(xMaxExpB32, (Reg::RegTensor<uint32_t>&)xExp0, (Reg::RegTensor<uint32_t>&)xExp1, MaskB32);676 Reg::Max(xMaxExpB32, (Reg::RegTensor<uint32_t>&)xExp0, (Reg::RegTensor<uint32_t>&)xExp1, MaskB32);
677 Reg::Max(xMaxExpB32, (Reg::RegTensor<uint32_t>&)xMaxExpB32, (Reg::RegTensor<uint32_t>&)xExp2, MaskB32);677 Reg::Max(xMaxExpB32, (Reg::RegTensor<uint32_t>&)xMaxExpB32, (Reg::RegTensor<uint32_t>&)xExp2, MaskB32);
678 Reg::Max(xMaxExpB32, (Reg::RegTensor<uint32_t>&)xMaxExpB32, (Reg::RegTensor<uint32_t>&)xExp3, MaskB32);678 Reg::Max(xMaxExpB32, (Reg::RegTensor<uint32_t>&)xMaxExpB32, (Reg::RegTensor<uint32_t>&)xExp3, MaskB32);
679- Reg::ReduceMaxWithDataBlock(xMaxExpB32, xMaxExpB32, MaskB32);679+ Reg::ReduceDataBlock<ReduceType::MAX>(xMaxExpB32, xMaxExpB32, MaskB32);
680 Reg::ShiftRights(xMaxExpB32, xMaxExpB32, FP32_PACK_SHR_NUM, MaskB32);680 Reg::ShiftRights(xMaxExpB32, xMaxExpB32, FP32_PACK_SHR_NUM, MaskB32);
681 Reg::Pack<uint16_t, uint32_t, Reg::HighLowPart::LOWEST>(xMaxExpB16, xMaxExpB32);681 Reg::Pack<uint16_t, uint32_t, Reg::HighLowPart::LOWEST>(xMaxExpB16, xMaxExpB32);
682 Reg::StoreUnAlign<uint16_t, Reg::PostLiteral::POST_MODE_UPDATE>(maxExpAddr, xMaxExpB16, ureg,682 Reg::StoreUnAlign<uint16_t, Reg::PostLiteral::POST_MODE_UPDATE>(maxExpAddr, xMaxExpB16, ureg,
@@ -702,7 +702,7 @@ __aicore__ inline void DynamicMxQuantTailAxis<T, U, SCALE_ALG>::ComputeMaxExpCub
702 Reg::Duplicate(absMask16Bit, BF16_ABS_MASK);702 Reg::Duplicate(absMask16Bit, BF16_ABS_MASK);
703 703 
704 Reg::MaskReg Mask = Reg::CreateMask<uint16_t, Reg::MaskPattern::ALL>();704 Reg::MaskReg Mask = Reg::CreateMask<uint16_t, Reg::MaskPattern::ALL>();
705- Reg::UnalignReg ureg;705+ Reg::UnalignRegForStore ureg;
706 706 
707 for (uint16_t i = 0; i < loopNum2VF; i++) {707 for (uint16_t i = 0; i < loopNum2VF; i++) {
708 Reg::LoadAlign<T, Reg::PostLiteral::POST_MODE_UPDATE, Reg::LoadDist::DIST_DINTLV_B16>(708 Reg::LoadAlign<T, Reg::PostLiteral::POST_MODE_UPDATE, Reg::LoadDist::DIST_DINTLV_B16>(
@@ -710,7 +710,7 @@ __aicore__ inline void DynamicMxQuantTailAxis<T, U, SCALE_ALG>::ComputeMaxExpCub
710 Reg::And((Reg::RegTensor<uint16_t>&)xExp0, (Reg::RegTensor<uint16_t>&)xExp0, absMask16Bit, Mask);710 Reg::And((Reg::RegTensor<uint16_t>&)xExp0, (Reg::RegTensor<uint16_t>&)xExp0, absMask16Bit, Mask);
711 Reg::And((Reg::RegTensor<uint16_t>&)xExp1, (Reg::RegTensor<uint16_t>&)xExp1, absMask16Bit, Mask);711 Reg::And((Reg::RegTensor<uint16_t>&)xExp1, (Reg::RegTensor<uint16_t>&)xExp1, absMask16Bit, Mask);
712 Reg::Max(xMaxExp, (Reg::RegTensor<uint16_t>&)xExp0, (Reg::RegTensor<uint16_t>&)xExp1, Mask);712 Reg::Max(xMaxExp, (Reg::RegTensor<uint16_t>&)xExp0, (Reg::RegTensor<uint16_t>&)xExp1, Mask);
713- Reg::ReduceMaxWithDataBlock(xMaxExp, xMaxExp, Mask);713+ Reg::ReduceDataBlock<ReduceType::MAX>(xMaxExp, xMaxExp, Mask);
714 Reg::StoreUnAlign<uint16_t, Reg::PostLiteral::POST_MODE_UPDATE>(maxExpAddr, xMaxExp, ureg,714 Reg::StoreUnAlign<uint16_t, Reg::PostLiteral::POST_MODE_UPDATE>(maxExpAddr, xMaxExp, ureg,
715 ELEMENT_AFTER_REDUCE_);715 ELEMENT_AFTER_REDUCE_);
716 }716 }
@@ -734,7 +734,7 @@ __aicore__ inline void DynamicMxQuantTailAxis<T, U, SCALE_ALG>::ComputeMaxExpCub
734 Reg::Duplicate(absMask16Bit, BF16_ABS_MASK);734 Reg::Duplicate(absMask16Bit, BF16_ABS_MASK);
735 735 
736 Reg::MaskReg Mask = Reg::CreateMask<uint16_t, Reg::MaskPattern::ALL>();736 Reg::MaskReg Mask = Reg::CreateMask<uint16_t, Reg::MaskPattern::ALL>();
737- Reg::UnalignReg ureg;737+ Reg::UnalignRegForStore ureg;
738 738 
739 for (uint16_t i = 0; i < loopNum2VF; i++) {739 for (uint16_t i = 0; i < loopNum2VF; i++) {
740 Reg::LoadAlign<bfloat16_t, Reg::PostLiteral::POST_MODE_UPDATE, Reg::LoadDist::DIST_DINTLV_B16>(740 Reg::LoadAlign<bfloat16_t, Reg::PostLiteral::POST_MODE_UPDATE, Reg::LoadDist::DIST_DINTLV_B16>(
@@ -742,7 +742,7 @@ __aicore__ inline void DynamicMxQuantTailAxis<T, U, SCALE_ALG>::ComputeMaxExpCub
742 Reg::And((Reg::RegTensor<uint16_t>&)xExp0, (Reg::RegTensor<uint16_t>&)xExp0, absMask16Bit, Mask);742 Reg::And((Reg::RegTensor<uint16_t>&)xExp0, (Reg::RegTensor<uint16_t>&)xExp0, absMask16Bit, Mask);
743 Reg::And((Reg::RegTensor<uint16_t>&)xExp1, (Reg::RegTensor<uint16_t>&)xExp1, absMask16Bit, Mask);743 Reg::And((Reg::RegTensor<uint16_t>&)xExp1, (Reg::RegTensor<uint16_t>&)xExp1, absMask16Bit, Mask);
744 Reg::Max(xMaxExp, (Reg::RegTensor<uint16_t>&)xExp0, (Reg::RegTensor<uint16_t>&)xExp1, Mask);744 Reg::Max(xMaxExp, (Reg::RegTensor<uint16_t>&)xExp0, (Reg::RegTensor<uint16_t>&)xExp1, Mask);
745- Reg::ReduceMaxWithDataBlock(xMaxExp, xMaxExp, Mask);745+ Reg::ReduceDataBlock<ReduceType::MAX>(xMaxExp, xMaxExp, Mask);
746 Reg::StoreUnAlign<uint16_t, Reg::PostLiteral::POST_MODE_UPDATE>(maxExpAddr, xMaxExp, ureg,746 Reg::StoreUnAlign<uint16_t, Reg::PostLiteral::POST_MODE_UPDATE>(maxExpAddr, xMaxExp, ureg,
747 ELEMENT_AFTER_REDUCE_);747 ELEMENT_AFTER_REDUCE_);
748 }748 }
@@ -768,7 +768,7 @@ __aicore__ inline void DynamicMxQuantTailAxis<T, U, SCALE_ALG>::ComputeMaxExpCub
768 Reg::Duplicate(absMask32Bit, FP32_ABS_MASK);768 Reg::Duplicate(absMask32Bit, FP32_ABS_MASK);
769 769 
770 Reg::MaskReg Mask = Reg::CreateMask<uint16_t, Reg::MaskPattern::ALL>();770 Reg::MaskReg Mask = Reg::CreateMask<uint16_t, Reg::MaskPattern::ALL>();
771- Reg::UnalignReg ureg;771+ Reg::UnalignRegForStore ureg;
772 772 
773 for (uint16_t i = 0; i < loopNum2VF; i++) {773 for (uint16_t i = 0; i < loopNum2VF; i++) {
774 Reg::LoadAlign<float, Reg::PostLiteral::POST_MODE_UPDATE, Reg::LoadDist::DIST_DINTLV_B32>(774 Reg::LoadAlign<float, Reg::PostLiteral::POST_MODE_UPDATE, Reg::LoadDist::DIST_DINTLV_B32>(
@@ -784,7 +784,7 @@ __aicore__ inline void DynamicMxQuantTailAxis<T, U, SCALE_ALG>::ComputeMaxExpCub
784 Reg::Max(xMaxExp, (Reg::RegTensor<uint32_t>&)xExp0, (Reg::RegTensor<uint32_t>&)xExp1, Mask);784 Reg::Max(xMaxExp, (Reg::RegTensor<uint32_t>&)xExp0, (Reg::RegTensor<uint32_t>&)xExp1, Mask);
785 Reg::Max(xMaxExp, (Reg::RegTensor<uint32_t>&)xMaxExp, (Reg::RegTensor<uint32_t>&)xExp2, Mask);785 Reg::Max(xMaxExp, (Reg::RegTensor<uint32_t>&)xMaxExp, (Reg::RegTensor<uint32_t>&)xExp2, Mask);
786 Reg::Max(xMaxExp, (Reg::RegTensor<uint32_t>&)xMaxExp, (Reg::RegTensor<uint32_t>&)xExp3, Mask);786 Reg::Max(xMaxExp, (Reg::RegTensor<uint32_t>&)xMaxExp, (Reg::RegTensor<uint32_t>&)xExp3, Mask);
787- Reg::ReduceMaxWithDataBlock(xMaxExp, xMaxExp, Mask);787+ Reg::ReduceDataBlock<ReduceType::MAX>(xMaxExp, xMaxExp, Mask);
788 Reg::StoreUnAlign<uint32_t, Reg::PostLiteral::POST_MODE_UPDATE>(maxExpAddr, xMaxExp, ureg,788 Reg::StoreUnAlign<uint32_t, Reg::PostLiteral::POST_MODE_UPDATE>(maxExpAddr, xMaxExp, ureg,
789 ELEMENT_AFTER_REDUCE_);789 ELEMENT_AFTER_REDUCE_);
790 }790 }
@@ -995,11 +995,11 @@ __aicore__ inline void DynamicMxQuantTailAxis<T, U, SCALE_ALG>::ComputeScaleCubl
995 Reg::ShiftRights(exp32, max32, FP32_SHR_NUM, preMaskScale);995 Reg::ShiftRights(exp32, max32, FP32_SHR_NUM, preMaskScale);
996 Reg::And(man32, max32, manMaskFP32, preMaskScale);996 Reg::And(man32, max32, manMaskFP32, preMaskScale);
997 997 
998- Reg::CompareScalar<uint32_t, CMPMODE::GT>(p0, exp32, FP32_NUMBER_ZERO, preMaskScale);998+ Reg::Compares<uint32_t, CMPMODE::GT>(p0, exp32, FP32_NUMBER_ZERO, preMaskScale);
999- Reg::CompareScalar<uint32_t, CMPMODE::LT>(p1, exp32, FP32_NUMBER_254, preMaskScale);999+ Reg::Compares<uint32_t, CMPMODE::LT>(p1, exp32, FP32_NUMBER_254, preMaskScale);
1000- Reg::CompareScalar<uint32_t, CMPMODE::GT>(p2, man32, FP32_NUMBER_ZERO, preMaskScale);1000+ Reg::Compares<uint32_t, CMPMODE::GT>(p2, man32, FP32_NUMBER_ZERO, preMaskScale);
1001- Reg::MaskAnd(p0, p0, p1, preMaskScale);1001+ Reg::And(p0, p0, p1, preMaskScale);
1002- Reg::MaskAnd(p0, p0, p2, preMaskScale);1002+ Reg::And(p0, p0, p2, preMaskScale);
1003 1003 
1004 Reg::Adds(expAddOne32, exp32, 1, preMaskScale);1004 Reg::Adds(expAddOne32, exp32, 1, preMaskScale);
1005 Reg::Select(extractExp, expAddOne32, exp32, p0);1005 Reg::Select(extractExp, expAddOne32, exp32, p0);
@@ -1599,9 +1599,9 @@ __aicore__ inline void DynamicMxQuantTailAxis<T, U, SCALE_ALG>::ComputeFP4FromHa
1599 Reg::Duplicate(specialValueTensor, specialValue);1599 Reg::Duplicate(specialValueTensor, specialValue);
1600 Reg::Duplicate(newMantissa, FP4_NEW_MANTISSA);1600 Reg::Duplicate(newMantissa, FP4_NEW_MANTISSA);
1601 Reg::And(andResult, (Reg::RegTensor<uint16_t>&)input, specialValueTensor, mask);1601 Reg::And(andResult, (Reg::RegTensor<uint16_t>&)input, specialValueTensor, mask);
1602- Reg::CompareScalar<uint16_t, CMPMODE::GT>(nonzeroMask, andResult, 0, mask);1602+ Reg::Compares<uint16_t, CMPMODE::GT>(nonzeroMask, andResult, 0, mask);
1603- Reg::CompareScalar<uint16_t, CMPMODE::LT>(specialMask, andResult, FP4_NEW_MANTISSA, mask);1603+ Reg::Compares<uint16_t, CMPMODE::LT>(specialMask, andResult, FP4_NEW_MANTISSA, mask);
1604- Reg::MaskAnd(specialMask, specialMask, nonzeroMask, mask);1604+ Reg::And(specialMask, specialMask, nonzeroMask, mask);
1605 Reg::Or(newValue, (Reg::RegTensor<uint16_t>&)input, newMantissa, mask);1605 Reg::Or(newValue, (Reg::RegTensor<uint16_t>&)input, newMantissa, mask);
1606 Reg::Select<uint16_t>((Reg::RegTensor<uint16_t>&)output, newValue, (Reg::RegTensor<uint16_t>&)input,1606 Reg::Select<uint16_t>((Reg::RegTensor<uint16_t>&)output, newValue, (Reg::RegTensor<uint16_t>&)input,
1607 specialMask);1607 specialMask);
@@ -1627,7 +1627,7 @@ __aicore__ inline void DynamicMxQuantTailAxis<T, U, SCALE_ALG>::ComputeFP4FromFp
1627 Reg::Compare<int32_t, CMPMODE::EQ>(negInfMask, (Reg::RegTensor<int32_t>&)Reg, negZero, pregAll32);1627 Reg::Compare<int32_t, CMPMODE::EQ>(negInfMask, (Reg::RegTensor<int32_t>&)Reg, negZero, pregAll32);
1628 if constexpr (IsSameType<U, fp4x2_e1m2_t>::value) {1628 if constexpr (IsSameType<U, fp4x2_e1m2_t>::value) {
1629 Reg::Muls(Reg, Reg, FP4_SCALE_FACTOR, pregAll32);1629 Reg::Muls(Reg, Reg, FP4_SCALE_FACTOR, pregAll32);
1630- Reg::CompareScalar<float, CMPMODE::LT>(specialMask, Reg, 0, pregAll32);1630+ Reg::Compares<float, CMPMODE::LT>(specialMask, Reg, 0, pregAll32);
1631 Reg::Truncate<float, roundMode>(Reg, Reg, pregAll32);1631 Reg::Truncate<float, roundMode>(Reg, Reg, pregAll32);
1632 Reg::Muls(Reg, Reg, FP4_INV_SCALE_FACTOR, pregAll32);1632 Reg::Muls(Reg, Reg, FP4_INV_SCALE_FACTOR, pregAll32);
1633 } else {1633 } else {
@@ -1644,13 +1644,13 @@ __aicore__ inline void DynamicMxQuantTailAxis<T, U, SCALE_ALG>::ComputeFP4FromFp
1644 Reg::Mul(Reg, Reg, (Reg::RegTensor<float>&)exp1FP32, pregAll32);1644 Reg::Mul(Reg, Reg, (Reg::RegTensor<float>&)exp1FP32, pregAll32);
1645 Reg::Adds(exp0FP32, exp0FP32, FP32_BIAS_VALUE, pregAll32);1645 Reg::Adds(exp0FP32, exp0FP32, FP32_BIAS_VALUE, pregAll32);
1646 Reg::ShiftLefts(exp0FP32, exp0FP32, FP32_SHR_NUM, pregAll32);1646 Reg::ShiftLefts(exp0FP32, exp0FP32, FP32_SHR_NUM, pregAll32);
1647- Reg::CompareScalar<float, CMPMODE::LT>(specialMask, Reg, 0, pregAll32);1647+ Reg::Compares<float, CMPMODE::LT>(specialMask, Reg, 0, pregAll32);
1648 Reg::Truncate<float, roundMode>(Reg, Reg, pregAll32);1648 Reg::Truncate<float, roundMode>(Reg, Reg, pregAll32);
1649 Reg::Mul(Reg, Reg, (Reg::RegTensor<float>&)exp0FP32, pregAll32);1649 Reg::Mul(Reg, Reg, (Reg::RegTensor<float>&)exp0FP32, pregAll32);
1650 }1650 }
1651- Reg::CompareScalar<float, CMPMODE::EQ>(zeroMask, Reg, 0, pregAll32);1651+ Reg::Compares<float, CMPMODE::EQ>(zeroMask, Reg, 0, pregAll32);
1652- Reg::MaskAnd(zeroMask, specialMask, zeroMask, pregAll32);1652+ Reg::And(zeroMask, specialMask, zeroMask, pregAll32);
1653- Reg::MaskOr(zeroMask, negInfMask, zeroMask, pregAll32);1653+ Reg::Or(zeroMask, negInfMask, zeroMask, pregAll32);
1654 Reg::Select<int32_t>((Reg::RegTensor<int32_t>&)Reg, negZero, (Reg::RegTensor<int32_t>&)Reg, zeroMask);1654 Reg::Select<int32_t>((Reg::RegTensor<int32_t>&)Reg, negZero, (Reg::RegTensor<int32_t>&)Reg, zeroMask);
1655}1655}
1656 1656 
Mquant/dynamic_mx_quant/op_kernel/arch35/dynamic_mx_quant_tail_axis_fp8.h+21-21
@@ -376,7 +376,7 @@ __aicore__ inline void DynamicMxQuantTailAxisFP8<T, U, SCALE_ALG>::ComputeMaxExp
376 Reg::Duplicate(expMaskBF16, BF16_MAX_EXP);376 Reg::Duplicate(expMaskBF16, BF16_MAX_EXP);
377 377 
378 Reg::MaskReg Mask = Reg::CreateMask<uint16_t, Reg::MaskPattern::ALL>();378 Reg::MaskReg Mask = Reg::CreateMask<uint16_t, Reg::MaskPattern::ALL>();
379- Reg::UnalignReg ureg;379+ Reg::UnalignRegForStore ureg;
380 380 
381 for (uint16_t i = 0; i < loopNum2VF; i++) {381 for (uint16_t i = 0; i < loopNum2VF; i++) {
382 Reg::LoadAlign<bfloat16_t, Reg::PostLiteral::POST_MODE_UPDATE, Reg::LoadDist::DIST_DINTLV_B16>(382 Reg::LoadAlign<bfloat16_t, Reg::PostLiteral::POST_MODE_UPDATE, Reg::LoadDist::DIST_DINTLV_B16>(
@@ -384,7 +384,7 @@ __aicore__ inline void DynamicMxQuantTailAxisFP8<T, U, SCALE_ALG>::ComputeMaxExp
384 Reg::And(xExpExtract0, (Reg::RegTensor<uint16_t>&)x0, expMaskBF16, Mask);384 Reg::And(xExpExtract0, (Reg::RegTensor<uint16_t>&)x0, expMaskBF16, Mask);
385 Reg::And(xExpExtract1, (Reg::RegTensor<uint16_t>&)x1, expMaskBF16, Mask);385 Reg::And(xExpExtract1, (Reg::RegTensor<uint16_t>&)x1, expMaskBF16, Mask);
386 Reg::Max(xMaxExp, xExpExtract0, xExpExtract1, Mask);386 Reg::Max(xMaxExp, xExpExtract0, xExpExtract1, Mask);
387- Reg::ReduceMaxWithDataBlock(xMaxExp, xMaxExp, Mask);387+ Reg::ReduceDataBlock<ReduceType::MAX>(xMaxExp, xMaxExp, Mask);
388 388 
389 Reg::StoreUnAlign<uint16_t, Reg::PostLiteral::POST_MODE_UPDATE>(maxExpAddr, xMaxExp, ureg,389 Reg::StoreUnAlign<uint16_t, Reg::PostLiteral::POST_MODE_UPDATE>(maxExpAddr, xMaxExp, ureg,
390 ELEMENT_AFTER_REDUCE_);390 ELEMENT_AFTER_REDUCE_);
@@ -408,7 +408,7 @@ __aicore__ inline void DynamicMxQuantTailAxisFP8<T, U, SCALE_ALG>::ComputeMaxExp
408 Reg::Duplicate(absMask16Bit, BF16_ABS_MASK);408 Reg::Duplicate(absMask16Bit, BF16_ABS_MASK);
409 409 
410 Reg::MaskReg Mask = Reg::CreateMask<uint16_t, Reg::MaskPattern::ALL>();410 Reg::MaskReg Mask = Reg::CreateMask<uint16_t, Reg::MaskPattern::ALL>();
411- Reg::UnalignReg ureg;411+ Reg::UnalignRegForStore ureg;
412 412 
413 for (uint16_t i = 0; i < loopNum2VF; i++) {413 for (uint16_t i = 0; i < loopNum2VF; i++) {
414 Reg::LoadAlign<bfloat16_t, Reg::PostLiteral::POST_MODE_UPDATE, Reg::LoadDist::DIST_DINTLV_B16>(414 Reg::LoadAlign<bfloat16_t, Reg::PostLiteral::POST_MODE_UPDATE, Reg::LoadDist::DIST_DINTLV_B16>(
@@ -416,7 +416,7 @@ __aicore__ inline void DynamicMxQuantTailAxisFP8<T, U, SCALE_ALG>::ComputeMaxExp
416 Reg::And((Reg::RegTensor<uint16_t>&)x0, (Reg::RegTensor<uint16_t>&)x0, absMask16Bit, Mask);416 Reg::And((Reg::RegTensor<uint16_t>&)x0, (Reg::RegTensor<uint16_t>&)x0, absMask16Bit, Mask);
417 Reg::And((Reg::RegTensor<uint16_t>&)x1, (Reg::RegTensor<uint16_t>&)x1, absMask16Bit, Mask);417 Reg::And((Reg::RegTensor<uint16_t>&)x1, (Reg::RegTensor<uint16_t>&)x1, absMask16Bit, Mask);
418 Reg::Max(xMaxExp, (Reg::RegTensor<uint16_t>&)x0, (Reg::RegTensor<uint16_t>&)x1, Mask);418 Reg::Max(xMaxExp, (Reg::RegTensor<uint16_t>&)x0, (Reg::RegTensor<uint16_t>&)x1, Mask);
419- Reg::ReduceMaxWithDataBlock(xMaxExp, xMaxExp, Mask);419+ Reg::ReduceDataBlock<ReduceType::MAX>(xMaxExp, xMaxExp, Mask);
420 Reg::StoreUnAlign<uint16_t, Reg::PostLiteral::POST_MODE_UPDATE>(maxExpAddr, xMaxExp, ureg,420 Reg::StoreUnAlign<uint16_t, Reg::PostLiteral::POST_MODE_UPDATE>(maxExpAddr, xMaxExp, ureg,
421 ELEMENT_AFTER_REDUCE_);421 ELEMENT_AFTER_REDUCE_);
422 }422 }
@@ -450,7 +450,7 @@ __aicore__ inline void DynamicMxQuantTailAxisFP8<T, U, SCALE_ALG>::ComputeMaxExp
450 Reg::MaskReg Mask = Reg::CreateMask<uint16_t, Reg::MaskPattern::ALL>();450 Reg::MaskReg Mask = Reg::CreateMask<uint16_t, Reg::MaskPattern::ALL>();
451 Reg::MaskReg invalidDataMask0;451 Reg::MaskReg invalidDataMask0;
452 Reg::MaskReg invalidDataMask1;452 Reg::MaskReg invalidDataMask1;
453- Reg::UnalignReg ureg;453+ Reg::UnalignRegForStore ureg;
454 454 
455 static constexpr Reg::CastTrait castTraitHalf2Bf16 = {Reg::RegLayout::UNKNOWN, Reg::SatMode::UNKNOWN,455 static constexpr Reg::CastTrait castTraitHalf2Bf16 = {Reg::RegLayout::UNKNOWN, Reg::SatMode::UNKNOWN,
456 Reg::MaskMergeMode::ZEROING, RoundMode::CAST_TRUNC};456 Reg::MaskMergeMode::ZEROING, RoundMode::CAST_TRUNC};
@@ -470,7 +470,7 @@ __aicore__ inline void DynamicMxQuantTailAxisFP8<T, U, SCALE_ALG>::ComputeMaxExp
470 Reg::Select<uint16_t>(xExpExtract1, xExpExtract1, expMaskBF16, invalidDataMask1);470 Reg::Select<uint16_t>(xExpExtract1, xExpExtract1, expMaskBF16, invalidDataMask1);
471 471 
472 Reg::Max(xMaxExp, xExpExtract0, xExpExtract1, Mask);472 Reg::Max(xMaxExp, xExpExtract0, xExpExtract1, Mask);
473- Reg::ReduceMaxWithDataBlock(xMaxExp, xMaxExp, Mask);473+ Reg::ReduceDataBlock<ReduceType::MAX>(xMaxExp, xMaxExp, Mask);
474 474 
475 Reg::StoreUnAlign<uint16_t, Reg::PostLiteral::POST_MODE_UPDATE>(maxExpAddr, xMaxExp, ureg,475 Reg::StoreUnAlign<uint16_t, Reg::PostLiteral::POST_MODE_UPDATE>(maxExpAddr, xMaxExp, ureg,
476 ELEMENT_AFTER_REDUCE_);476 ELEMENT_AFTER_REDUCE_);
@@ -494,7 +494,7 @@ __aicore__ inline void DynamicMxQuantTailAxisFP8<T, U, SCALE_ALG>::ComputeMaxExp
494 Reg::Duplicate(absMask16Bit, BF16_ABS_MASK);494 Reg::Duplicate(absMask16Bit, BF16_ABS_MASK);
495 495 
496 Reg::MaskReg Mask = Reg::CreateMask<uint16_t, Reg::MaskPattern::ALL>();496 Reg::MaskReg Mask = Reg::CreateMask<uint16_t, Reg::MaskPattern::ALL>();
497- Reg::UnalignReg ureg;497+ Reg::UnalignRegForStore ureg;
498 498 
499 for (uint16_t i = 0; i < loopNum2VF; i++) {499 for (uint16_t i = 0; i < loopNum2VF; i++) {
500 Reg::LoadAlign<half, Reg::PostLiteral::POST_MODE_UPDATE, Reg::LoadDist::DIST_DINTLV_B16>(x0, x1, xLocalAddr,500 Reg::LoadAlign<half, Reg::PostLiteral::POST_MODE_UPDATE, Reg::LoadDist::DIST_DINTLV_B16>(x0, x1, xLocalAddr,
@@ -502,7 +502,7 @@ __aicore__ inline void DynamicMxQuantTailAxisFP8<T, U, SCALE_ALG>::ComputeMaxExp
502 Reg::And((Reg::RegTensor<uint16_t>&)x0, (Reg::RegTensor<uint16_t>&)x0, absMask16Bit, Mask);502 Reg::And((Reg::RegTensor<uint16_t>&)x0, (Reg::RegTensor<uint16_t>&)x0, absMask16Bit, Mask);
503 Reg::And((Reg::RegTensor<uint16_t>&)x1, (Reg::RegTensor<uint16_t>&)x1, absMask16Bit, Mask);503 Reg::And((Reg::RegTensor<uint16_t>&)x1, (Reg::RegTensor<uint16_t>&)x1, absMask16Bit, Mask);
504 Reg::Max(xMaxExp, (Reg::RegTensor<uint16_t>&)x0, (Reg::RegTensor<uint16_t>&)x1, Mask);504 Reg::Max(xMaxExp, (Reg::RegTensor<uint16_t>&)x0, (Reg::RegTensor<uint16_t>&)x1, Mask);
505- Reg::ReduceMaxWithDataBlock(xMaxExp, xMaxExp, Mask);505+ Reg::ReduceDataBlock<ReduceType::MAX>(xMaxExp, xMaxExp, Mask);
506 Reg::StoreUnAlign<uint16_t, Reg::PostLiteral::POST_MODE_UPDATE>(maxExpAddr, xMaxExp, ureg,506 Reg::StoreUnAlign<uint16_t, Reg::PostLiteral::POST_MODE_UPDATE>(maxExpAddr, xMaxExp, ureg,
507 ELEMENT_AFTER_REDUCE_);507 ELEMENT_AFTER_REDUCE_);
508 }508 }
@@ -532,7 +532,7 @@ __aicore__ inline void DynamicMxQuantTailAxisFP8<T, U, SCALE_ALG>::ComputeMaxExp
532 532 
533 Reg::MaskReg MaskB32 = Reg::CreateMask<uint32_t, Reg::MaskPattern::ALL>();533 Reg::MaskReg MaskB32 = Reg::CreateMask<uint32_t, Reg::MaskPattern::ALL>();
534 Reg::MaskReg Mask = Reg::CreateMask<uint16_t, Reg::MaskPattern::ALL>();534 Reg::MaskReg Mask = Reg::CreateMask<uint16_t, Reg::MaskPattern::ALL>();
535- Reg::UnalignReg ureg;535+ Reg::UnalignRegForStore ureg;
536 536 
537 for (uint16_t i = 0; i < loopNum2VF; i++) {537 for (uint16_t i = 0; i < loopNum2VF; i++) {
538 // x0ZeroFP32: 1,3,5,7,9,...,127 x1ZeroFP32: 2,4,6,8,10,...,128538 // x0ZeroFP32: 1,3,5,7,9,...,127 x1ZeroFP32: 2,4,6,8,10,...,128
@@ -554,7 +554,7 @@ __aicore__ inline void DynamicMxQuantTailAxisFP8<T, U, SCALE_ALG>::ComputeMaxExp
554 Reg::Max(x0MaxExpB32, (Reg::RegTensor<uint32_t>&)x0ZeroFP32, (Reg::RegTensor<uint32_t>&)x0OneFP32, MaskB32);554 Reg::Max(x0MaxExpB32, (Reg::RegTensor<uint32_t>&)x0ZeroFP32, (Reg::RegTensor<uint32_t>&)x0OneFP32, MaskB32);
555 Reg::Max(x1MaxExpB32, (Reg::RegTensor<uint32_t>&)x1ZeroFP32, (Reg::RegTensor<uint32_t>&)x1OneFP32, MaskB32);555 Reg::Max(x1MaxExpB32, (Reg::RegTensor<uint32_t>&)x1ZeroFP32, (Reg::RegTensor<uint32_t>&)x1OneFP32, MaskB32);
556 Reg::Max(xMaxExpB32, x0MaxExpB32, x1MaxExpB32, MaskB32);556 Reg::Max(xMaxExpB32, x0MaxExpB32, x1MaxExpB32, MaskB32);
557- Reg::ReduceMaxWithDataBlock(xMaxExpB32, xMaxExpB32, MaskB32);557+ Reg::ReduceDataBlock<ReduceType::MAX>(xMaxExpB32, xMaxExpB32, MaskB32);
558 Reg::ShiftRights(xMaxExpB32, xMaxExpB32, FP32_PACK_SHR_NUM, MaskB32);558 Reg::ShiftRights(xMaxExpB32, xMaxExpB32, FP32_PACK_SHR_NUM, MaskB32);
559 Reg::Pack<uint16_t, uint32_t, Reg::HighLowPart::LOWEST>(xMaxExpB16, xMaxExpB32);559 Reg::Pack<uint16_t, uint32_t, Reg::HighLowPart::LOWEST>(xMaxExpB16, xMaxExpB32);
560 Reg::StoreUnAlign<uint16_t, Reg::PostLiteral::POST_MODE_UPDATE>(maxExpAddr, xMaxExpB16, ureg,560 Reg::StoreUnAlign<uint16_t, Reg::PostLiteral::POST_MODE_UPDATE>(maxExpAddr, xMaxExpB16, ureg,
@@ -584,7 +584,7 @@ __aicore__ inline void DynamicMxQuantTailAxisFP8<T, U, SCALE_ALG>::ComputeMaxExp
584 584 
585 Reg::MaskReg MaskB32 = Reg::CreateMask<uint32_t, Reg::MaskPattern::ALL>();585 Reg::MaskReg MaskB32 = Reg::CreateMask<uint32_t, Reg::MaskPattern::ALL>();
586 Reg::MaskReg Mask = Reg::CreateMask<uint16_t, Reg::MaskPattern::ALL>();586 Reg::MaskReg Mask = Reg::CreateMask<uint16_t, Reg::MaskPattern::ALL>();
587- Reg::UnalignReg ureg;587+ Reg::UnalignRegForStore ureg;
588 588 
589 for (uint16_t i = 0; i < loopNum2VF; i++) {589 for (uint16_t i = 0; i < loopNum2VF; i++) {
590 // x0ZeroFP32: 1,3,5,7,9,...,127 x1ZeroFP32: 2,4,6,8,10,...,128590 // x0ZeroFP32: 1,3,5,7,9,...,127 x1ZeroFP32: 2,4,6,8,10,...,128
@@ -606,7 +606,7 @@ __aicore__ inline void DynamicMxQuantTailAxisFP8<T, U, SCALE_ALG>::ComputeMaxExp
606 Reg::Max(x0MaxExpB32, (Reg::RegTensor<uint32_t>&)x0ZeroFP32, (Reg::RegTensor<uint32_t>&)x0OneFP32, MaskB32);606 Reg::Max(x0MaxExpB32, (Reg::RegTensor<uint32_t>&)x0ZeroFP32, (Reg::RegTensor<uint32_t>&)x0OneFP32, MaskB32);
607 Reg::Max(x1MaxExpB32, (Reg::RegTensor<uint32_t>&)x1ZeroFP32, (Reg::RegTensor<uint32_t>&)x1OneFP32, MaskB32);607 Reg::Max(x1MaxExpB32, (Reg::RegTensor<uint32_t>&)x1ZeroFP32, (Reg::RegTensor<uint32_t>&)x1OneFP32, MaskB32);
608 Reg::Max(xMaxExpB32, x0MaxExpB32, x1MaxExpB32, MaskB32);608 Reg::Max(xMaxExpB32, x0MaxExpB32, x1MaxExpB32, MaskB32);
609- Reg::ReduceMaxWithDataBlock(xMaxExpB32, xMaxExpB32, MaskB32);609+ Reg::ReduceDataBlock<ReduceType::MAX>(xMaxExpB32, xMaxExpB32, MaskB32);
610 Reg::StoreUnAlign<uint32_t, Reg::PostLiteral::POST_MODE_UPDATE>(maxExpAddr, xMaxExpB32, ureg,610 Reg::StoreUnAlign<uint32_t, Reg::PostLiteral::POST_MODE_UPDATE>(maxExpAddr, xMaxExpB32, ureg,
611 ELEMENT_AFTER_REDUCE_);611 ELEMENT_AFTER_REDUCE_);
612 }612 }
@@ -748,16 +748,16 @@ __aicore__ inline void DynamicMxQuantTailAxisFP8<T, U, SCALE_ALG>::ComputeScaleC
748 Reg::ShiftRights(exp32, max32, FP32_SHR_NUM, maskFloat);748 Reg::ShiftRights(exp32, max32, FP32_SHR_NUM, maskFloat);
749 Reg::And(man32, max32, manMaskFP32, maskFloat);749 Reg::And(man32, max32, manMaskFP32, maskFloat);
750 750 
751- Reg::CompareScalar<uint32_t, CMPMODE::GT>(p0, exp32, FP32_NUMBER_ZERO, maskFloat);751+ Reg::Compares<uint32_t, CMPMODE::GT>(p0, exp32, FP32_NUMBER_ZERO, maskFloat);
752- Reg::CompareScalar<uint32_t, CMPMODE::LT>(p1, exp32, FP32_NUMBER_254, maskFloat);752+ Reg::Compares<uint32_t, CMPMODE::LT>(p1, exp32, FP32_NUMBER_254, maskFloat);
753- Reg::CompareScalar<uint32_t, CMPMODE::GT>(p2, man32, FP32_NUMBER_ZERO, maskFloat);753+ Reg::Compares<uint32_t, CMPMODE::GT>(p2, man32, FP32_NUMBER_ZERO, maskFloat);
754- Reg::MaskAnd(p0, p0, p1, maskFloat);754+ Reg::And(p0, p0, p1, maskFloat);
755- Reg::MaskAnd(p0, p0, p2, maskFloat);755+ Reg::And(p0, p0, p2, maskFloat);
756 756 
757- Reg::CompareScalar<uint32_t, CMPMODE::EQ>(p1, exp32, FP32_NUMBER_ZERO, maskFloat);757+ Reg::Compares<uint32_t, CMPMODE::EQ>(p1, exp32, FP32_NUMBER_ZERO, maskFloat);
758- Reg::CompareScalar<uint32_t, CMPMODE::GT>(p2, man32, FP32_NUMBER_HALF, maskFloat);758+ Reg::Compares<uint32_t, CMPMODE::GT>(p2, man32, FP32_NUMBER_HALF, maskFloat);
759- Reg::MaskAnd(p1, p1, p2, maskFloat);759+ Reg::And(p1, p1, p2, maskFloat);
760- Reg::MaskOr(p0, p0, p1, maskFloat);760+ Reg::Or(p0, p0, p1, maskFloat);
761 761 
762 Reg::Adds(expAddOne32, exp32, 1, maskFloat);762 Reg::Adds(expAddOne32, exp32, 1, maskFloat);
763 Reg::Select(extractExp, expAddOne32, exp32, p0);763 Reg::Select(extractExp, expAddOne32, exp32, p0);
Mquant/dynamic_mx_quant_with_dual_axis/op_kernel/arch35/dynamic_mx_quant_with_dual_axis_base.h+107-107
@@ -482,7 +482,7 @@ __aicore__ inline void DynamicMxQuantWithDualAxisBase<xDtype, y1Dtype, y2Dtype,
482 482 
483 for (uint16_t i = 0; i < blockCount; i++) {483 for (uint16_t i = 0; i < blockCount; i++) {
484 // 交织搬运，一次搬256个B16484 // 交织搬运，一次搬256个B16
485- MicroAPI::DataCopy<xDtype, MicroAPI::PostLiteral::POST_MODE_UPDATE, MicroAPI::LoadDist::DIST_DINTLV_B16>(485+ MicroAPI::LoadAlign<xDtype, MicroAPI::PostLiteral::POST_MODE_UPDATE, MicroAPI::LoadDist::DIST_DINTLV_B16>(
486 x0, x1, xAddr, vlForHalfNumber_ * DIGIT_TWO);486 x0, x1, xAddr, vlForHalfNumber_ * DIGIT_TWO);
487 if constexpr (IsSameType<xDtype, half>::value) {487 if constexpr (IsSameType<xDtype, half>::value) {
488 // 原始数据转成bf16488 // 原始数据转成bf16
@@ -499,7 +499,7 @@ __aicore__ inline void DynamicMxQuantWithDualAxisBase<xDtype, y1Dtype, y2Dtype,
499 // 计算x0和x1的最大值，相当于计算原始相邻两个数据的最大值499 // 计算x0和x1的最大值，相当于计算原始相邻两个数据的最大值
500 MicroAPI::Max(expMaxDim1, x0ExpBF16, x1ExpBF16, maskAll);500 MicroAPI::Max(expMaxDim1, x0ExpBF16, x1ExpBF16, maskAll);
501 // ReduceMax一个block，即16个数，配合上一步，可以计算出每32个数的最大值，一共256/32个501 // ReduceMax一个block，即16个数，配合上一步，可以计算出每32个数的最大值，一共256/32个
502- MicroAPI::ReduceMaxWithDataBlock(expMaxDim1, expMaxDim1, maskAll);502+ MicroAPI::ReduceDataBlock<MicroAPI::ReduceType::MAX>(expMaxDim1, expMaxDim1, maskAll);
503 // 二分性能更高，待定503 // 二分性能更高，待定
504 MicroAPI::Max(expMax1Dim2, expMax1Dim2, x0ExpBF16, maskAll);504 MicroAPI::Max(expMax1Dim2, expMax1Dim2, x0ExpBF16, maskAll);
505 MicroAPI::Max(expMax2Dim2, expMax2Dim2, x1ExpBF16, maskAll);505 MicroAPI::Max(expMax2Dim2, expMax2Dim2, x1ExpBF16, maskAll);
@@ -520,7 +520,7 @@ __aicore__ inline void DynamicMxQuantWithDualAxisBase<xDtype, y1Dtype, y2Dtype,
520 MicroAPI::Select<uint16_t>(mxScale1B16, mxScale1B16, zero, zeroMask);520 MicroAPI::Select<uint16_t>(mxScale1B16, mxScale1B16, zero, zeroMask);
521 521 
522 MicroAPI::Pack<uint8_t, uint16_t, MicroAPI::HighLowPart::LOWEST>(mxScale1B8, mxScale1B16);522 MicroAPI::Pack<uint8_t, uint16_t, MicroAPI::HighLowPart::LOWEST>(mxScale1B8, mxScale1B16);
523- MicroAPI::DataCopy<uint8_t>(mxScale1Addr + i * oneBlockCountB8_, mxScale1B8, maskReduceB8);523+ MicroAPI::StoreAlign<uint8_t>(mxScale1Addr + i * oneBlockCountB8_, mxScale1B8, maskReduceB8);
524 524 
525 // 公式中的1/X525 // 公式中的1/X
526 // 只有在E1M2时，yMaxExp=0，expMaxDim1可能会等于biasE8M0526 // 只有在E1M2时，yMaxExp=0，expMaxDim1可能会等于biasE8M0
@@ -530,8 +530,8 @@ __aicore__ inline void DynamicMxQuantWithDualAxisBase<xDtype, y1Dtype, y2Dtype,
530 MicroAPI::Select<uint16_t>(reversedShareExp1, reversedShareExp1, nanBF16, infMask);530 MicroAPI::Select<uint16_t>(reversedShareExp1, reversedShareExp1, nanBF16, infMask);
531 MicroAPI::Select<uint16_t>(reversedShareExp1, reversedShareExp1, zero, zeroMask);531 MicroAPI::Select<uint16_t>(reversedShareExp1, reversedShareExp1, zero, zeroMask);
532 MicroAPI::Select<uint16_t>(reversedShareExp1, specialExp, reversedShareExp1, invalidDataMask);532 MicroAPI::Select<uint16_t>(reversedShareExp1, specialExp, reversedShareExp1, invalidDataMask);
533- MicroAPI::DataCopy<uint16_t>(mxScale1ReciprocalAddr + i * oneBlockCountB16_, reversedShareExp1,533+ MicroAPI::StoreAlign<uint16_t>(mxScale1ReciprocalAddr + i * oneBlockCountB16_, reversedShareExp1,
534- maskReduceB16);534+ maskReduceB16);
535 }535 }
536 // 计算-2轴的scale2和1/scale2 交织第一部分536 // 计算-2轴的scale2和1/scale2 交织第一部分
537 // inf/nan值单独处理，结果为E8M0的nan537 // inf/nan值单独处理，结果为E8M0的nan
@@ -583,10 +583,10 @@ __aicore__ inline void DynamicMxQuantWithDualAxisBase<xDtype, y1Dtype, y2Dtype,
583 MicroAPI::Select<uint16_t>(reversedShareExp2One, reversedShareExp2One, zero, zeroMask);583 MicroAPI::Select<uint16_t>(reversedShareExp2One, reversedShareExp2One, zero, zeroMask);
584 MicroAPI::Select<uint16_t>(reversedShareExp2One, specialExp, reversedShareExp2One, invalidDataMask);584 MicroAPI::Select<uint16_t>(reversedShareExp2One, specialExp, reversedShareExp2One, invalidDataMask);
585 // 交织搬出mxScale和1/scale585 // 交织搬出mxScale和1/scale
586- MicroAPI::DataCopy<uint8_t, MicroAPI::StoreDist::DIST_INTLV_B8>(mxScale2Addr, mxScale2ZeroB8, mxScale2OneB8,586+ MicroAPI::StoreAlign<uint8_t, MicroAPI::StoreDist::DIST_INTLV_B8>(mxScale2Addr, mxScale2ZeroB8, mxScale2OneB8,
587- maskB8);587+ maskB8);
588- MicroAPI::DataCopy<uint16_t, MicroAPI::StoreDist::DIST_INTLV_B16>(mxScale2ReciprocalAddr, reversedShareExp2Zero,588+ MicroAPI::StoreAlign<uint16_t, MicroAPI::StoreDist::DIST_INTLV_B16>(
589- reversedShareExp2One, maskAll);589+ mxScale2ReciprocalAddr, reversedShareExp2Zero, reversedShareExp2One, maskAll);
590 }590 }
591#endif591#endif
592}592}
@@ -684,7 +684,7 @@ DynamicMxQuantWithDualAxisBase<xDtype, y1Dtype, y2Dtype, roundMode, scaleAlg>::C
684 // ========================================================================684 // ========================================================================
685 for (uint16_t i = 0; i < blockCount; i++) {685 for (uint16_t i = 0; i < blockCount; i++) {
686 // 1. 交织搬运输入数据: 将256个xDtype按偶奇拆分为x0(偶), x1(奇)686 // 1. 交织搬运输入数据: 将256个xDtype按偶奇拆分为x0(偶), x1(奇)
687- MicroAPI::DataCopy<xDtype, MicroAPI::PostLiteral::POST_MODE_UPDATE, MicroAPI::LoadDist::DIST_DINTLV_B16>(687+ MicroAPI::LoadAlign<xDtype, MicroAPI::PostLiteral::POST_MODE_UPDATE, MicroAPI::LoadDist::DIST_DINTLV_B16>(
688 x0, x1, xAddr, vlForHalfNumber_ * DIGIT_TWO);688 x0, x1, xAddr, vlForHalfNumber_ * DIGIT_TWO);
689 689 
690 // 2. 取绝对值: 清除符号位，保留指数和尾数690 // 2. 取绝对值: 清除符号位，保留指数和尾数
@@ -693,7 +693,7 @@ DynamicMxQuantWithDualAxisBase<xDtype, y1Dtype, y2Dtype, roundMode, scaleAlg>::C
693 693 
694 // 3. -1轴: 先取偶奇max，再ReduceMaxWithDataBlock得到每32个元素的绝对值max694 // 3. -1轴: 先取偶奇max，再ReduceMaxWithDataBlock得到每32个元素的绝对值max
695 MicroAPI::Max(absMaxDim1, absMax0, absMax1, maskAll);695 MicroAPI::Max(absMaxDim1, absMax0, absMax1, maskAll);
696- MicroAPI::ReduceMaxWithDataBlock(absMaxDim1, absMaxDim1, maskAll);696+ MicroAPI::ReduceDataBlock<MicroAPI::ReduceType::MAX>(absMaxDim1, absMaxDim1, maskAll);
697 697 
698 // 4. -2轴: 逐行累积偶数列和奇数列的绝对值max698 // 4. -2轴: 逐行累积偶数列和奇数列的绝对值max
699 MicroAPI::Max(absMax1Dim2, absMax1Dim2, absMax0, maskAll);699 MicroAPI::Max(absMax1Dim2, absMax1Dim2, absMax0, maskAll);
@@ -718,13 +718,13 @@ DynamicMxQuantWithDualAxisBase<xDtype, y1Dtype, y2Dtype, roundMode, scaleAlg>::C
718 // 提取FP32尾数: 与尾数掩码718 // 提取FP32尾数: 与尾数掩码
719 MicroAPI::And(manFP32_0, maxFP32_0, manMaskFP32, maskFP32);719 MicroAPI::And(manFP32_0, maxFP32_0, manMaskFP32, maskFP32);
720 // 条件舍入: normal场景 (exp>0 && exp<254 && man>0) → exp+1720 // 条件舍入: normal场景 (exp>0 && exp<254 && man>0) → exp+1
721- MicroAPI::CompareScalar<uint32_t, CMPMODE::GT>(p0, expFP32_0, NUMBER_ZERO_U32, maskFP32);721+ MicroAPI::Compares<uint32_t, CMPMODE::GT>(p0, expFP32_0, NUMBER_ZERO_U32, maskFP32);
722- MicroAPI::CompareScalar<uint32_t, CMPMODE::LT>(p0, expFP32_0, NUMBER_TWO_FIVE_FOUR, p0);722+ MicroAPI::Compares<uint32_t, CMPMODE::LT>(p0, expFP32_0, NUMBER_TWO_FIVE_FOUR, p0);
723- MicroAPI::CompareScalar<uint32_t, CMPMODE::GT>(p0, manFP32_0, NUMBER_ZERO_U32, p0);723+ MicroAPI::Compares<uint32_t, CMPMODE::GT>(p0, manFP32_0, NUMBER_ZERO_U32, p0);
724 // 条件舍入: subnormal场景 (exp==0 && man>HALF) → exp+1724 // 条件舍入: subnormal场景 (exp==0 && man>HALF) → exp+1
725- MicroAPI::CompareScalar<uint32_t, CMPMODE::EQ>(p1, expFP32_0, NUMBER_ZERO_U32, maskFP32);725+ MicroAPI::Compares<uint32_t, CMPMODE::EQ>(p1, expFP32_0, NUMBER_ZERO_U32, maskFP32);
726- MicroAPI::CompareScalar<uint32_t, CMPMODE::GT>(p1, manFP32_0, NUMBER_HALF_U32, p1);726+ MicroAPI::Compares<uint32_t, CMPMODE::GT>(p1, manFP32_0, NUMBER_HALF_U32, p1);
727- MicroAPI::MaskOr(p0, p0, p1, maskFP32);727+ MicroAPI::Or(p0, p0, p1, maskFP32);
728 // 执行条件加1728 // 执行条件加1
729 MicroAPI::Adds(maxFP32_0, expFP32_0, 1, maskFP32);729 MicroAPI::Adds(maxFP32_0, expFP32_0, 1, maskFP32);
730 MicroAPI::Select(manFP32_0, maxFP32_0, expFP32_0, p0);730 MicroAPI::Select(manFP32_0, maxFP32_0, expFP32_0, p0);
@@ -736,7 +736,7 @@ DynamicMxQuantWithDualAxisBase<xDtype, y1Dtype, y2Dtype, roundMode, scaleAlg>::C
736 736 
737 // --- 输出-1轴scale (uint8) ---737 // --- 输出-1轴scale (uint8) ---
738 MicroAPI::Pack<uint8_t, uint16_t, MicroAPI::HighLowPart::LOWEST>(mxScale1B8, scale1B16_0);738 MicroAPI::Pack<uint8_t, uint16_t, MicroAPI::HighLowPart::LOWEST>(mxScale1B8, scale1B16_0);
739- MicroAPI::DataCopy<uint8_t>(mxScale1Addr + i * oneBlockCountB8_, mxScale1B8, maskReduceB8);739+ MicroAPI::StoreAlign<uint8_t>(mxScale1Addr + i * oneBlockCountB8_, mxScale1B8, maskReduceB8);
740 740 
741 // --- 计算并输出-1轴 1/scale (与原始DynamicMxQuant一致: inf→nan, special→specialExp, 无零值检查) ---741 // --- 计算并输出-1轴 1/scale (与原始DynamicMxQuant一致: inf→nan, special→specialExp, 无零值检查) ---
742 MicroAPI::Compare<uint16_t, CMPMODE::NE>(infMask, scale1BF16, maxEleBF16, maskAll);742 MicroAPI::Compare<uint16_t, CMPMODE::NE>(infMask, scale1BF16, maxEleBF16, maskAll);
@@ -744,8 +744,8 @@ DynamicMxQuantWithDualAxisBase<xDtype, y1Dtype, y2Dtype, roundMode, scaleAlg>::C
744 MicroAPI::Sub(reversedShareExp1, biasE8M0, scale1BF16, maskAll);744 MicroAPI::Sub(reversedShareExp1, biasE8M0, scale1BF16, maskAll);
745 MicroAPI::Select<uint16_t>(reversedShareExp1, reversedShareExp1, nanBF16, infMask);745 MicroAPI::Select<uint16_t>(reversedShareExp1, reversedShareExp1, nanBF16, infMask);
746 MicroAPI::Select<uint16_t>(reversedShareExp1, specialExp, reversedShareExp1, invalidDataMask);746 MicroAPI::Select<uint16_t>(reversedShareExp1, specialExp, reversedShareExp1, invalidDataMask);
747- MicroAPI::DataCopy<uint16_t>(mxScale1ReciprocalAddr + i * oneBlockCountB16_, reversedShareExp1,747+ MicroAPI::StoreAlign<uint16_t>(mxScale1ReciprocalAddr + i * oneBlockCountB16_, reversedShareExp1,
748- maskReduceB16);748+ maskReduceB16);
749 // 恢复zeroB16 (Interleave会修改dst1)749 // 恢复zeroB16 (Interleave会修改dst1)
750 MicroAPI::Duplicate(zeroB16, 0);750 MicroAPI::Duplicate(zeroB16, 0);
751 }751 }
@@ -765,15 +765,15 @@ DynamicMxQuantWithDualAxisBase<xDtype, y1Dtype, y2Dtype, roundMode, scaleAlg>::C
765 (MicroAPI::RegTensor<float>&)invMax, maskFP32);765 (MicroAPI::RegTensor<float>&)invMax, maskFP32);
766 MicroAPI::ShiftRights(expFP32_0, maxFP32_0, SHR_NUM_FOR_FP32, maskFP32);766 MicroAPI::ShiftRights(expFP32_0, maxFP32_0, SHR_NUM_FOR_FP32, maskFP32);
767 MicroAPI::And(manFP32_0, maxFP32_0, manMaskFP32, maskFP32);767 MicroAPI::And(manFP32_0, maxFP32_0, manMaskFP32, maskFP32);
768- MicroAPI::CompareScalar<uint32_t, CMPMODE::GT>(p0, expFP32_0, NUMBER_ZERO_U32, maskFP32);768+ MicroAPI::Compares<uint32_t, CMPMODE::GT>(p0, expFP32_0, NUMBER_ZERO_U32, maskFP32);
769- MicroAPI::CompareScalar<uint32_t, CMPMODE::LT>(p0, expFP32_0, NUMBER_TWO_FIVE_FOUR, p0);769+ MicroAPI::Compares<uint32_t, CMPMODE::LT>(p0, expFP32_0, NUMBER_TWO_FIVE_FOUR, p0);
770- MicroAPI::CompareScalar<uint32_t, CMPMODE::GT>(p0, manFP32_0, NUMBER_ZERO_U32, p0);770+ MicroAPI::Compares<uint32_t, CMPMODE::GT>(p0, manFP32_0, NUMBER_ZERO_U32, p0);
771- MicroAPI::CompareScalar<uint32_t, CMPMODE::EQ>(p1, expFP32_0, NUMBER_ZERO_U32, maskFP32);771+ MicroAPI::Compares<uint32_t, CMPMODE::EQ>(p1, expFP32_0, NUMBER_ZERO_U32, maskFP32);
772- MicroAPI::CompareScalar<uint32_t, CMPMODE::GT>(p1, manFP32_0, NUMBER_HALF_U32, p1);772+ MicroAPI::Compares<uint32_t, CMPMODE::GT>(p1, manFP32_0, NUMBER_HALF_U32, p1);
773- MicroAPI::MaskOr(p0, p0, p1, maskFP32);773+ MicroAPI::Or(p0, p0, p1, maskFP32);
774 // 链内复用: maxFP32_0→expPlusOne (maxFP32_0已死亡@And)774 // 链内复用: maxFP32_0→expPlusOne (maxFP32_0已死亡@And)
775 MicroAPI::Adds(maxFP32_0, expFP32_0, 1, maskFP32);775 MicroAPI::Adds(maxFP32_0, expFP32_0, 1, maskFP32);
776- // 链内复用: manFP32_0→extractExp (manFP32_0已死亡@CompareScalar)776+ // 链内复用: manFP32_0→extractExp (manFP32_0已死亡@Compares)
777 MicroAPI::Select(manFP32_0, maxFP32_0, expFP32_0, p0);777 MicroAPI::Select(manFP32_0, maxFP32_0, expFP32_0, p0);
778 MicroAPI::Pack<uint16_t, uint32_t, MicroAPI::HighLowPart::LOWEST>(scale1B16_0, manFP32_0);778 MicroAPI::Pack<uint16_t, uint32_t, MicroAPI::HighLowPart::LOWEST>(scale1B16_0, manFP32_0);
779 779 
@@ -784,12 +784,12 @@ DynamicMxQuantWithDualAxisBase<xDtype, y1Dtype, y2Dtype, roundMode, scaleAlg>::C
784 (MicroAPI::RegTensor<float>&)invMax, maskFP32);784 (MicroAPI::RegTensor<float>&)invMax, maskFP32);
785 MicroAPI::ShiftRights(expFP32_1, maxFP32_1, SHR_NUM_FOR_FP32, maskFP32);785 MicroAPI::ShiftRights(expFP32_1, maxFP32_1, SHR_NUM_FOR_FP32, maskFP32);
786 MicroAPI::And(manFP32_1, maxFP32_1, manMaskFP32, maskFP32);786 MicroAPI::And(manFP32_1, maxFP32_1, manMaskFP32, maskFP32);
787- MicroAPI::CompareScalar<uint32_t, CMPMODE::GT>(p0Odd, expFP32_1, NUMBER_ZERO_U32, maskFP32);787+ MicroAPI::Compares<uint32_t, CMPMODE::GT>(p0Odd, expFP32_1, NUMBER_ZERO_U32, maskFP32);
788- MicroAPI::CompareScalar<uint32_t, CMPMODE::LT>(p0Odd, expFP32_1, NUMBER_TWO_FIVE_FOUR, p0Odd);788+ MicroAPI::Compares<uint32_t, CMPMODE::LT>(p0Odd, expFP32_1, NUMBER_TWO_FIVE_FOUR, p0Odd);
789- MicroAPI::CompareScalar<uint32_t, CMPMODE::GT>(p0Odd, manFP32_1, NUMBER_ZERO_U32, p0Odd);789+ MicroAPI::Compares<uint32_t, CMPMODE::GT>(p0Odd, manFP32_1, NUMBER_ZERO_U32, p0Odd);
790- MicroAPI::CompareScalar<uint32_t, CMPMODE::EQ>(p1Odd, expFP32_1, NUMBER_ZERO_U32, maskFP32);790+ MicroAPI::Compares<uint32_t, CMPMODE::EQ>(p1Odd, expFP32_1, NUMBER_ZERO_U32, maskFP32);
791- MicroAPI::CompareScalar<uint32_t, CMPMODE::GT>(p1Odd, manFP32_1, NUMBER_HALF_U32, p1Odd);791+ MicroAPI::Compares<uint32_t, CMPMODE::GT>(p1Odd, manFP32_1, NUMBER_HALF_U32, p1Odd);
792- MicroAPI::MaskOr(p0Odd, p0Odd, p1Odd, maskFP32);792+ MicroAPI::Or(p0Odd, p0Odd, p1Odd, maskFP32);
793 // 链内复用: maxFP32_1→expPlusOne, manFP32_1→extractExp793 // 链内复用: maxFP32_1→expPlusOne, manFP32_1→extractExp
794 MicroAPI::Adds(maxFP32_1, expFP32_1, 1, maskFP32);794 MicroAPI::Adds(maxFP32_1, expFP32_1, 1, maskFP32);
795 MicroAPI::Select(manFP32_1, maxFP32_1, expFP32_1, p0Odd);795 MicroAPI::Select(manFP32_1, maxFP32_1, expFP32_1, p0Odd);
@@ -817,12 +817,12 @@ DynamicMxQuantWithDualAxisBase<xDtype, y1Dtype, y2Dtype, roundMode, scaleAlg>::C
817 (MicroAPI::RegTensor<float>&)invMax, maskFP32);817 (MicroAPI::RegTensor<float>&)invMax, maskFP32);
818 MicroAPI::ShiftRights(expFP32_0, maxFP32_0, SHR_NUM_FOR_FP32, maskFP32);818 MicroAPI::ShiftRights(expFP32_0, maxFP32_0, SHR_NUM_FOR_FP32, maskFP32);
819 MicroAPI::And(manFP32_0, maxFP32_0, manMaskFP32, maskFP32);819 MicroAPI::And(manFP32_0, maxFP32_0, manMaskFP32, maskFP32);
820- MicroAPI::CompareScalar<uint32_t, CMPMODE::GT>(p0, expFP32_0, NUMBER_ZERO_U32, maskFP32);820+ MicroAPI::Compares<uint32_t, CMPMODE::GT>(p0, expFP32_0, NUMBER_ZERO_U32, maskFP32);
821- MicroAPI::CompareScalar<uint32_t, CMPMODE::LT>(p0, expFP32_0, NUMBER_TWO_FIVE_FOUR, p0);821+ MicroAPI::Compares<uint32_t, CMPMODE::LT>(p0, expFP32_0, NUMBER_TWO_FIVE_FOUR, p0);
822- MicroAPI::CompareScalar<uint32_t, CMPMODE::GT>(p0, manFP32_0, NUMBER_ZERO_U32, p0);822+ MicroAPI::Compares<uint32_t, CMPMODE::GT>(p0, manFP32_0, NUMBER_ZERO_U32, p0);
823- MicroAPI::CompareScalar<uint32_t, CMPMODE::EQ>(p1, expFP32_0, NUMBER_ZERO_U32, maskFP32);823+ MicroAPI::Compares<uint32_t, CMPMODE::EQ>(p1, expFP32_0, NUMBER_ZERO_U32, maskFP32);
824- MicroAPI::CompareScalar<uint32_t, CMPMODE::GT>(p1, manFP32_0, NUMBER_HALF_U32, p1);824+ MicroAPI::Compares<uint32_t, CMPMODE::GT>(p1, manFP32_0, NUMBER_HALF_U32, p1);
825- MicroAPI::MaskOr(p0, p0, p1, maskFP32);825+ MicroAPI::Or(p0, p0, p1, maskFP32);
826 // 链内复用: maxFP32_0→expPlusOne, manFP32_0→extractExp826 // 链内复用: maxFP32_0→expPlusOne, manFP32_0→extractExp
827 MicroAPI::Adds(maxFP32_0, expFP32_0, 1, maskFP32);827 MicroAPI::Adds(maxFP32_0, expFP32_0, 1, maskFP32);
828 MicroAPI::Select(manFP32_0, maxFP32_0, expFP32_0, p0);828 MicroAPI::Select(manFP32_0, maxFP32_0, expFP32_0, p0);
@@ -835,12 +835,12 @@ DynamicMxQuantWithDualAxisBase<xDtype, y1Dtype, y2Dtype, roundMode, scaleAlg>::C
835 (MicroAPI::RegTensor<float>&)invMax, maskFP32);835 (MicroAPI::RegTensor<float>&)invMax, maskFP32);
836 MicroAPI::ShiftRights(expFP32_1, maxFP32_1, SHR_NUM_FOR_FP32, maskFP32);836 MicroAPI::ShiftRights(expFP32_1, maxFP32_1, SHR_NUM_FOR_FP32, maskFP32);
837 MicroAPI::And(manFP32_1, maxFP32_1, manMaskFP32, maskFP32);837 MicroAPI::And(manFP32_1, maxFP32_1, manMaskFP32, maskFP32);
838- MicroAPI::CompareScalar<uint32_t, CMPMODE::GT>(p0Odd, expFP32_1, NUMBER_ZERO_U32, maskFP32);838+ MicroAPI::Compares<uint32_t, CMPMODE::GT>(p0Odd, expFP32_1, NUMBER_ZERO_U32, maskFP32);
839- MicroAPI::CompareScalar<uint32_t, CMPMODE::LT>(p0Odd, expFP32_1, NUMBER_TWO_FIVE_FOUR, p0Odd);839+ MicroAPI::Compares<uint32_t, CMPMODE::LT>(p0Odd, expFP32_1, NUMBER_TWO_FIVE_FOUR, p0Odd);
840- MicroAPI::CompareScalar<uint32_t, CMPMODE::GT>(p0Odd, manFP32_1, NUMBER_ZERO_U32, p0Odd);840+ MicroAPI::Compares<uint32_t, CMPMODE::GT>(p0Odd, manFP32_1, NUMBER_ZERO_U32, p0Odd);
841- MicroAPI::CompareScalar<uint32_t, CMPMODE::EQ>(p1Odd, expFP32_1, NUMBER_ZERO_U32, maskFP32);841+ MicroAPI::Compares<uint32_t, CMPMODE::EQ>(p1Odd, expFP32_1, NUMBER_ZERO_U32, maskFP32);
842- MicroAPI::CompareScalar<uint32_t, CMPMODE::GT>(p1Odd, manFP32_1, NUMBER_HALF_U32, p1Odd);842+ MicroAPI::Compares<uint32_t, CMPMODE::GT>(p1Odd, manFP32_1, NUMBER_HALF_U32, p1Odd);
843- MicroAPI::MaskOr(p0Odd, p0Odd, p1Odd, maskFP32);843+ MicroAPI::Or(p0Odd, p0Odd, p1Odd, maskFP32);
844 // 链内复用: maxFP32_1→expPlusOne, manFP32_1→extractExp844 // 链内复用: maxFP32_1→expPlusOne, manFP32_1→extractExp
845 MicroAPI::Adds(maxFP32_1, expFP32_1, 1, maskFP32);845 MicroAPI::Adds(maxFP32_1, expFP32_1, 1, maskFP32);
846 MicroAPI::Select(manFP32_1, maxFP32_1, expFP32_1, p0Odd);846 MicroAPI::Select(manFP32_1, maxFP32_1, expFP32_1, p0Odd);
@@ -860,10 +860,10 @@ DynamicMxQuantWithDualAxisBase<xDtype, y1Dtype, y2Dtype, roundMode, scaleAlg>::C
860 MicroAPI::Select<uint16_t>(absMax0, specialExp, absMax0, invalidDataMask);860 MicroAPI::Select<uint16_t>(absMax0, specialExp, absMax0, invalidDataMask);
861 861 
862 // 交织搬出-2轴的mxScale和1/scale862 // 交织搬出-2轴的mxScale和1/scale
863- MicroAPI::DataCopy<uint8_t, MicroAPI::StoreDist::DIST_INTLV_B8>(mxScale2Addr, mxScale1B8, mxScale2OneB8,863+ MicroAPI::StoreAlign<uint8_t, MicroAPI::StoreDist::DIST_INTLV_B8>(mxScale2Addr, mxScale1B8, mxScale2OneB8,
864- maskB8);864+ maskB8);
865- MicroAPI::DataCopy<uint16_t, MicroAPI::StoreDist::DIST_INTLV_B16>(mxScale2ReciprocalAddr, reversedShareExp1,865+ MicroAPI::StoreAlign<uint16_t, MicroAPI::StoreDist::DIST_INTLV_B16>(mxScale2ReciprocalAddr, reversedShareExp1,
866- absMax0, maskAll);866+ absMax0, maskAll);
867 }867 }
868#endif868#endif
869}869}
@@ -950,7 +950,7 @@ DynamicMxQuantWithDualAxisBase<xDtype, y1Dtype, y2Dtype, roundMode, scaleAlg>::C
950 // ========================================================================950 // ========================================================================
951 for (uint16_t i = 0; i < blockCount; i++) {951 for (uint16_t i = 0; i < blockCount; i++) {
952 // 1. 交织搬运输入数据: 将256个xDtype按偶奇拆分为x0(偶), x1(奇)952 // 1. 交织搬运输入数据: 将256个xDtype按偶奇拆分为x0(偶), x1(奇)
953- MicroAPI::DataCopy<xDtype, MicroAPI::PostLiteral::POST_MODE_UPDATE, MicroAPI::LoadDist::DIST_DINTLV_B16>(953+ MicroAPI::LoadAlign<xDtype, MicroAPI::PostLiteral::POST_MODE_UPDATE, MicroAPI::LoadDist::DIST_DINTLV_B16>(
954 x0, x1, xAddr, vlForHalfNumber_ * DIGIT_TWO);954 x0, x1, xAddr, vlForHalfNumber_ * DIGIT_TWO);
955 955 
956 // 2. 获取BF16绝对值 (区分half和bf16输入)956 // 2. 获取BF16绝对值 (区分half和bf16输入)
@@ -977,7 +977,7 @@ DynamicMxQuantWithDualAxisBase<xDtype, y1Dtype, y2Dtype, roundMode, scaleAlg>::C
977 977 
978 // 3. -1轴: 偶奇Max + ReduceMaxWithDataBlock，得到每32个元素的绝对值max978 // 3. -1轴: 偶奇Max + ReduceMaxWithDataBlock，得到每32个元素的绝对值max
979 MicroAPI::Max(absMaxDim1, absVal0, absVal1, maskAll);979 MicroAPI::Max(absMaxDim1, absVal0, absVal1, maskAll);
980- MicroAPI::ReduceMaxWithDataBlock(absMaxDim1, absMaxDim1, maskAll);980+ MicroAPI::ReduceDataBlock<MicroAPI::ReduceType::MAX>(absMaxDim1, absMaxDim1, maskAll);
981 981 
982 // 4. -2轴: 逐行累积偶数列和奇数列的BF16绝对值max982 // 4. -2轴: 逐行累积偶数列和奇数列的BF16绝对值max
983 MicroAPI::Max(absMax1Dim2, absMax1Dim2, absVal0, maskAll);983 MicroAPI::Max(absMax1Dim2, absMax1Dim2, absVal0, maskAll);
@@ -1016,8 +1016,8 @@ DynamicMxQuantWithDualAxisBase<xDtype, y1Dtype, y2Dtype, roundMode, scaleAlg>::C
1016 1016 
1017 // 输出-1轴scale (uint8)1017 // 输出-1轴scale (uint8)
1018 MicroAPI::Pack<uint8_t, uint16_t, MicroAPI::HighLowPart::LOWEST>(mxScale1B8, scaleValue);1018 MicroAPI::Pack<uint8_t, uint16_t, MicroAPI::HighLowPart::LOWEST>(mxScale1B8, scaleValue);
1019- MicroAPI::DataCopy<uint8_t, MicroAPI::PostLiteral::POST_MODE_UPDATE>(mxScale1Addr, mxScale1B8,1019+ MicroAPI::StoreAlign<uint8_t, MicroAPI::PostLiteral::POST_MODE_UPDATE>(mxScale1Addr, mxScale1B8,
1020- oneBlockCountB8_, maskReduceB8);1020+ oneBlockCountB8_, maskReduceB8);
1021 1021 
1022 // 计算-1轴1/scale1022 // 计算-1轴1/scale
1023 // sharedExp是左移7位前的指数差值，可直接用于BF16域1/scale计算1023 // sharedExp是左移7位前的指数差值，可直接用于BF16域1/scale计算
@@ -1026,7 +1026,7 @@ DynamicMxQuantWithDualAxisBase<xDtype, y1Dtype, y2Dtype, roundMode, scaleAlg>::C
1026 MicroAPI::Select<uint16_t>(reversedShareExp1, reversedShareExp1, nanBF16, infMask);1026 MicroAPI::Select<uint16_t>(reversedShareExp1, reversedShareExp1, nanBF16, infMask);
1027 MicroAPI::Select<uint16_t>(reversedShareExp1, reversedShareExp1, zero, zeroMask);1027 MicroAPI::Select<uint16_t>(reversedShareExp1, reversedShareExp1, zero, zeroMask);
1028 MicroAPI::Select<uint16_t>(reversedShareExp1, specialExp, reversedShareExp1, invalidDataMask);1028 MicroAPI::Select<uint16_t>(reversedShareExp1, specialExp, reversedShareExp1, invalidDataMask);
1029- MicroAPI::DataCopy<uint16_t, MicroAPI::PostLiteral::POST_MODE_UPDATE>(1029+ MicroAPI::StoreAlign<uint16_t, MicroAPI::PostLiteral::POST_MODE_UPDATE>(
1030 mxScale1ReciprocalAddr, reversedShareExp1, oneBlockCountB16_, maskReduceB16);1030 mxScale1ReciprocalAddr, reversedShareExp1, oneBlockCountB16_, maskReduceB16);
1031 }1031 }
1032 1032 
@@ -1096,10 +1096,10 @@ DynamicMxQuantWithDualAxisBase<xDtype, y1Dtype, y2Dtype, roundMode, scaleAlg>::C
1096 MicroAPI::Select<uint16_t>(absVal0, specialExp, absVal0, invalidDataMask);1096 MicroAPI::Select<uint16_t>(absVal0, specialExp, absVal0, invalidDataMask);
1097 1097 
1098 // 交织搬出-2轴的mxScale和1/scale1098 // 交织搬出-2轴的mxScale和1/scale
1099- MicroAPI::DataCopy<uint8_t, MicroAPI::StoreDist::DIST_INTLV_B8>(mxScale2Addr, mxScale1B8, mxScale2OneB8,1099+ MicroAPI::StoreAlign<uint8_t, MicroAPI::StoreDist::DIST_INTLV_B8>(mxScale2Addr, mxScale1B8, mxScale2OneB8,
1100- maskB8);1100+ maskB8);
1101- MicroAPI::DataCopy<uint16_t, MicroAPI::StoreDist::DIST_INTLV_B16>(mxScale2ReciprocalAddr, reversedShareExp1,1101+ MicroAPI::StoreAlign<uint16_t, MicroAPI::StoreDist::DIST_INTLV_B16>(mxScale2ReciprocalAddr, reversedShareExp1,
1102- absVal0, maskAll);1102+ absVal0, maskAll);
1103 }1103 }
1104}1104}
1105 1105 
@@ -1192,7 +1192,7 @@ DynamicMxQuantWithDualAxisBase<xDtype, y1Dtype, y2Dtype, roundMode, scaleAlg>::C
1192 // ========================================================================1192 // ========================================================================
1193 for (uint16_t i = 0; i < blockCount; i++) {1193 for (uint16_t i = 0; i < blockCount; i++) {
1194 // 1. 交织搬运输入数据: 将256个xDtype按偶奇拆分为x0(偶), x1(奇)1194 // 1. 交织搬运输入数据: 将256个xDtype按偶奇拆分为x0(偶), x1(奇)
1195- MicroAPI::DataCopy<xDtype, MicroAPI::PostLiteral::POST_MODE_UPDATE, MicroAPI::LoadDist::DIST_DINTLV_B16>(1195+ MicroAPI::LoadAlign<xDtype, MicroAPI::PostLiteral::POST_MODE_UPDATE, MicroAPI::LoadDist::DIST_DINTLV_B16>(
1196 x0, x1, xAddr, vlForHalfNumber_ * DIGIT_TWO);1196 x0, x1, xAddr, vlForHalfNumber_ * DIGIT_TWO);
1197 1197 
1198 // 2. 取绝对值: 清除符号位，保留指数和尾数1198 // 2. 取绝对值: 清除符号位，保留指数和尾数
@@ -1201,7 +1201,7 @@ DynamicMxQuantWithDualAxisBase<xDtype, y1Dtype, y2Dtype, roundMode, scaleAlg>::C
1201 1201 
1202 // 3. -1轴: 先取偶奇max，再ReduceMaxWithDataBlock得到每32个元素的绝对值max1202 // 3. -1轴: 先取偶奇max，再ReduceMaxWithDataBlock得到每32个元素的绝对值max
1203 MicroAPI::Max(absMaxDim1, absMax0, absMax1, maskAll);1203 MicroAPI::Max(absMaxDim1, absMax0, absMax1, maskAll);
1204- MicroAPI::ReduceMaxWithDataBlock(absMaxDim1, absMaxDim1, maskAll);1204+ MicroAPI::ReduceDataBlock<MicroAPI::ReduceType::MAX>(absMaxDim1, absMaxDim1, maskAll);
1205 1205 
1206 // 4. -2轴: 逐行累积偶数列和奇数列的绝对值max1206 // 4. -2轴: 逐行累积偶数列和奇数列的绝对值max
1207 MicroAPI::Max(absMax1Dim2, absMax1Dim2, absMax0, maskAll);1207 MicroAPI::Max(absMax1Dim2, absMax1Dim2, absMax0, maskAll);
@@ -1229,9 +1229,9 @@ DynamicMxQuantWithDualAxisBase<xDtype, y1Dtype, y2Dtype, roundMode, scaleAlg>::C
1229 MicroAPI::And(manFP32_0, maxFP32_0, manMaskFP32, maskFP32);1229 MicroAPI::And(manFP32_0, maxFP32_0, manMaskFP32, maskFP32);
1230 // 条件舍入: 仅normal场景 (exp>0 && exp<254 && man>0) → exp+11230 // 条件舍入: 仅normal场景 (exp>0 && exp<254 && man>0) → exp+1
1231 // 注意: 与CuBALS不同，DynamicDtypeRange Custom不处理subnormal场景1231 // 注意: 与CuBALS不同，DynamicDtypeRange Custom不处理subnormal场景
1232- MicroAPI::CompareScalar<uint32_t, CMPMODE::GT>(p0, expFP32_0, NUMBER_ZERO_U32, maskFP32);1232+ MicroAPI::Compares<uint32_t, CMPMODE::GT>(p0, expFP32_0, NUMBER_ZERO_U32, maskFP32);
1233- MicroAPI::CompareScalar<uint32_t, CMPMODE::LT>(p0, expFP32_0, NUMBER_TWO_FIVE_FOUR, p0);1233+ MicroAPI::Compares<uint32_t, CMPMODE::LT>(p0, expFP32_0, NUMBER_TWO_FIVE_FOUR, p0);
1234- MicroAPI::CompareScalar<uint32_t, CMPMODE::GT>(p0, manFP32_0, NUMBER_ZERO_U32, p0);1234+ MicroAPI::Compares<uint32_t, CMPMODE::GT>(p0, manFP32_0, NUMBER_ZERO_U32, p0);
1235 // 执行条件加11235 // 执行条件加1
1236 MicroAPI::Adds(maxFP32_0, expFP32_0, 1, maskFP32);1236 MicroAPI::Adds(maxFP32_0, expFP32_0, 1, maskFP32);
1237 MicroAPI::Select(manFP32_0, maxFP32_0, expFP32_0, p0);1237 MicroAPI::Select(manFP32_0, maxFP32_0, expFP32_0, p0);
@@ -1243,8 +1243,8 @@ DynamicMxQuantWithDualAxisBase<xDtype, y1Dtype, y2Dtype, roundMode, scaleAlg>::C
1243 1243 
1244 // --- 输出-1轴scale (uint8) ---1244 // --- 输出-1轴scale (uint8) ---
1245 MicroAPI::Pack<uint8_t, uint16_t, MicroAPI::HighLowPart::LOWEST>(mxScale1B8, scale1B16_0);1245 MicroAPI::Pack<uint8_t, uint16_t, MicroAPI::HighLowPart::LOWEST>(mxScale1B8, scale1B16_0);
1246- MicroAPI::DataCopy<uint8_t, MicroAPI::PostLiteral::POST_MODE_UPDATE>(mxScale1Addr, mxScale1B8,1246+ MicroAPI::StoreAlign<uint8_t, MicroAPI::PostLiteral::POST_MODE_UPDATE>(mxScale1Addr, mxScale1B8,
1247- oneBlockCountB8_, maskReduceB8);1247+ oneBlockCountB8_, maskReduceB8);
1248 1248 
1249 // --- 计算并输出-1轴 1/scale (与原始DynamicMxQuant一致: inf→nan, special→specialExp, 无零值检查) ---1249 // --- 计算并输出-1轴 1/scale (与原始DynamicMxQuant一致: inf→nan, special→specialExp, 无零值检查) ---
1250 MicroAPI::Compare<uint16_t, CMPMODE::NE>(infMask, scale1BF16, maxEleBF16, maskAll);1250 MicroAPI::Compare<uint16_t, CMPMODE::NE>(infMask, scale1BF16, maxEleBF16, maskAll);
@@ -1252,7 +1252,7 @@ DynamicMxQuantWithDualAxisBase<xDtype, y1Dtype, y2Dtype, roundMode, scaleAlg>::C
1252 MicroAPI::Sub(reversedShareExp1, biasE8M0, scale1BF16, maskAll);1252 MicroAPI::Sub(reversedShareExp1, biasE8M0, scale1BF16, maskAll);
1253 MicroAPI::Select<uint16_t>(reversedShareExp1, reversedShareExp1, nanBF16, infMask);1253 MicroAPI::Select<uint16_t>(reversedShareExp1, reversedShareExp1, nanBF16, infMask);
1254 MicroAPI::Select<uint16_t>(reversedShareExp1, specialExp, reversedShareExp1, invalidDataMask);1254 MicroAPI::Select<uint16_t>(reversedShareExp1, specialExp, reversedShareExp1, invalidDataMask);
1255- MicroAPI::DataCopy<uint16_t, MicroAPI::PostLiteral::POST_MODE_UPDATE>(1255+ MicroAPI::StoreAlign<uint16_t, MicroAPI::PostLiteral::POST_MODE_UPDATE>(
1256 mxScale1ReciprocalAddr, reversedShareExp1, oneBlockCountB16_, maskReduceB16);1256 mxScale1ReciprocalAddr, reversedShareExp1, oneBlockCountB16_, maskReduceB16);
1257 // 恢复zeroB16 (Interleave会修改dst1)1257 // 恢复zeroB16 (Interleave会修改dst1)
1258 MicroAPI::Duplicate(zeroB16, 0);1258 MicroAPI::Duplicate(zeroB16, 0);
@@ -1274,9 +1274,9 @@ DynamicMxQuantWithDualAxisBase<xDtype, y1Dtype, y2Dtype, roundMode, scaleAlg>::C
1274 MicroAPI::ShiftRights(expFP32_0, maxFP32_0, SHR_NUM_FOR_FP32, maskFP32);1274 MicroAPI::ShiftRights(expFP32_0, maxFP32_0, SHR_NUM_FOR_FP32, maskFP32);
1275 MicroAPI::And(manFP32_0, maxFP32_0, manMaskFP32, maskFP32);1275 MicroAPI::And(manFP32_0, maxFP32_0, manMaskFP32, maskFP32);
1276 // 条件舍入: 仅normal场景 (无subnormal)1276 // 条件舍入: 仅normal场景 (无subnormal)
1277- MicroAPI::CompareScalar<uint32_t, CMPMODE::GT>(p0, expFP32_0, NUMBER_ZERO_U32, maskFP32);1277+ MicroAPI::Compares<uint32_t, CMPMODE::GT>(p0, expFP32_0, NUMBER_ZERO_U32, maskFP32);
1278- MicroAPI::CompareScalar<uint32_t, CMPMODE::LT>(p0, expFP32_0, NUMBER_TWO_FIVE_FOUR, p0);1278+ MicroAPI::Compares<uint32_t, CMPMODE::LT>(p0, expFP32_0, NUMBER_TWO_FIVE_FOUR, p0);
1279- MicroAPI::CompareScalar<uint32_t, CMPMODE::GT>(p0, manFP32_0, NUMBER_ZERO_U32, p0);1279+ MicroAPI::Compares<uint32_t, CMPMODE::GT>(p0, manFP32_0, NUMBER_ZERO_U32, p0);
1280 // 链内复用: maxFP32_0→expPlusOne, expFP32_0同时作为exp和最终结果1280 // 链内复用: maxFP32_0→expPlusOne, expFP32_0同时作为exp和最终结果
1281 MicroAPI::Adds(maxFP32_0, expFP32_0, 1, maskFP32);1281 MicroAPI::Adds(maxFP32_0, expFP32_0, 1, maskFP32);
1282 MicroAPI::Select(expFP32_0, maxFP32_0, expFP32_0, p0);1282 MicroAPI::Select(expFP32_0, maxFP32_0, expFP32_0, p0);
@@ -1289,9 +1289,9 @@ DynamicMxQuantWithDualAxisBase<xDtype, y1Dtype, y2Dtype, roundMode, scaleAlg>::C
1289 maskFP32);1289 maskFP32);
1290 MicroAPI::ShiftRights(expFP32_1, maxFP32_1, SHR_NUM_FOR_FP32, maskFP32);1290 MicroAPI::ShiftRights(expFP32_1, maxFP32_1, SHR_NUM_FOR_FP32, maskFP32);
1291 MicroAPI::And(manFP32_1, maxFP32_1, manMaskFP32, maskFP32);1291 MicroAPI::And(manFP32_1, maxFP32_1, manMaskFP32, maskFP32);
1292- MicroAPI::CompareScalar<uint32_t, CMPMODE::GT>(p0, expFP32_1, NUMBER_ZERO_U32, maskFP32);1292+ MicroAPI::Compares<uint32_t, CMPMODE::GT>(p0, expFP32_1, NUMBER_ZERO_U32, maskFP32);
1293- MicroAPI::CompareScalar<uint32_t, CMPMODE::LT>(p0, expFP32_1, NUMBER_TWO_FIVE_FOUR, p0);1293+ MicroAPI::Compares<uint32_t, CMPMODE::LT>(p0, expFP32_1, NUMBER_TWO_FIVE_FOUR, p0);
1294- MicroAPI::CompareScalar<uint32_t, CMPMODE::GT>(p0, manFP32_1, NUMBER_ZERO_U32, p0);1294+ MicroAPI::Compares<uint32_t, CMPMODE::GT>(p0, manFP32_1, NUMBER_ZERO_U32, p0);
1295 MicroAPI::Adds(maxFP32_1, expFP32_1, 1, maskFP32);1295 MicroAPI::Adds(maxFP32_1, expFP32_1, 1, maskFP32);
1296 MicroAPI::Select(expFP32_1, maxFP32_1, expFP32_1, p0);1296 MicroAPI::Select(expFP32_1, maxFP32_1, expFP32_1, p0);
1297 MicroAPI::Pack<uint16_t, uint32_t, MicroAPI::HighLowPart::LOWEST>(scale1B16_1, expFP32_1);1297 MicroAPI::Pack<uint16_t, uint32_t, MicroAPI::HighLowPart::LOWEST>(scale1B16_1, expFP32_1);
@@ -1318,9 +1318,9 @@ DynamicMxQuantWithDualAxisBase<xDtype, y1Dtype, y2Dtype, roundMode, scaleAlg>::C
1318 maskFP32);1318 maskFP32);
1319 MicroAPI::ShiftRights(expFP32_0, maxFP32_0, SHR_NUM_FOR_FP32, maskFP32);1319 MicroAPI::ShiftRights(expFP32_0, maxFP32_0, SHR_NUM_FOR_FP32, maskFP32);
1320 MicroAPI::And(manFP32_0, maxFP32_0, manMaskFP32, maskFP32);1320 MicroAPI::And(manFP32_0, maxFP32_0, manMaskFP32, maskFP32);
1321- MicroAPI::CompareScalar<uint32_t, CMPMODE::GT>(p0, expFP32_0, NUMBER_ZERO_U32, maskFP32);1321+ MicroAPI::Compares<uint32_t, CMPMODE::GT>(p0, expFP32_0, NUMBER_ZERO_U32, maskFP32);
1322- MicroAPI::CompareScalar<uint32_t, CMPMODE::LT>(p0, expFP32_0, NUMBER_TWO_FIVE_FOUR, p0);1322+ MicroAPI::Compares<uint32_t, CMPMODE::LT>(p0, expFP32_0, NUMBER_TWO_FIVE_FOUR, p0);
1323- MicroAPI::CompareScalar<uint32_t, CMPMODE::GT>(p0, manFP32_0, NUMBER_ZERO_U32, p0);1323+ MicroAPI::Compares<uint32_t, CMPMODE::GT>(p0, manFP32_0, NUMBER_ZERO_U32, p0);
1324 MicroAPI::Adds(maxFP32_0, expFP32_0, 1, maskFP32);1324 MicroAPI::Adds(maxFP32_0, expFP32_0, 1, maskFP32);
1325 MicroAPI::Select(expFP32_0, maxFP32_0, expFP32_0, p0);1325 MicroAPI::Select(expFP32_0, maxFP32_0, expFP32_0, p0);
1326 MicroAPI::Pack<uint16_t, uint32_t, MicroAPI::HighLowPart::LOWEST>(scale1B16_0, expFP32_0);1326 MicroAPI::Pack<uint16_t, uint32_t, MicroAPI::HighLowPart::LOWEST>(scale1B16_0, expFP32_0);
@@ -1332,9 +1332,9 @@ DynamicMxQuantWithDualAxisBase<xDtype, y1Dtype, y2Dtype, roundMode, scaleAlg>::C
1332 maskFP32);1332 maskFP32);
1333 MicroAPI::ShiftRights(expFP32_1, maxFP32_1, SHR_NUM_FOR_FP32, maskFP32);1333 MicroAPI::ShiftRights(expFP32_1, maxFP32_1, SHR_NUM_FOR_FP32, maskFP32);
1334 MicroAPI::And(manFP32_1, maxFP32_1, manMaskFP32, maskFP32);1334 MicroAPI::And(manFP32_1, maxFP32_1, manMaskFP32, maskFP32);
1335- MicroAPI::CompareScalar<uint32_t, CMPMODE::GT>(p0, expFP32_1, NUMBER_ZERO_U32, maskFP32);1335+ MicroAPI::Compares<uint32_t, CMPMODE::GT>(p0, expFP32_1, NUMBER_ZERO_U32, maskFP32);
1336- MicroAPI::CompareScalar<uint32_t, CMPMODE::LT>(p0, expFP32_1, NUMBER_TWO_FIVE_FOUR, p0);1336+ MicroAPI::Compares<uint32_t, CMPMODE::LT>(p0, expFP32_1, NUMBER_TWO_FIVE_FOUR, p0);
1337- MicroAPI::CompareScalar<uint32_t, CMPMODE::GT>(p0, manFP32_1, NUMBER_ZERO_U32, p0);1337+ MicroAPI::Compares<uint32_t, CMPMODE::GT>(p0, manFP32_1, NUMBER_ZERO_U32, p0);
1338 MicroAPI::Adds(maxFP32_1, expFP32_1, 1, maskFP32);1338 MicroAPI::Adds(maxFP32_1, expFP32_1, 1, maskFP32);
1339 MicroAPI::Select(expFP32_1, maxFP32_1, expFP32_1, p0);1339 MicroAPI::Select(expFP32_1, maxFP32_1, expFP32_1, p0);
1340 MicroAPI::Pack<uint16_t, uint32_t, MicroAPI::HighLowPart::LOWEST>(scale1B16_1, expFP32_1);1340 MicroAPI::Pack<uint16_t, uint32_t, MicroAPI::HighLowPart::LOWEST>(scale1B16_1, expFP32_1);
@@ -1354,10 +1354,10 @@ DynamicMxQuantWithDualAxisBase<xDtype, y1Dtype, y2Dtype, roundMode, scaleAlg>::C
1354 MicroAPI::Select<uint16_t>(absMax0, specialExp, absMax0, invalidDataMask);1354 MicroAPI::Select<uint16_t>(absMax0, specialExp, absMax0, invalidDataMask);
1355 1355 
1356 // 交织搬出-2轴的mxScale和1/scale1356 // 交织搬出-2轴的mxScale和1/scale
1357- MicroAPI::DataCopy<uint8_t, MicroAPI::StoreDist::DIST_INTLV_B8>(mxScale2Addr, mxScale1B8, mxScale2OneB8,1357+ MicroAPI::StoreAlign<uint8_t, MicroAPI::StoreDist::DIST_INTLV_B8>(mxScale2Addr, mxScale1B8, mxScale2OneB8,
1358- maskB8);1358+ maskB8);
1359- MicroAPI::DataCopy<uint16_t, MicroAPI::StoreDist::DIST_INTLV_B16>(mxScale2ReciprocalAddr, reversedShareExp1,1359+ MicroAPI::StoreAlign<uint16_t, MicroAPI::StoreDist::DIST_INTLV_B16>(mxScale2ReciprocalAddr, reversedShareExp1,
1360- absMax0, maskAll);1360+ absMax0, maskAll);
1361 }1361 }
1362}1362}
1363 1363 
@@ -1407,13 +1407,13 @@ __aicore__ inline void DynamicMxQuantWithDualAxisBase<xDtype, y1Dtype, y2Dtype,
1407 MicroAPI::RegTensor<y1Dtype> dim1x0FP4;1407 MicroAPI::RegTensor<y1Dtype> dim1x0FP4;
1408 MicroAPI::RegTensor<y1Dtype> dim1x1FP4;1408 MicroAPI::RegTensor<y1Dtype> dim1x1FP4;
1409 1409 
1410- MicroAPI::DataCopy<uint16_t, MicroAPI::PostLiteral::POST_MODE_UPDATE, MicroAPI::LoadDist::DIST_DINTLV_B16>(1410+ MicroAPI::LoadAlign<uint16_t, MicroAPI::PostLiteral::POST_MODE_UPDATE, MicroAPI::LoadDist::DIST_DINTLV_B16>(
1411 reversedShareExp0, reversedShareExp1, mxScale2ReciprocalAddr, vlForHalfNumber_ * DIGIT_TWO);1411 reversedShareExp0, reversedShareExp1, mxScale2ReciprocalAddr, vlForHalfNumber_ * DIGIT_TWO);
1412 1412 
1413 for (uint16_t i = 0; i < blockCount; i++) {1413 for (uint16_t i = 0; i < blockCount; i++) {
1414- MicroAPI::DataCopy<xDtype, MicroAPI::PostLiteral::POST_MODE_UPDATE, MicroAPI::LoadDist::DIST_DINTLV_B16>(1414+ MicroAPI::LoadAlign<xDtype, MicroAPI::PostLiteral::POST_MODE_UPDATE, MicroAPI::LoadDist::DIST_DINTLV_B16>(
1415 x0, x1, xAddr, vlForHalfNumber_ * DIGIT_TWO);1415 x0, x1, xAddr, vlForHalfNumber_ * DIGIT_TWO);
1416- MicroAPI::DataCopy<uint16_t, MicroAPI::PostLiteral::POST_MODE_UPDATE, MicroAPI::LoadDist::DIST_E2B_B16>(1416+ MicroAPI::LoadAlign<uint16_t, MicroAPI::PostLiteral::POST_MODE_UPDATE, MicroAPI::LoadDist::DIST_E2B_B16>(
1417 scaleForMulFP16, mxScale1ReciprocalAddr, oneBlockCountB16_);1417 scaleForMulFP16, mxScale1ReciprocalAddr, oneBlockCountB16_);
1418 1418 
1419 MicroAPI::Mul(dim0x0, x0, (MicroAPI::RegTensor<xDtype>&)reversedShareExp0, dataMaskB16);1419 MicroAPI::Mul(dim0x0, x0, (MicroAPI::RegTensor<xDtype>&)reversedShareExp0, dataMaskB16);
@@ -1428,14 +1428,14 @@ __aicore__ inline void DynamicMxQuantWithDualAxisBase<xDtype, y1Dtype, y2Dtype,
1428 MicroAPI::Cast<y1Dtype, xDtype, castTraitBF16toFp4>(dim1x1FP4, dim1x1, dataMaskB16);1428 MicroAPI::Cast<y1Dtype, xDtype, castTraitBF16toFp4>(dim1x1FP4, dim1x1, dataMaskB16);
1429 1429 
1430 // copy to ub1430 // copy to ub
1431- MicroAPI::DataCopy<uint8_t, MicroAPI::StoreDist::DIST_PACK4_B32>(1431+ MicroAPI::StoreAlign<uint8_t, MicroAPI::StoreDist::DIST_PACK4_B32>(
1432 y2Addr + (i * ubRowLen_ / DIGIT_TWO), (MicroAPI::RegTensor<uint8_t>&)dim0x0FP4, dataMaskB8);1432 y2Addr + (i * ubRowLen_ / DIGIT_TWO), (MicroAPI::RegTensor<uint8_t>&)dim0x0FP4, dataMaskB8);
1433- MicroAPI::DataCopy<uint8_t, MicroAPI::StoreDist::DIST_PACK4_B32>(1433+ MicroAPI::StoreAlign<uint8_t, MicroAPI::StoreDist::DIST_PACK4_B32>(
1434 y2Addr + OUT_ELE_NUM_ONE_BLK + (i * ubRowLen_ / DIGIT_TWO), (MicroAPI::RegTensor<uint8_t>&)dim0x1FP4,1434 y2Addr + OUT_ELE_NUM_ONE_BLK + (i * ubRowLen_ / DIGIT_TWO), (MicroAPI::RegTensor<uint8_t>&)dim0x1FP4,
1435 dataMaskB8);1435 dataMaskB8);
1436- MicroAPI::DataCopy<uint8_t, MicroAPI::PostLiteral::POST_MODE_UPDATE, MicroAPI::StoreDist::DIST_PACK4_B32>(1436+ MicroAPI::StoreAlign<uint8_t, MicroAPI::PostLiteral::POST_MODE_UPDATE, MicroAPI::StoreDist::DIST_PACK4_B32>(
1437 y1Addr, (MicroAPI::RegTensor<uint8_t>&)dim1x0FP4, OUT_ELE_NUM_ONE_BLK, dataMaskB8);1437 y1Addr, (MicroAPI::RegTensor<uint8_t>&)dim1x0FP4, OUT_ELE_NUM_ONE_BLK, dataMaskB8);
1438- MicroAPI::DataCopy<uint8_t, MicroAPI::PostLiteral::POST_MODE_UPDATE, MicroAPI::StoreDist::DIST_PACK4_B32>(1438+ MicroAPI::StoreAlign<uint8_t, MicroAPI::PostLiteral::POST_MODE_UPDATE, MicroAPI::StoreDist::DIST_PACK4_B32>(
1439 y1Addr, (MicroAPI::RegTensor<uint8_t>&)dim1x1FP4, OUT_ELE_NUM_ONE_BLK, dataMaskB8);1439 y1Addr, (MicroAPI::RegTensor<uint8_t>&)dim1x1FP4, OUT_ELE_NUM_ONE_BLK, dataMaskB8);
1440 }1440 }
1441 }1441 }
@@ -1494,7 +1494,7 @@ __aicore__ inline void DynamicMxQuantWithDualAxisBase<xDtype, y1Dtype, y2Dtype,
1494 MicroAPI::RegTensor<y1Dtype> dim1x0FP4;1494 MicroAPI::RegTensor<y1Dtype> dim1x0FP4;
1495 MicroAPI::RegTensor<y1Dtype> dim1x1FP4;1495 MicroAPI::RegTensor<y1Dtype> dim1x1FP4;
1496 //1496 //
1497- MicroAPI::DataCopy<uint16_t, MicroAPI::PostLiteral::POST_MODE_UPDATE, MicroAPI::LoadDist::DIST_DINTLV_B16>(1497+ MicroAPI::LoadAlign<uint16_t, MicroAPI::PostLiteral::POST_MODE_UPDATE, MicroAPI::LoadDist::DIST_DINTLV_B16>(
1498 reversedShareExp0, reversedShareExp1, mxScale2ReciprocalAddr, vlForHalfNumber_ * DIGIT_TWO);1498 reversedShareExp0, reversedShareExp1, mxScale2ReciprocalAddr, vlForHalfNumber_ * DIGIT_TWO);
1499 MicroAPI::Cast<float, bfloat16_t, castTraitXdtypetoFp32Zero>(1499 MicroAPI::Cast<float, bfloat16_t, castTraitXdtypetoFp32Zero>(
1500 reversedShareExp0ZeroFP32, (MicroAPI::RegTensor<bfloat16_t>&)reversedShareExp0, dataMaskB16);1500 reversedShareExp0ZeroFP32, (MicroAPI::RegTensor<bfloat16_t>&)reversedShareExp0, dataMaskB16);
@@ -1506,9 +1506,9 @@ __aicore__ inline void DynamicMxQuantWithDualAxisBase<xDtype, y1Dtype, y2Dtype,
1506 reversedShareExp1OneFP32, (MicroAPI::RegTensor<bfloat16_t>&)reversedShareExp1, dataMaskB16);1506 reversedShareExp1OneFP32, (MicroAPI::RegTensor<bfloat16_t>&)reversedShareExp1, dataMaskB16);
1507 1507 
1508 for (uint16_t i = 0; i < blockCount; i++) {1508 for (uint16_t i = 0; i < blockCount; i++) {
1509- MicroAPI::DataCopy<xDtype, MicroAPI::PostLiteral::POST_MODE_UPDATE, MicroAPI::LoadDist::DIST_DINTLV_B16>(1509+ MicroAPI::LoadAlign<xDtype, MicroAPI::PostLiteral::POST_MODE_UPDATE, MicroAPI::LoadDist::DIST_DINTLV_B16>(
1510 x0, x1, xAddr, vlForHalfNumber_ * DIGIT_TWO);1510 x0, x1, xAddr, vlForHalfNumber_ * DIGIT_TWO);
1511- MicroAPI::DataCopy<uint16_t, MicroAPI::PostLiteral::POST_MODE_UPDATE, MicroAPI::LoadDist::DIST_E2B_B16>(1511+ MicroAPI::LoadAlign<uint16_t, MicroAPI::PostLiteral::POST_MODE_UPDATE, MicroAPI::LoadDist::DIST_E2B_B16>(
1512 scaleForMulFP16, mxScale1ReciprocalAddr, oneBlockCountB16_);1512 scaleForMulFP16, mxScale1ReciprocalAddr, oneBlockCountB16_);
1513 MicroAPI::Cast<float, bfloat16_t, castTraitXdtypetoFp32Zero>(1513 MicroAPI::Cast<float, bfloat16_t, castTraitXdtypetoFp32Zero>(
1514 scaleForMulZeroFP32, (MicroAPI::RegTensor<bfloat16_t>&)scaleForMulFP16, dataMaskB16);1514 scaleForMulZeroFP32, (MicroAPI::RegTensor<bfloat16_t>&)scaleForMulFP16, dataMaskB16);
@@ -1579,14 +1579,14 @@ __aicore__ inline void DynamicMxQuantWithDualAxisBase<xDtype, y1Dtype, y2Dtype,
1579 MicroAPI::Cast<y1Dtype, bfloat16_t, castTraitBF16toFp4>(dim1x1FP4, dim1x1ZeroBF16, dataMaskB16);1579 MicroAPI::Cast<y1Dtype, bfloat16_t, castTraitBF16toFp4>(dim1x1FP4, dim1x1ZeroBF16, dataMaskB16);
1580 1580 
1581 // copy to ub1581 // copy to ub
1582- MicroAPI::DataCopy<uint8_t, MicroAPI::StoreDist::DIST_PACK4_B32>(1582+ MicroAPI::StoreAlign<uint8_t, MicroAPI::StoreDist::DIST_PACK4_B32>(
1583 y2Addr + (i * ubRowLen_ / DIGIT_TWO), (MicroAPI::RegTensor<uint8_t>&)dim0x0FP4, dataMaskB8);1583 y2Addr + (i * ubRowLen_ / DIGIT_TWO), (MicroAPI::RegTensor<uint8_t>&)dim0x0FP4, dataMaskB8);
1584- MicroAPI::DataCopy<uint8_t, MicroAPI::StoreDist::DIST_PACK4_B32>(1584+ MicroAPI::StoreAlign<uint8_t, MicroAPI::StoreDist::DIST_PACK4_B32>(
1585 y2Addr + OUT_ELE_NUM_ONE_BLK + (i * ubRowLen_ / DIGIT_TWO), (MicroAPI::RegTensor<uint8_t>&)dim0x1FP4,1585 y2Addr + OUT_ELE_NUM_ONE_BLK + (i * ubRowLen_ / DIGIT_TWO), (MicroAPI::RegTensor<uint8_t>&)dim0x1FP4,
1586 dataMaskB8);1586 dataMaskB8);
1587- MicroAPI::DataCopy<uint8_t, MicroAPI::PostLiteral::POST_MODE_UPDATE, MicroAPI::StoreDist::DIST_PACK4_B32>(1587+ MicroAPI::StoreAlign<uint8_t, MicroAPI::PostLiteral::POST_MODE_UPDATE, MicroAPI::StoreDist::DIST_PACK4_B32>(
1588 y1Addr, (MicroAPI::RegTensor<uint8_t>&)dim1x0FP4, OUT_ELE_NUM_ONE_BLK, dataMaskB8);1588 y1Addr, (MicroAPI::RegTensor<uint8_t>&)dim1x0FP4, OUT_ELE_NUM_ONE_BLK, dataMaskB8);
1589- MicroAPI::DataCopy<uint8_t, MicroAPI::PostLiteral::POST_MODE_UPDATE, MicroAPI::StoreDist::DIST_PACK4_B32>(1589+ MicroAPI::StoreAlign<uint8_t, MicroAPI::PostLiteral::POST_MODE_UPDATE, MicroAPI::StoreDist::DIST_PACK4_B32>(
1590 y1Addr, (MicroAPI::RegTensor<uint8_t>&)dim1x1FP4, OUT_ELE_NUM_ONE_BLK, dataMaskB8);1590 y1Addr, (MicroAPI::RegTensor<uint8_t>&)dim1x1FP4, OUT_ELE_NUM_ONE_BLK, dataMaskB8);
1591 }1591 }
1592 }1592 }
@@ -1622,10 +1622,10 @@ __aicore__ inline void DynamicMxQuantWithDualAxisBase<xDtype, y1Dtype, y2Dtype,
1622 1622 
1623 for (uint16_t i = 0; i < blockCount; i++) {1623 for (uint16_t i = 0; i < blockCount; i++) {
1624 // 交织搬运: 256个xDtype按偶奇拆分为x0(偶128), x1(奇128)1624 // 交织搬运: 256个xDtype按偶奇拆分为x0(偶128), x1(奇128)
1625- MicroAPI::DataCopy<xDtype, MicroAPI::PostLiteral::POST_MODE_UPDATE, MicroAPI::LoadDist::DIST_DINTLV_B16>(1625+ MicroAPI::LoadAlign<xDtype, MicroAPI::PostLiteral::POST_MODE_UPDATE, MicroAPI::LoadDist::DIST_DINTLV_B16>(
1626 x0, x1, xAddr, vlForHalfNumber_ * DIGIT_TWO);1626 x0, x1, xAddr, vlForHalfNumber_ * DIGIT_TWO);
1627 // 搬运1/scale: 8个scale广播到128个位置1627 // 搬运1/scale: 8个scale广播到128个位置
1628- MicroAPI::DataCopy<uint16_t, MicroAPI::PostLiteral::POST_MODE_UPDATE, MicroAPI::LoadDist::DIST_E2B_B16>(1628+ MicroAPI::LoadAlign<uint16_t, MicroAPI::PostLiteral::POST_MODE_UPDATE, MicroAPI::LoadDist::DIST_E2B_B16>(
1629 scaleForMulFP16, mxScale1ReciprocalAddr, oneBlockCountB16_);1629 scaleForMulFP16, mxScale1ReciprocalAddr, oneBlockCountB16_);
1630 if constexpr (IsSameType<xDtype, half>::value) {1630 if constexpr (IsSameType<xDtype, half>::value) {
1631 // half输入: 先Cast到FP32再乘scale (避免half精度损失)1631 // half输入: 先Cast到FP32再乘scale (避免half精度损失)
@@ -1665,7 +1665,7 @@ __aicore__ inline void DynamicMxQuantWithDualAxisBase<xDtype, y1Dtype, y2Dtype,
1665 MicroAPI::Add((MicroAPI::RegTensor<uint8_t>&)fp8Layout0, (MicroAPI::RegTensor<uint8_t>&)fp8Layout0,1665 MicroAPI::Add((MicroAPI::RegTensor<uint8_t>&)fp8Layout0, (MicroAPI::RegTensor<uint8_t>&)fp8Layout0,
1666 (MicroAPI::RegTensor<uint8_t>&)fp8Layout1, maskFP8);1666 (MicroAPI::RegTensor<uint8_t>&)fp8Layout1, maskFP8);
1667 // 一次性输出256个FP8值1667 // 一次性输出256个FP8值
1668- MicroAPI::DataCopy<uint8_t, MicroAPI::StoreDist::DIST_NORM_B8>(1668+ MicroAPI::StoreAlign<uint8_t, MicroAPI::StoreDist::DIST_NORM_B8>(
1669 y1Addr + i * vlForHalfNumber_ * DIGIT_TWO, (MicroAPI::RegTensor<uint8_t>&)fp8Layout0, maskFP8);1669 y1Addr + i * vlForHalfNumber_ * DIGIT_TWO, (MicroAPI::RegTensor<uint8_t>&)fp8Layout0, maskFP8);
1670 }1670 }
1671 }1671 }
@@ -1699,13 +1699,13 @@ __aicore__ inline void DynamicMxQuantWithDualAxisBase<xDtype, y1Dtype, y2Dtype,
1699 MicroAPI::MaskReg pregAll32 = MicroAPI::CreateMask<uint32_t, MicroAPI::MaskPattern::ALL>();1699 MicroAPI::MaskReg pregAll32 = MicroAPI::CreateMask<uint32_t, MicroAPI::MaskPattern::ALL>();
1700 MicroAPI::MaskReg maskFP8 = MicroAPI::CreateMask<y1Dtype>();1700 MicroAPI::MaskReg maskFP8 = MicroAPI::CreateMask<y1Dtype>();
1701 1701 
1702- MicroAPI::DataCopy<uint16_t, MicroAPI::LoadDist::DIST_NORM>(reversedShareExp, mxScale2ReciprocalAddr);1702+ MicroAPI::LoadAlign<uint16_t, MicroAPI::LoadDist::DIST_NORM>(reversedShareExp, mxScale2ReciprocalAddr);
1703 MicroAPI::Cast<float, bfloat16_t, castTraitXdtypetoFp32Zero>(1703 MicroAPI::Cast<float, bfloat16_t, castTraitXdtypetoFp32Zero>(
1704 reversedShareExp0FP32, (MicroAPI::RegTensor<bfloat16_t>&)reversedShareExp, pregAll16);1704 reversedShareExp0FP32, (MicroAPI::RegTensor<bfloat16_t>&)reversedShareExp, pregAll16);
1705 MicroAPI::Cast<float, bfloat16_t, castTraitXdtypetoFp32One>(1705 MicroAPI::Cast<float, bfloat16_t, castTraitXdtypetoFp32One>(
1706 reversedShareExp1FP32, (MicroAPI::RegTensor<bfloat16_t>&)reversedShareExp, pregAll16);1706 reversedShareExp1FP32, (MicroAPI::RegTensor<bfloat16_t>&)reversedShareExp, pregAll16);
1707 for (uint16_t j = 0; j < blockCount; j++) {1707 for (uint16_t j = 0; j < blockCount; j++) {
1708- MicroAPI::DataCopy<xDtype, MicroAPI::LoadDist::DIST_NORM>(x, xAddr + j * ubRowLen_);1708+ MicroAPI::LoadAlign<xDtype, MicroAPI::LoadDist::DIST_NORM>(x, xAddr + j * ubRowLen_);
1709 MicroAPI::Cast<float, xDtype, castTraitXdtypetoFp32Zero>(x0FP32, x, pregAll16);1709 MicroAPI::Cast<float, xDtype, castTraitXdtypetoFp32Zero>(x0FP32, x, pregAll16);
1710 MicroAPI::Cast<float, xDtype, castTraitXdtypetoFp32One>(x1FP32, x, pregAll16);1710 MicroAPI::Cast<float, xDtype, castTraitXdtypetoFp32One>(x1FP32, x, pregAll16);
1711 1711 
@@ -1724,7 +1724,7 @@ __aicore__ inline void DynamicMxQuantWithDualAxisBase<xDtype, y1Dtype, y2Dtype,
1724 MicroAPI::Pack<uint16_t, uint32_t, MicroAPI::HighLowPart::LOWEST>(1724 MicroAPI::Pack<uint16_t, uint32_t, MicroAPI::HighLowPart::LOWEST>(
1725 (MicroAPI::RegTensor<uint16_t>&)fp8Layout0, (MicroAPI::RegTensor<uint32_t>&)fp8Layout0);1725 (MicroAPI::RegTensor<uint16_t>&)fp8Layout0, (MicroAPI::RegTensor<uint32_t>&)fp8Layout0);
1726 1726 
1727- DataCopy(y2Addr + (j * ubRowLen_), (MicroAPI::RegTensor<uint8_t>&)fp8Layout0, pregAll8);1727+ Reg::StoreAlign(y2Addr + (j * ubRowLen_), (MicroAPI::RegTensor<uint8_t>&)fp8Layout0, pregAll8);
1728 }1728 }
1729 }1729 }
1730#endif1730#endif
@@ -1749,7 +1749,7 @@ __aicore__ inline void DynamicMxQuantWithDualAxisBase<xDtype, y1Dtype, y2Dtype,
1749 MicroAPI::Compare<int32_t, CMPMODE::EQ>(negInfMask, (MicroAPI::RegTensor<int32_t>&)Reg, negZero, pregAll32);1749 MicroAPI::Compare<int32_t, CMPMODE::EQ>(negInfMask, (MicroAPI::RegTensor<int32_t>&)Reg, negZero, pregAll32);
1750 if constexpr (IsSameType<y1Dtype, fp4x2_e1m2_t>::value) {1750 if constexpr (IsSameType<y1Dtype, fp4x2_e1m2_t>::value) {
1751 MicroAPI::Muls(Reg, Reg, FOUR, pregAll32);1751 MicroAPI::Muls(Reg, Reg, FOUR, pregAll32);
1752- MicroAPI::CompareScalar<float, CMPMODE::LT>(specialMask, Reg, 0, pregAll32);1752+ MicroAPI::Compares<float, CMPMODE::LT>(specialMask, Reg, 0, pregAll32);
1753 MicroAPI::Truncate<float, roundMode>(Reg, Reg, pregAll32);1753 MicroAPI::Truncate<float, roundMode>(Reg, Reg, pregAll32);
1754 MicroAPI::Muls(Reg, Reg, ONE_FOURTH, pregAll32);1754 MicroAPI::Muls(Reg, Reg, ONE_FOURTH, pregAll32);
1755 } else {1755 } else {
@@ -1767,13 +1767,13 @@ __aicore__ inline void DynamicMxQuantWithDualAxisBase<xDtype, y1Dtype, y2Dtype,
1767 MicroAPI::Mul(Reg, Reg, (MicroAPI::RegTensor<float>&)exp1FP32, pregAll32);1767 MicroAPI::Mul(Reg, Reg, (MicroAPI::RegTensor<float>&)exp1FP32, pregAll32);
1768 MicroAPI::Adds(exp0FP32, exp0FP32, FP32_BIAS, pregAll32);1768 MicroAPI::Adds(exp0FP32, exp0FP32, FP32_BIAS, pregAll32);
1769 MicroAPI::ShiftLefts(exp0FP32, exp0FP32, SHR_NUM_FOR_FP32, pregAll32);1769 MicroAPI::ShiftLefts(exp0FP32, exp0FP32, SHR_NUM_FOR_FP32, pregAll32);
1770- MicroAPI::CompareScalar<float, CMPMODE::LT>(specialMask, Reg, 0, pregAll32);1770+ MicroAPI::Compares<float, CMPMODE::LT>(specialMask, Reg, 0, pregAll32);
1771 MicroAPI::Truncate<float, roundMode>(Reg, Reg, pregAll32);1771 MicroAPI::Truncate<float, roundMode>(Reg, Reg, pregAll32);
1772 MicroAPI::Mul(Reg, Reg, (MicroAPI::RegTensor<float>&)exp0FP32, pregAll32);1772 MicroAPI::Mul(Reg, Reg, (MicroAPI::RegTensor<float>&)exp0FP32, pregAll32);
1773 }1773 }
1774- MicroAPI::CompareScalar<float, CMPMODE::EQ>(zeroMask, Reg, 0, pregAll32);1774+ MicroAPI::Compares<float, CMPMODE::EQ>(zeroMask, Reg, 0, pregAll32);
1775- MicroAPI::MaskAnd(zeroMask, specialMask, zeroMask, pregAll32);1775+ MicroAPI::And(zeroMask, specialMask, zeroMask, pregAll32);
1776- MicroAPI::MaskOr(zeroMask, negInfMask, zeroMask, pregAll32);1776+ MicroAPI::Or(zeroMask, negInfMask, zeroMask, pregAll32);
1777 MicroAPI::Select<int32_t>((MicroAPI::RegTensor<int32_t>&)Reg, negZero, (MicroAPI::RegTensor<int32_t>&)Reg,1777 MicroAPI::Select<int32_t>((MicroAPI::RegTensor<int32_t>&)Reg, negZero, (MicroAPI::RegTensor<int32_t>&)Reg,
1778 zeroMask);1778 zeroMask);
1779}1779}
Mquant/dynamic_quant/op_kernel/arch35/dynamic_quant_regbase_full_load.h+31-33
@@ -159,21 +159,21 @@ private:
159 LocalTensor<xDtype> smoothLocal;159 LocalTensor<xDtype> smoothLocal;
160 LocalTensor<float> offsetLocal;160 LocalTensor<float> offsetLocal;
161 161 
162- __local_mem__ xDtype* xAddr = (__local_mem__ xDtype*)xLocal.GetPhyAddr();162+ __ubuf__ xDtype* xAddr = (__ubuf__ xDtype*)xLocal.GetPhyAddr();
163- __local_mem__ xDtype* smoothAddr;163+ __ubuf__ xDtype* smoothAddr;
164 164 
165- __local_mem__ yCopyDtype* yAddr = (__local_mem__ yCopyDtype*)yLocal.GetPhyAddr();165+ __ubuf__ yCopyDtype* yAddr = (__ubuf__ yCopyDtype*)yLocal.GetPhyAddr();
166- __local_mem__ float* scaleAddr = (__local_mem__ float*)scaleLocal.GetPhyAddr();166+ __ubuf__ float* scaleAddr = (__ubuf__ float*)scaleLocal.GetPhyAddr();
167- __local_mem__ float* offsetAddr;167+ __ubuf__ float* offsetAddr;
168 168 
169 if constexpr (isSymmetrical == false) {169 if constexpr (isSymmetrical == false) {
170 offsetLocal = offsetQueue.template AllocTensor<float>();170 offsetLocal = offsetQueue.template AllocTensor<float>();
171- offsetAddr = (__local_mem__ float*)offsetLocal.GetPhyAddr();171+ offsetAddr = (__ubuf__ float*)offsetLocal.GetPhyAddr();
172 }172 }
173 173 
174 if constexpr (hasSmooth) {174 if constexpr (hasSmooth) {
175 smoothLocal = smoothQueue.template DeQue<xDtype>();175 smoothLocal = smoothQueue.template DeQue<xDtype>();
176- smoothAddr = (__local_mem__ xDtype*)smoothLocal.GetPhyAddr();176+ smoothAddr = (__ubuf__ xDtype*)smoothLocal.GetPhyAddr();
177 }177 }
178 178 
179 ComputeVF(xAddr, smoothAddr, yAddr, scaleAddr, offsetAddr, multiRow);179 ComputeVF(xAddr, smoothAddr, yAddr, scaleAddr, offsetAddr, multiRow);
@@ -218,7 +218,7 @@ private:
218 scaleQueue.FreeTensor(scaleLocal);218 scaleQueue.FreeTensor(scaleLocal);
219 }219 }
220 220 
221- __aicore__ inline void DataCopyInputVF(__local_mem__ xDtype* xAddr, __local_mem__ xDtype* smoothAddr,221+ __aicore__ inline void DataCopyInputVF(__ubuf__ xDtype* xAddr, __ubuf__ xDtype* smoothAddr,
222 AscendC::MicroAPI::RegTensor<float>& vregRes,222 AscendC::MicroAPI::RegTensor<float>& vregRes,
223 AscendC::MicroAPI::MaskReg pregMask)223 AscendC::MicroAPI::MaskReg pregMask)
224 {224 {
@@ -226,17 +226,16 @@ private:
226 AscendC::MicroAPI::RegTensor<xDtype> vregSmooth;226 AscendC::MicroAPI::RegTensor<xDtype> vregSmooth;
227 AscendC::MicroAPI::RegTensor<float> vregSmoothFp32;227 AscendC::MicroAPI::RegTensor<float> vregSmoothFp32;
228 228 
229- AscendC::MicroAPI::DataCopy<xDtype, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(vregX, xAddr);229+ AscendC::MicroAPI::LoadAlign<xDtype, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(vregX, xAddr);
230 AscendC::MicroAPI::Cast<float, xDtype, castTraitB16ToB32>(vregRes, vregX, pregMask);230 AscendC::MicroAPI::Cast<float, xDtype, castTraitB16ToB32>(vregRes, vregX, pregMask);
231 if constexpr (hasSmooth) {231 if constexpr (hasSmooth) {
232- AscendC::MicroAPI::DataCopy<xDtype, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(vregSmooth, smoothAddr);232+ AscendC::MicroAPI::LoadAlign<xDtype, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(vregSmooth, smoothAddr);
233 AscendC::MicroAPI::Cast<float, xDtype, castTraitB16ToB32>(vregSmoothFp32, vregSmooth, pregMask);233 AscendC::MicroAPI::Cast<float, xDtype, castTraitB16ToB32>(vregSmoothFp32, vregSmooth, pregMask);
234 AscendC::MicroAPI::Mul(vregRes, vregRes, vregSmoothFp32, pregMask);234 AscendC::MicroAPI::Mul(vregRes, vregRes, vregSmoothFp32, pregMask);
235 }235 }
236 }236 }
237 237 
238- __aicore__ inline void ComputeYVF(__local_mem__ xDtype* xAddr, __local_mem__ xDtype* smoothAddr,238+ __aicore__ inline void ComputeYVF(__ubuf__ xDtype* xAddr, __ubuf__ xDtype* smoothAddr, __ubuf__ yCopyDtype* yAddr,
239- __local_mem__ yCopyDtype* yAddr,
240 AscendC::MicroAPI::RegTensor<float>& vregDupScale,239 AscendC::MicroAPI::RegTensor<float>& vregDupScale,
241 AscendC::MicroAPI::RegTensor<float>& vregDupOffset, int32_t indexRow)240 AscendC::MicroAPI::RegTensor<float>& vregDupOffset, int32_t indexRow)
242 {241 {
@@ -281,17 +280,17 @@ private:
281 addr = yAddr + (indexRow * outAlignLen + j * VL) / 2;280 addr = yAddr + (indexRow * outAlignLen + j * VL) / 2;
282 }281 }
283 if constexpr (IsSameType<yDtype, int4b_t>::value) {282 if constexpr (IsSameType<yDtype, int4b_t>::value) {
284- AscendC::MicroAPI::DataCopy<yCopyDtype, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(addr, vregY,283+ AscendC::MicroAPI::StoreAlign<yCopyDtype, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(addr, vregY,
285- pregHalf);284+ pregHalf);
286 } else {285 } else {
287- AscendC::MicroAPI::DataCopy<yCopyDtype, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(addr, vregY,286+ AscendC::MicroAPI::StoreAlign<yCopyDtype, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(addr, vregY,
288- preg2);287+ preg2);
289 }288 }
290 }289 }
291 }290 }
292 291 
293- __aicore__ inline void ComputeScaleVF(__local_mem__ xDtype* xAddr, __local_mem__ xDtype* smoothAddr,292+ __aicore__ inline void ComputeScaleVF(__ubuf__ xDtype* xAddr, __ubuf__ xDtype* smoothAddr,
294- __local_mem__ float* scaleAddr, __local_mem__ float* offsetAddr,293+ __ubuf__ float* scaleAddr, __ubuf__ float* offsetAddr,
295 AscendC::MicroAPI::RegTensor<float>& vregDupScale,294 AscendC::MicroAPI::RegTensor<float>& vregDupScale,
296 AscendC::MicroAPI::RegTensor<float>& vregDupOffset, int32_t indexRow)295 AscendC::MicroAPI::RegTensor<float>& vregDupOffset, int32_t indexRow)
297 {296 {
@@ -316,8 +315,8 @@ private:
316 AscendC::MicroAPI::MaskReg preg4;315 AscendC::MicroAPI::MaskReg preg4;
317 AscendC::MicroAPI::MaskReg preg5 = AscendC::MicroAPI::CreateMask<float, AscendC::MicroAPI::MaskPattern::VL1>();316 AscendC::MicroAPI::MaskReg preg5 = AscendC::MicroAPI::CreateMask<float, AscendC::MicroAPI::MaskPattern::VL1>();
318 317 
319- AscendC::MicroAPI::UnalignReg ureg0;318+ AscendC::MicroAPI::UnalignRegForStore ureg0;
320- AscendC::MicroAPI::UnalignReg ureg1;319+ AscendC::MicroAPI::UnalignRegForStore ureg1;
321 320 
322 uint32_t rowCount = sizeHalfLen;321 uint32_t rowCount = sizeHalfLen;
323 uint16_t vfLoop = (rowCount + VL - 1) / VL;322 uint16_t vfLoop = (rowCount + VL - 1) / VL;
@@ -332,10 +331,10 @@ private:
332 AscendC::MicroAPI::Abs(vregAbs, vregInput, preg0);331 AscendC::MicroAPI::Abs(vregAbs, vregInput, preg0);
333 AscendC::MicroAPI::Max(vregMaxX, vregAbs, vregMaxX, preg1);332 AscendC::MicroAPI::Max(vregMaxX, vregAbs, vregMaxX, preg1);
334 }333 }
335- AscendC::MicroAPI::ReduceMax(vregReduceMaxX, vregMaxX, preg1);334+ AscendC::MicroAPI::Reduce<MicroAPI::ReduceType::MAX>(vregReduceMaxX, vregMaxX, preg1);
336 AscendC::MicroAPI::Muls(vregScale, vregReduceMaxX, maxValue, preg1);335 AscendC::MicroAPI::Muls(vregScale, vregReduceMaxX, maxValue, preg1);
337 AscendC::MicroAPI::Duplicate(vregDupScale, vregScale, preg1);336 AscendC::MicroAPI::Duplicate(vregDupScale, vregScale, preg1);
338- AscendC::MicroAPI::DataCopyUnAlign<float, AscendC::MicroAPI::PostLiteral::POST_MODE_UPDATE>(337+ AscendC::MicroAPI::StoreUnAlign<float, AscendC::MicroAPI::PostLiteral::POST_MODE_UPDATE>(
339 scaleAddr, vregScale, ureg0, 1);338 scaleAddr, vregScale, ureg0, 1);
340 } else if constexpr (isSymmetrical == false) {339 } else if constexpr (isSymmetrical == false) {
341 AscendC::MicroAPI::Duplicate(vregMinX, POS_INFINITY, preg1);340 AscendC::MicroAPI::Duplicate(vregMinX, POS_INFINITY, preg1);
@@ -346,15 +345,15 @@ private:
346 AscendC::MicroAPI::Max(vregMaxX, vregInput, vregMaxX, preg1);345 AscendC::MicroAPI::Max(vregMaxX, vregInput, vregMaxX, preg1);
347 AscendC::MicroAPI::Min(vregMinX, vregInput, vregMinX, preg1);346 AscendC::MicroAPI::Min(vregMinX, vregInput, vregMinX, preg1);
348 }347 }
349- AscendC::MicroAPI::ReduceMax(vregReduceMaxX, vregMaxX, preg1);348+ AscendC::MicroAPI::Reduce<MicroAPI::ReduceType::MAX>(vregReduceMaxX, vregMaxX, preg1);
350- AscendC::MicroAPI::ReduceMin(vregReduceMinX, vregMinX, preg1);349+ AscendC::MicroAPI::Reduce<MicroAPI::ReduceType::MIN>(vregReduceMinX, vregMinX, preg1);
351 350 
352 // finnal compute max and min351 // finnal compute max and min
353 preg4 = AscendC::MicroAPI::UpdateMask<float>(sregTail);352 preg4 = AscendC::MicroAPI::UpdateMask<float>(sregTail);
354 DataCopyInputVF(xAddr + indexRow * rowCount + (vfLoop - 1) * VL, smoothAddr + (vfLoop - 1) * VL, vregInput,353 DataCopyInputVF(xAddr + indexRow * rowCount + (vfLoop - 1) * VL, smoothAddr + (vfLoop - 1) * VL, vregInput,
355 preg4);354 preg4);
356- AscendC::MicroAPI::ReduceMax(vregReduceMaxXTail, vregInput, preg4);355+ AscendC::MicroAPI::Reduce<MicroAPI::ReduceType::MAX>(vregReduceMaxXTail, vregInput, preg4);
357- AscendC::MicroAPI::ReduceMin(vregReduceMinXTail, vregInput, preg4);356+ AscendC::MicroAPI::Reduce<MicroAPI::ReduceType::MIN>(vregReduceMinXTail, vregInput, preg4);
358 AscendC::MicroAPI::Max(vregFinalMax, vregReduceMaxX, vregReduceMaxXTail, preg5);357 AscendC::MicroAPI::Max(vregFinalMax, vregReduceMaxX, vregReduceMaxXTail, preg5);
359 AscendC::MicroAPI::Min(vregFinalMin, vregReduceMinX, vregReduceMinXTail, preg5);358 AscendC::MicroAPI::Min(vregFinalMin, vregReduceMinX, vregReduceMinXTail, preg5);
360 359 
@@ -362,24 +361,23 @@ private:
362 AscendC::MicroAPI::Sub(vregMaxSubMin, vregFinalMax, vregFinalMin, preg5);361 AscendC::MicroAPI::Sub(vregMaxSubMin, vregFinalMax, vregFinalMin, preg5);
363 AscendC::MicroAPI::Muls(vregScale, vregMaxSubMin, offsetDivValue, preg5);362 AscendC::MicroAPI::Muls(vregScale, vregMaxSubMin, offsetDivValue, preg5);
364 AscendC::MicroAPI::Duplicate(vregDupScale, vregScale, preg1);363 AscendC::MicroAPI::Duplicate(vregDupScale, vregScale, preg1);
365- AscendC::MicroAPI::DataCopyUnAlign<float, AscendC::MicroAPI::PostLiteral::POST_MODE_UPDATE>(364+ AscendC::MicroAPI::StoreUnAlign<float, AscendC::MicroAPI::PostLiteral::POST_MODE_UPDATE>(
366 scaleAddr, vregScale, ureg0, 1);365 scaleAddr, vregScale, ureg0, 1);
367 AscendC::MicroAPI::Div<float, &mode>(vregMaxDivScale, vregFinalMax, vregScale, preg5);366 AscendC::MicroAPI::Div<float, &mode>(vregMaxDivScale, vregFinalMax, vregScale, preg5);
368 AscendC::MicroAPI::Muls(vregNegMaxDivScale, vregMaxDivScale, NEGATIVE_ONE, preg5);367 AscendC::MicroAPI::Muls(vregNegMaxDivScale, vregMaxDivScale, NEGATIVE_ONE, preg5);
369 AscendC::MicroAPI::Adds(vregOffset, vregNegMaxDivScale, offsetValue, preg5); //368 AscendC::MicroAPI::Adds(vregOffset, vregNegMaxDivScale, offsetValue, preg5); //
370 AscendC::MicroAPI::Duplicate(vregDupOffset, vregOffset, preg1);369 AscendC::MicroAPI::Duplicate(vregDupOffset, vregOffset, preg1);
371- AscendC::MicroAPI::DataCopyUnAlign<float, AscendC::MicroAPI::PostLiteral::POST_MODE_UPDATE>(370+ AscendC::MicroAPI::StoreUnAlign<float, AscendC::MicroAPI::PostLiteral::POST_MODE_UPDATE>(
372 offsetAddr, vregOffset, ureg1, 1);371 offsetAddr, vregOffset, ureg1, 1);
373 }372 }
374- AscendC::MicroAPI::DataCopyUnAlignPost(scaleAddr, ureg0, 0);373+ AscendC::MicroAPI::StoreUnAlignPost(scaleAddr, ureg0, 0);
375 if constexpr (isSymmetrical == false) {374 if constexpr (isSymmetrical == false) {
376- AscendC::MicroAPI::DataCopyUnAlignPost(offsetAddr, ureg1, 0);375+ AscendC::MicroAPI::StoreUnAlignPost(offsetAddr, ureg1, 0);
377 }376 }
378 }377 }
379 378 
380- __aicore__ inline void ComputeVF(__local_mem__ xDtype* xAddr, __local_mem__ xDtype* smoothAddr,379+ __aicore__ inline void ComputeVF(__ubuf__ xDtype* xAddr, __ubuf__ xDtype* smoothAddr, __ubuf__ yCopyDtype* yAddr,
381- __local_mem__ yCopyDtype* yAddr, __local_mem__ float* scaleAddr,380+ __ubuf__ float* scaleAddr, __ubuf__ float* offsetAddr, int32_t multiRow)
382- __local_mem__ float* offsetAddr, int32_t multiRow)
383 {381 {
384 __VEC_SCOPE__382 __VEC_SCOPE__
385 {383 {
Mquant/dynamic_quant/op_kernel/arch35/dynamic_quant_regbase_full_load_pertensor.h+108-110
@@ -95,31 +95,31 @@ public:
95 LocalTensor<float> MinOutLocal;95 LocalTensor<float> MinOutLocal;
96 LocalTensor<float> scaleOutLocal;96 LocalTensor<float> scaleOutLocal;
97 LocalTensor<float> offsetLocal;97 LocalTensor<float> offsetLocal;
98- __local_mem__ float* MaxOutLocalAddr;98+ __ubuf__ float* MaxOutLocalAddr;
99- __local_mem__ float* MinOutLocalAddr;99+ __ubuf__ float* MinOutLocalAddr;
100- __local_mem__ float* scaleOutLocalAddr;100+ __ubuf__ float* scaleOutLocalAddr;
101- __local_mem__ float* offsetLocalAddr;101+ __ubuf__ float* offsetLocalAddr;
102 102 
103 if constexpr (isSymmertrical == false) {103 if constexpr (isSymmertrical == false) {
104 MaxOutLocal = MaxOutQueue.template AllocTensor<float>();104 MaxOutLocal = MaxOutQueue.template AllocTensor<float>();
105 AscendC::Duplicate(MaxOutLocal, MIN_FLOAT_VALUE, SIXTY_FOUR, ONE, ONE, EIGHT);105 AscendC::Duplicate(MaxOutLocal, MIN_FLOAT_VALUE, SIXTY_FOUR, ONE, ONE, EIGHT);
106- MaxOutLocalAddr = (__local_mem__ float*)MaxOutLocal.GetPhyAddr();106+ MaxOutLocalAddr = (__ubuf__ float*)MaxOutLocal.GetPhyAddr();
107 107 
108 MinOutLocal = MinOutQueue.template AllocTensor<float>();108 MinOutLocal = MinOutQueue.template AllocTensor<float>();
109 AscendC::Duplicate(MinOutLocal, MAX_FLOAT_VALUE, SIXTY_FOUR, ONE, ONE, EIGHT);109 AscendC::Duplicate(MinOutLocal, MAX_FLOAT_VALUE, SIXTY_FOUR, ONE, ONE, EIGHT);
110- MinOutLocalAddr = (__local_mem__ float*)MinOutLocal.GetPhyAddr();110+ MinOutLocalAddr = (__ubuf__ float*)MinOutLocal.GetPhyAddr();
111 111 
112 scaleOutLocal = scaleOutQueue.template AllocTensor<float>();112 scaleOutLocal = scaleOutQueue.template AllocTensor<float>();
113 AscendC::Duplicate(scaleOutLocal, (float)0.0, SIXTY_FOUR, ONE, ONE, EIGHT);113 AscendC::Duplicate(scaleOutLocal, (float)0.0, SIXTY_FOUR, ONE, ONE, EIGHT);
114- scaleOutLocalAddr = (__local_mem__ float*)scaleOutLocal.GetPhyAddr();114+ scaleOutLocalAddr = (__ubuf__ float*)scaleOutLocal.GetPhyAddr();
115 115 
116 offsetLocal = offsetQueue.template AllocTensor<float>();116 offsetLocal = offsetQueue.template AllocTensor<float>();
117 AscendC::Duplicate(offsetLocal, (float)0.0, SIXTY_FOUR, ONE, ONE, EIGHT);117 AscendC::Duplicate(offsetLocal, (float)0.0, SIXTY_FOUR, ONE, ONE, EIGHT);
118- offsetLocalAddr = (__local_mem__ float*)offsetLocal.GetPhyAddr();118+ offsetLocalAddr = (__ubuf__ float*)offsetLocal.GetPhyAddr();
119 } else {119 } else {
120 scaleOutLocal = scaleOutQueue.template AllocTensor<float>();120 scaleOutLocal = scaleOutQueue.template AllocTensor<float>();
121 AscendC::Duplicate(scaleOutLocal, (float)0.0, SIXTY_FOUR, ONE, ONE, EIGHT);121 AscendC::Duplicate(scaleOutLocal, (float)0.0, SIXTY_FOUR, ONE, ONE, EIGHT);
122- scaleOutLocalAddr = (__local_mem__ float*)scaleOutLocal.GetPhyAddr();122+ scaleOutLocalAddr = (__ubuf__ float*)scaleOutLocal.GetPhyAddr();
123 }123 }
124 scaleOffset = 0;124 scaleOffset = 0;
125 CopyInScaleByEle(scaleOffset, tilingData_.coreNum);125 CopyInScaleByEle(scaleOffset, tilingData_.coreNum);
@@ -141,8 +141,8 @@ public:
141 ProcessScaleCol();141 ProcessScaleCol();
142 }142 }
143 143 
144- __aicore__ inline void ProcessYRow(int32_t multiRow, int32_t loopNum, __local_mem__ float* scaleAddr,144+ __aicore__ inline void ProcessYRow(int32_t multiRow, int32_t loopNum, __ubuf__ float* scaleAddr,
145- __local_mem__ float* offsetAddr)145+ __ubuf__ float* offsetAddr)
146 {146 {
147 CopyIn(multiRow, loopNum);147 CopyIn(multiRow, loopNum);
148 ComputeY(multiRow, scaleAddr, offsetAddr);148 ComputeY(multiRow, scaleAddr, offsetAddr);
@@ -152,13 +152,13 @@ public:
152 __aicore__ inline void ProcessY()152 __aicore__ inline void ProcessY()
153 {153 {
154 LocalTensor<float> scaleOutLocal = scaleOutQueue.template DeQue<float>();154 LocalTensor<float> scaleOutLocal = scaleOutQueue.template DeQue<float>();
155- __local_mem__ float* scaleOutLocalAddr = (__local_mem__ float*)scaleOutLocal.GetPhyAddr();155+ __ubuf__ float* scaleOutLocalAddr = (__ubuf__ float*)scaleOutLocal.GetPhyAddr();
156 LocalTensor<float> offsetLocal;156 LocalTensor<float> offsetLocal;
157- __local_mem__ float* offsetLocalAddr;157+ __ubuf__ float* offsetLocalAddr;
158 158 
159 if constexpr (isSymmertrical == false) {159 if constexpr (isSymmertrical == false) {
160 offsetLocal = offsetQueue.template DeQue<float>();160 offsetLocal = offsetQueue.template DeQue<float>();
161- offsetLocalAddr = (__local_mem__ float*)offsetLocal.GetPhyAddr();161+ offsetLocalAddr = (__ubuf__ float*)offsetLocal.GetPhyAddr();
162 }162 }
163 163 
164 for (int32_t i = 0; i < loopCnt; i++) {164 for (int32_t i = 0; i < loopCnt; i++) {
@@ -294,10 +294,9 @@ private:
294 }294 }
295 }295 }
296 296 
297- __aicore__ inline void ComputeMaxRowScaleVF(__local_mem__ xDtype* inLocalAddr,297+ __aicore__ inline void ComputeMaxRowScaleVF(__ubuf__ xDtype* inLocalAddr, __ubuf__ xDtype* smoothLocalAddr,
298- __local_mem__ xDtype* smoothLocalAddr,298+ __ubuf__ float* scaleLocalAddr, __ubuf__ float* maxLocalAddr,
299- __local_mem__ float* scaleLocalAddr, __local_mem__ float* maxLocalAddr,299+ __ubuf__ float* minLocalAddr, uint32_t multiRow)
300- __local_mem__ float* minLocalAddr, uint32_t multiRow)
301 {300 {
302 uint32_t dtypeSize = sizeof(float);301 uint32_t dtypeSize = sizeof(float);
303 uint16_t VL = AscendC::VECTOR_REG_WIDTH / dtypeSize;302 uint16_t VL = AscendC::VECTOR_REG_WIDTH / dtypeSize;
@@ -327,10 +326,10 @@ private:
327 preg1 = AscendC::MicroAPI::CreateMask<float, AscendC::MicroAPI::MaskPattern::ALL>();326 preg1 = AscendC::MicroAPI::CreateMask<float, AscendC::MicroAPI::MaskPattern::ALL>();
328 327 
329 if constexpr (isSymmertrical == false) {328 if constexpr (isSymmertrical == false) {
330- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_BRC_B32>(vreg9_1, maxLocalAddr);329+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_BRC_B32>(vreg9_1, maxLocalAddr);
331- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_BRC_B32>(vreg9_2, minLocalAddr);330+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_BRC_B32>(vreg9_2, minLocalAddr);
332 } else {331 } else {
333- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_BRC_B32>(vreg9_1, scaleLocalAddr);332+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_BRC_B32>(vreg9_1, scaleLocalAddr);
334 }333 }
335 334 
336 for (uint16_t i = 0; i < static_cast<uint16_t>(multiRow); i++) {335 for (uint16_t i = 0; i < static_cast<uint16_t>(multiRow); i++) {
@@ -343,11 +342,11 @@ private:
343 }342 }
344 for (uint16_t j = 0; j < static_cast<uint16_t>(vfLoop - 1); j++) {343 for (uint16_t j = 0; j < static_cast<uint16_t>(vfLoop - 1); j++) {
345 preg0 = AscendC::MicroAPI::UpdateMask<float>(sreg0);344 preg0 = AscendC::MicroAPI::UpdateMask<float>(sreg0);
346- AscendC::MicroAPI::DataCopy<xDtype, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(345+ AscendC::MicroAPI::LoadAlign<xDtype, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(
347 vreg1, inLocalAddr + i * rowCount + j * VL);346 vreg1, inLocalAddr + i * rowCount + j * VL);
348 AscendC::MicroAPI::Cast<float, xDtype, castTraitB16ToB32>(vreg3, vreg1, preg0);347 AscendC::MicroAPI::Cast<float, xDtype, castTraitB16ToB32>(vreg3, vreg1, preg0);
349 if constexpr (hasSmooth) {348 if constexpr (hasSmooth) {
350- AscendC::MicroAPI::DataCopy<xDtype, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(349+ AscendC::MicroAPI::LoadAlign<xDtype, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(
351 vreg2, smoothLocalAddr + j * VL);350 vreg2, smoothLocalAddr + j * VL);
352 AscendC::MicroAPI::Cast<float, xDtype, castTraitB16ToB32>(vreg4, vreg2, preg0);351 AscendC::MicroAPI::Cast<float, xDtype, castTraitB16ToB32>(vreg4, vreg2, preg0);
353 AscendC::MicroAPI::Mul(vreg3, vreg3, vreg4, preg0);352 AscendC::MicroAPI::Mul(vreg3, vreg3, vreg4, preg0);
@@ -362,31 +361,31 @@ private:
362 }361 }
363 }362 }
364 if constexpr (isSymmertrical == false) {363 if constexpr (isSymmertrical == false) {
365- AscendC::MicroAPI::ReduceMax(vreg8_1, vreg7_1, preg1);364+ AscendC::MicroAPI::Reduce<MicroAPI::ReduceType::MAX>(vreg8_1, vreg7_1, preg1);
366- AscendC::MicroAPI::ReduceMin(vreg8_2, vreg7_2, preg1);365+ AscendC::MicroAPI::Reduce<MicroAPI::ReduceType::MIN>(vreg8_2, vreg7_2, preg1);
367 } else {366 } else {
368- AscendC::MicroAPI::ReduceMax(vreg8_1, vreg7_1, preg1);367+ AscendC::MicroAPI::Reduce<MicroAPI::ReduceType::MAX>(vreg8_1, vreg7_1, preg1);
369 }368 }
370 for (uint16_t j = static_cast<uint16_t>(vfLoop - 1); j < vfLoop; j++) {369 for (uint16_t j = static_cast<uint16_t>(vfLoop - 1); j < vfLoop; j++) {
371 preg0 = AscendC::MicroAPI::UpdateMask<float>(sreg0);370 preg0 = AscendC::MicroAPI::UpdateMask<float>(sreg0);
372- AscendC::MicroAPI::DataCopy<xDtype, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(371+ AscendC::MicroAPI::LoadAlign<xDtype, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(
373 vreg1, inLocalAddr + i * rowCount + j * VL);372 vreg1, inLocalAddr + i * rowCount + j * VL);
374 AscendC::MicroAPI::Cast<float, xDtype, castTraitB16ToB32>(vreg3, vreg1, preg0);373 AscendC::MicroAPI::Cast<float, xDtype, castTraitB16ToB32>(vreg3, vreg1, preg0);
375 if constexpr (hasSmooth) {374 if constexpr (hasSmooth) {
376- AscendC::MicroAPI::DataCopy<xDtype, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(375+ AscendC::MicroAPI::LoadAlign<xDtype, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(
377 vreg2, smoothLocalAddr + j * VL);376 vreg2, smoothLocalAddr + j * VL);
378 AscendC::MicroAPI::Cast<float, xDtype, castTraitB16ToB32>(vreg4, vreg2, preg0);377 AscendC::MicroAPI::Cast<float, xDtype, castTraitB16ToB32>(vreg4, vreg2, preg0);
379 AscendC::MicroAPI::Mul(vreg3, vreg3, vreg4, preg0);378 AscendC::MicroAPI::Mul(vreg3, vreg3, vreg4, preg0);
380 }379 }
381 if constexpr (isSymmertrical == false) {380 if constexpr (isSymmertrical == false) {
382- AscendC::MicroAPI::ReduceMax(vreg7_1, vreg3, preg0);381+ AscendC::MicroAPI::Reduce<MicroAPI::ReduceType::MAX>(vreg7_1, vreg3, preg0);
383- AscendC::MicroAPI::ReduceMin(vreg7_2, vreg3, preg0);382+ AscendC::MicroAPI::Reduce<MicroAPI::ReduceType::MIN>(vreg7_2, vreg3, preg0);
384 AscendC::MicroAPI::Max(vreg8_1, vreg8_1, vreg7_1, preg0);383 AscendC::MicroAPI::Max(vreg8_1, vreg8_1, vreg7_1, preg0);
385 AscendC::MicroAPI::Min(vreg8_2, vreg8_2, vreg7_2, preg0);384 AscendC::MicroAPI::Min(vreg8_2, vreg8_2, vreg7_2, preg0);
386 } else {385 } else {
387 AscendC::MicroAPI::Abs(vreg6_1, vreg3, preg0);386 AscendC::MicroAPI::Abs(vreg6_1, vreg3, preg0);
388 AscendC::MicroAPI::Muls(vreg6_1, vreg6_1, float(1.0) / maxValue, preg0);387 AscendC::MicroAPI::Muls(vreg6_1, vreg6_1, float(1.0) / maxValue, preg0);
389- AscendC::MicroAPI::ReduceMax(vreg7_1, vreg6_1, preg0);388+ AscendC::MicroAPI::Reduce<MicroAPI::ReduceType::MAX>(vreg7_1, vreg6_1, preg0);
390 AscendC::MicroAPI::Max(vreg8_1, vreg8_1, vreg7_1, preg0);389 AscendC::MicroAPI::Max(vreg8_1, vreg8_1, vreg7_1, preg0);
391 }390 }
392 }391 }
@@ -398,20 +397,19 @@ private:
398 }397 }
399 }398 }
400 if constexpr (isSymmertrical == false) {399 if constexpr (isSymmertrical == false) {
401- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::StoreDist::DIST_FIRST_ELEMENT_B32>(400+ AscendC::MicroAPI::StoreAlign<float, AscendC::MicroAPI::StoreDist::DIST_FIRST_ELEMENT_B32>(
402 maxLocalAddr, vreg9_1, preg0);401 maxLocalAddr, vreg9_1, preg0);
403- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::StoreDist::DIST_FIRST_ELEMENT_B32>(402+ AscendC::MicroAPI::StoreAlign<float, AscendC::MicroAPI::StoreDist::DIST_FIRST_ELEMENT_B32>(
404 minLocalAddr, vreg9_2, preg0);403 minLocalAddr, vreg9_2, preg0);
405 } else {404 } else {
406- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::StoreDist::DIST_FIRST_ELEMENT_B32>(405+ AscendC::MicroAPI::StoreAlign<float, AscendC::MicroAPI::StoreDist::DIST_FIRST_ELEMENT_B32>(
407 scaleLocalAddr, vreg9_1, preg0);406 scaleLocalAddr, vreg9_1, preg0);
408 }407 }
409 }408 }
410 }409 }
411- __aicore__ inline void ComputeMaxColScaleVF(__local_mem__ float* scaleLocalAddr,410+ __aicore__ inline void ComputeMaxColScaleVF(__ubuf__ float* scaleLocalAddr, __ubuf__ float* scaleOutLocalAddr,
412- __local_mem__ float* scaleOutLocalAddr,411+ __ubuf__ float* maxLocalAddr, __ubuf__ float* maxOutLocalAddr,
413- __local_mem__ float* maxLocalAddr, __local_mem__ float* maxOutLocalAddr,412+ __ubuf__ float* minLocalAddr, __ubuf__ float* minOutLocalAddr,
414- __local_mem__ float* minLocalAddr, __local_mem__ float* minOutLocalAddr,
415 uint32_t elementNum)413 uint32_t elementNum)
416 {414 {
417 uint32_t dtypeSize = sizeof(float);415 uint32_t dtypeSize = sizeof(float);
@@ -431,11 +429,13 @@ private:
431 AscendC::MicroAPI::RegTensor<float, MicroAPI::RegTraitNumOne> vreg3_2;429 AscendC::MicroAPI::RegTensor<float, MicroAPI::RegTraitNumOne> vreg3_2;
432 430 
433 if constexpr (isSymmertrical == false) {431 if constexpr (isSymmertrical == false) {
434- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_BRC_B32>(vreg1_1, maxOutLocalAddr);432+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_BRC_B32>(vreg1_1,
435- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_BRC_B32>(vreg1_2, minOutLocalAddr);433+ maxOutLocalAddr);
434+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_BRC_B32>(vreg1_2,
435+ minOutLocalAddr);
436 } else {436 } else {
437- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_BRC_B32>(vreg1_1,437+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_BRC_B32>(vreg1_1,
438- scaleOutLocalAddr);438+ scaleOutLocalAddr);
439 }439 }
440 440 
441 AscendC::MicroAPI::MaskReg441 AscendC::MicroAPI::MaskReg
@@ -445,62 +445,62 @@ private:
445 mask = AscendC::MicroAPI::UpdateMask<float>(maskNum);445 mask = AscendC::MicroAPI::UpdateMask<float>(maskNum);
446 446 
447 if constexpr (isSymmertrical == false) {447 if constexpr (isSymmertrical == false) {
448- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(vreg0_1,448+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(vreg0_1,
449- maxLocalAddr + i * VL);449+ maxLocalAddr + i * VL);
450 AscendC::MicroAPI::Max(vreg1_1, vreg0_1, vreg1_1, mask);450 AscendC::MicroAPI::Max(vreg1_1, vreg0_1, vreg1_1, mask);
451- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(vreg0_2,451+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(vreg0_2,
452- minLocalAddr + i * VL);452+ minLocalAddr + i * VL);
453 AscendC::MicroAPI::Min(vreg1_2, vreg0_2, vreg1_2, mask);453 AscendC::MicroAPI::Min(vreg1_2, vreg0_2, vreg1_2, mask);
454 } else {454 } else {
455- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(vreg0_1,455+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(
456- scaleLocalAddr + i * VL);456+ vreg0_1, scaleLocalAddr + i * VL);
457 AscendC::MicroAPI::Max(vreg1_1, vreg0_1, vreg1_1, mask);457 AscendC::MicroAPI::Max(vreg1_1, vreg0_1, vreg1_1, mask);
458 }458 }
459 }459 }
460 if constexpr (isSymmertrical == false) {460 if constexpr (isSymmertrical == false) {
461- AscendC::MicroAPI::ReduceMax<float>(vreg2_1, vreg1_1, mask);461+ AscendC::MicroAPI::Reduce<MicroAPI::ReduceType::MAX, float>(vreg2_1, vreg1_1, mask);
462- AscendC::MicroAPI::ReduceMin<float>(vreg2_2, vreg1_2, mask);462+ AscendC::MicroAPI::Reduce<MicroAPI::ReduceType::MIN, float>(vreg2_2, vreg1_2, mask);
463 } else {463 } else {
464- AscendC::MicroAPI::ReduceMax<float>(vreg2_1, vreg1_1, mask);464+ AscendC::MicroAPI::Reduce<MicroAPI::ReduceType::MAX, float>(vreg2_1, vreg1_1, mask);
465 }465 }
466 466 
467 for (uint16_t i = vfLoopNum - 1; i < vfLoopNum; i++) {467 for (uint16_t i = vfLoopNum - 1; i < vfLoopNum; i++) {
468 maskNum = elementNum - i * VL;468 maskNum = elementNum - i * VL;
469 mask = AscendC::MicroAPI::UpdateMask<float>(maskNum);469 mask = AscendC::MicroAPI::UpdateMask<float>(maskNum);
470 if constexpr (isSymmertrical == false) {470 if constexpr (isSymmertrical == false) {
471- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(vreg0_1,471+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(vreg0_1,
472- maxLocalAddr + i * VL);472+ maxLocalAddr + i * VL);
473 AscendC::MicroAPI::Max(vreg1_1, vreg0_1, vreg1_1, mask);473 AscendC::MicroAPI::Max(vreg1_1, vreg0_1, vreg1_1, mask);
474- AscendC::MicroAPI::ReduceMax<float>(vreg3_1, vreg1_1, mask);474+ AscendC::MicroAPI::Reduce<MicroAPI::ReduceType::MAX, float>(vreg3_1, vreg1_1, mask);
475 AscendC::MicroAPI::Max(vreg3_1, vreg2_1, vreg3_1, mask);475 AscendC::MicroAPI::Max(vreg3_1, vreg2_1, vreg3_1, mask);
476 476 
477- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(vreg0_2,477+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(vreg0_2,
478- minLocalAddr + i * VL);478+ minLocalAddr + i * VL);
479 AscendC::MicroAPI::Min(vreg1_2, vreg0_2, vreg1_2, mask);479 AscendC::MicroAPI::Min(vreg1_2, vreg0_2, vreg1_2, mask);
480- AscendC::MicroAPI::ReduceMin<float>(vreg3_2, vreg1_2, mask);480+ AscendC::MicroAPI::Reduce<MicroAPI::ReduceType::MIN, float>(vreg3_2, vreg1_2, mask);
481 AscendC::MicroAPI::Min(vreg3_2, vreg2_2, vreg3_2, mask);481 AscendC::MicroAPI::Min(vreg3_2, vreg2_2, vreg3_2, mask);
482 } else {482 } else {
483- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(vreg0_1,483+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(
484- scaleLocalAddr + i * VL);484+ vreg0_1, scaleLocalAddr + i * VL);
485 AscendC::MicroAPI::Max(vreg1_1, vreg0_1, vreg1_1, mask);485 AscendC::MicroAPI::Max(vreg1_1, vreg0_1, vreg1_1, mask);
486- AscendC::MicroAPI::ReduceMax<float>(vreg3_1, vreg1_1, mask);486+ AscendC::MicroAPI::Reduce<MicroAPI::ReduceType::MAX, float>(vreg3_1, vreg1_1, mask);
487 AscendC::MicroAPI::Max(vreg3_1, vreg2_1, vreg3_1, mask);487 AscendC::MicroAPI::Max(vreg3_1, vreg2_1, vreg3_1, mask);
488 }488 }
489 }489 }
490 if constexpr (isSymmertrical == false) {490 if constexpr (isSymmertrical == false) {
491- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::StoreDist::DIST_FIRST_ELEMENT_B32>(491+ AscendC::MicroAPI::StoreAlign<float, AscendC::MicroAPI::StoreDist::DIST_FIRST_ELEMENT_B32>(
492 maxOutLocalAddr, vreg3_1, mask);492 maxOutLocalAddr, vreg3_1, mask);
493- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::StoreDist::DIST_FIRST_ELEMENT_B32>(493+ AscendC::MicroAPI::StoreAlign<float, AscendC::MicroAPI::StoreDist::DIST_FIRST_ELEMENT_B32>(
494 minOutLocalAddr, vreg3_2, mask);494 minOutLocalAddr, vreg3_2, mask);
495 } else {495 } else {
496- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::StoreDist::DIST_FIRST_ELEMENT_B32>(496+ AscendC::MicroAPI::StoreAlign<float, AscendC::MicroAPI::StoreDist::DIST_FIRST_ELEMENT_B32>(
497 scaleOutLocalAddr, vreg3_1, mask);497 scaleOutLocalAddr, vreg3_1, mask);
498 }498 }
499 }499 }
500 }500 }
501 501 
502- __aicore__ inline void ComputeScaleSymVF(__local_mem__ float* maxLocalAddr, __local_mem__ float* minLocalAddr,502+ __aicore__ inline void ComputeScaleSymVF(__ubuf__ float* maxLocalAddr, __ubuf__ float* minLocalAddr,
503- __local_mem__ float* scaleLocalAddr, uint32_t elementNum)503+ __ubuf__ float* scaleLocalAddr, uint32_t elementNum)
504 {504 {
505 uint32_t dtypeSize = sizeof(float);505 uint32_t dtypeSize = sizeof(float);
506 uint32_t VL = AscendC::VECTOR_REG_WIDTH / dtypeSize;506 uint32_t VL = AscendC::VECTOR_REG_WIDTH / dtypeSize;
@@ -518,20 +518,20 @@ private:
518 maskNum = elementNum - i * VL;518 maskNum = elementNum - i * VL;
519 mask = AscendC::MicroAPI::UpdateMask<float>(maskNum);519 mask = AscendC::MicroAPI::UpdateMask<float>(maskNum);
520 520 
521- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(vreg0,521+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(vreg0,
522- maxLocalAddr + i * VL);522+ maxLocalAddr + i * VL);
523- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(vreg1,523+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(vreg1,
524- minLocalAddr + i * VL);524+ minLocalAddr + i * VL);
525 AscendC::MicroAPI::Sub(vreg1, vreg0, vreg1, mask);525 AscendC::MicroAPI::Sub(vreg1, vreg0, vreg1, mask);
526 AscendC::MicroAPI::Muls(vreg1, vreg1, float(1.0) / maxValueNoSym, mask);526 AscendC::MicroAPI::Muls(vreg1, vreg1, float(1.0) / maxValueNoSym, mask);
527- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::StoreDist::DIST_FIRST_ELEMENT_B32>(scaleLocalAddr,527+ AscendC::MicroAPI::StoreAlign<float, AscendC::MicroAPI::StoreDist::DIST_FIRST_ELEMENT_B32>(
528- vreg1, mask);528+ scaleLocalAddr, vreg1, mask);
529 }529 }
530 }530 }
531 }531 }
532 532 
533- __aicore__ inline void ComputeOffsetSymVF(__local_mem__ float* maxLocalAddr, __local_mem__ float* scaleLocalAddr,533+ __aicore__ inline void ComputeOffsetSymVF(__ubuf__ float* maxLocalAddr, __ubuf__ float* scaleLocalAddr,
534- __local_mem__ float* offsetLocalAddr, uint32_t elementNum)534+ __ubuf__ float* offsetLocalAddr, uint32_t elementNum)
535 {535 {
536 uint32_t dtypeSize = sizeof(float);536 uint32_t dtypeSize = sizeof(float);
537 uint32_t VL = AscendC::VECTOR_REG_WIDTH / dtypeSize;537 uint32_t VL = AscendC::VECTOR_REG_WIDTH / dtypeSize;
@@ -551,15 +551,15 @@ private:
551 maskNum = elementNum - i * VL;551 maskNum = elementNum - i * VL;
552 mask = AscendC::MicroAPI::UpdateMask<float>(maskNum);552 mask = AscendC::MicroAPI::UpdateMask<float>(maskNum);
553 553 
554- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(vreg0,554+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(vreg0,
555- maxLocalAddr + i * VL);555+ maxLocalAddr + i * VL);
556- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(vreg1,556+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(vreg1,
557- scaleLocalAddr + i * VL);557+ scaleLocalAddr + i * VL);
558 AscendC::MicroAPI::Div<float, &mode>(vreg1, vreg0, vreg1, mask);558 AscendC::MicroAPI::Div<float, &mode>(vreg1, vreg0, vreg1, mask);
559 AscendC::MicroAPI::Muls(vreg1, vreg1, float(-1.0), mask);559 AscendC::MicroAPI::Muls(vreg1, vreg1, float(-1.0), mask);
560 AscendC::MicroAPI::Adds(vreg1, vreg1, maxValue, mask);560 AscendC::MicroAPI::Adds(vreg1, vreg1, maxValue, mask);
561 561 
562- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::StoreDist::DIST_FIRST_ELEMENT_B32>(562+ AscendC::MicroAPI::StoreAlign<float, AscendC::MicroAPI::StoreDist::DIST_FIRST_ELEMENT_B32>(
563 offsetLocalAddr, vreg1, mask);563 offsetLocalAddr, vreg1, mask);
564 }564 }
565 }565 }
@@ -568,30 +568,30 @@ private:
568 __aicore__ inline void ComputeMaxRowScale(int32_t multiRow)568 __aicore__ inline void ComputeMaxRowScale(int32_t multiRow)
569 {569 {
570 LocalTensor<xDtype> inLocal = inQueue.template DeQue<xDtype>();570 LocalTensor<xDtype> inLocal = inQueue.template DeQue<xDtype>();
571- __local_mem__ xDtype* inLocalAddr = (__local_mem__ xDtype*)inLocal.GetPhyAddr();571+ __ubuf__ xDtype* inLocalAddr = (__ubuf__ xDtype*)inLocal.GetPhyAddr();
572 LocalTensor<xDtype> smoothLocal;572 LocalTensor<xDtype> smoothLocal;
573- __local_mem__ xDtype* smoothLocalAddr;573+ __ubuf__ xDtype* smoothLocalAddr;
574 574 
575 LocalTensor<float> maxToWorkSpaceLocal;575 LocalTensor<float> maxToWorkSpaceLocal;
576- __local_mem__ float* maxToWorkSpaceLocalAddr;576+ __ubuf__ float* maxToWorkSpaceLocalAddr;
577 LocalTensor<float> minToWorkSpaceLocal;577 LocalTensor<float> minToWorkSpaceLocal;
578- __local_mem__ float* minToWorkSpaceLocalAddr;578+ __ubuf__ float* minToWorkSpaceLocalAddr;
579 LocalTensor<float> scaleToWorkSpaceLocal;579 LocalTensor<float> scaleToWorkSpaceLocal;
580- __local_mem__ float* scaleToWorkSpaceLocalAddr;580+ __ubuf__ float* scaleToWorkSpaceLocalAddr;
581 581 
582 if constexpr (hasSmooth == 1) {582 if constexpr (hasSmooth == 1) {
583 smoothLocal = smoothQueue.template DeQue<xDtype>();583 smoothLocal = smoothQueue.template DeQue<xDtype>();
584- smoothLocalAddr = (__local_mem__ xDtype*)smoothLocal.GetPhyAddr();584+ smoothLocalAddr = (__ubuf__ xDtype*)smoothLocal.GetPhyAddr();
585 }585 }
586 if constexpr (isSymmertrical == false) {586 if constexpr (isSymmertrical == false) {
587 maxToWorkSpaceLocal = MaxToWorkSpaceQueue.template DeQue<float>();587 maxToWorkSpaceLocal = MaxToWorkSpaceQueue.template DeQue<float>();
588- maxToWorkSpaceLocalAddr = (__local_mem__ float*)maxToWorkSpaceLocal.GetPhyAddr();588+ maxToWorkSpaceLocalAddr = (__ubuf__ float*)maxToWorkSpaceLocal.GetPhyAddr();
589 589 
590 minToWorkSpaceLocal = MinToWorkSpaceQueue.template DeQue<float>();590 minToWorkSpaceLocal = MinToWorkSpaceQueue.template DeQue<float>();
591- minToWorkSpaceLocalAddr = (__local_mem__ float*)minToWorkSpaceLocal.GetPhyAddr();591+ minToWorkSpaceLocalAddr = (__ubuf__ float*)minToWorkSpaceLocal.GetPhyAddr();
592 } else {592 } else {
593 scaleToWorkSpaceLocal = scaleToWorkSpaceQueue.template DeQue<float>();593 scaleToWorkSpaceLocal = scaleToWorkSpaceQueue.template DeQue<float>();
594- scaleToWorkSpaceLocalAddr = (__local_mem__ float*)scaleToWorkSpaceLocal.GetPhyAddr();594+ scaleToWorkSpaceLocalAddr = (__ubuf__ float*)scaleToWorkSpaceLocal.GetPhyAddr();
595 }595 }
596 596 
597 ComputeMaxRowScaleVF(inLocalAddr, smoothLocalAddr, scaleToWorkSpaceLocalAddr, maxToWorkSpaceLocalAddr,597 ComputeMaxRowScaleVF(inLocalAddr, smoothLocalAddr, scaleToWorkSpaceLocalAddr, maxToWorkSpaceLocalAddr,
@@ -608,25 +608,24 @@ private:
608 inQueue.FreeTensor(inLocal);608 inQueue.FreeTensor(inLocal);
609 }609 }
610 610 
611- __aicore__ inline void ComputeMaxColScale(uint32_t elementNum, __local_mem__ float* maxAddr,611+ __aicore__ inline void ComputeMaxColScale(uint32_t elementNum, __ubuf__ float* maxAddr, __ubuf__ float* minAddr,
612- __local_mem__ float* minAddr, __local_mem__ float* scaleAddr,612+ __ubuf__ float* scaleAddr, __ubuf__ float* offsetAddr)
613- __local_mem__ float* offsetAddr)
614 {613 {
615 LocalTensor<float> scaleFromWorkSpaceLocal;614 LocalTensor<float> scaleFromWorkSpaceLocal;
616- __local_mem__ float* scaleFromWorkSpaceLocalAddr;615+ __ubuf__ float* scaleFromWorkSpaceLocalAddr;
617 616 
618 LocalTensor<float> maxFromWorkSpaceLocal;617 LocalTensor<float> maxFromWorkSpaceLocal;
619- __local_mem__ float* maxFromWorkSpaceLocalAddr;618+ __ubuf__ float* maxFromWorkSpaceLocalAddr;
620 619 
621 LocalTensor<float> minFromWorkSpaceLocal;620 LocalTensor<float> minFromWorkSpaceLocal;
622- __local_mem__ float* minFromWorkSpaceLocalAddr;621+ __ubuf__ float* minFromWorkSpaceLocalAddr;
623 622 
624 if constexpr (isSymmertrical == false) {623 if constexpr (isSymmertrical == false) {
625 maxFromWorkSpaceLocal = MaxFromWorkSpaceQueue.template DeQue<float>();624 maxFromWorkSpaceLocal = MaxFromWorkSpaceQueue.template DeQue<float>();
626- maxFromWorkSpaceLocalAddr = (__local_mem__ float*)maxFromWorkSpaceLocal.GetPhyAddr();625+ maxFromWorkSpaceLocalAddr = (__ubuf__ float*)maxFromWorkSpaceLocal.GetPhyAddr();
627 626 
628 minFromWorkSpaceLocal = MinFromWorkSpaceQueue.template DeQue<float>();627 minFromWorkSpaceLocal = MinFromWorkSpaceQueue.template DeQue<float>();
629- minFromWorkSpaceLocalAddr = (__local_mem__ float*)minFromWorkSpaceLocal.GetPhyAddr();628+ minFromWorkSpaceLocalAddr = (__ubuf__ float*)minFromWorkSpaceLocal.GetPhyAddr();
630 629 
631 ComputeMaxColScaleVF(scaleFromWorkSpaceLocalAddr, scaleAddr, maxFromWorkSpaceLocalAddr, maxAddr,630 ComputeMaxColScaleVF(scaleFromWorkSpaceLocalAddr, scaleAddr, maxFromWorkSpaceLocalAddr, maxAddr,
632 minFromWorkSpaceLocalAddr, minAddr, elementNum);631 minFromWorkSpaceLocalAddr, minAddr, elementNum);
@@ -637,7 +636,7 @@ private:
637 MinFromWorkSpaceQueue.FreeTensor(minFromWorkSpaceLocal);636 MinFromWorkSpaceQueue.FreeTensor(minFromWorkSpaceLocal);
638 } else {637 } else {
639 scaleFromWorkSpaceLocal = scaleFromWorkSpaceQueue.template DeQue<float>();638 scaleFromWorkSpaceLocal = scaleFromWorkSpaceQueue.template DeQue<float>();
640- scaleFromWorkSpaceLocalAddr = (__local_mem__ float*)scaleFromWorkSpaceLocal.GetPhyAddr();639+ scaleFromWorkSpaceLocalAddr = (__ubuf__ float*)scaleFromWorkSpaceLocal.GetPhyAddr();
641 640 
642 ComputeMaxColScaleVF(scaleFromWorkSpaceLocalAddr, scaleAddr, maxFromWorkSpaceLocalAddr, maxAddr,641 ComputeMaxColScaleVF(scaleFromWorkSpaceLocalAddr, scaleAddr, maxFromWorkSpaceLocalAddr, maxAddr,
643 minFromWorkSpaceLocalAddr, minAddr, elementNum);642 minFromWorkSpaceLocalAddr, minAddr, elementNum);
@@ -645,20 +644,20 @@ private:
645 }644 }
646 }645 }
647 646 
648- __aicore__ inline void ComputeY(int32_t multiRow, __local_mem__ float* scaleAddr, __local_mem__ float* offsetAddr)647+ __aicore__ inline void ComputeY(int32_t multiRow, __ubuf__ float* scaleAddr, __ubuf__ float* offsetAddr)
649 {648 {
650 uint32_t index = 0;649 uint32_t index = 0;
651 LocalTensor<yCopyDtype> yLocal = outQueue.template AllocTensor<yCopyDtype>();650 LocalTensor<yCopyDtype> yLocal = outQueue.template AllocTensor<yCopyDtype>();
652 LocalTensor<xDtype> xLocal = inQueue.template DeQue<xDtype>();651 LocalTensor<xDtype> xLocal = inQueue.template DeQue<xDtype>();
653 LocalTensor<xDtype> smoothLocal;652 LocalTensor<xDtype> smoothLocal;
654 653 
655- __local_mem__ xDtype* xAddr = (__local_mem__ xDtype*)xLocal.GetPhyAddr();654+ __ubuf__ xDtype* xAddr = (__ubuf__ xDtype*)xLocal.GetPhyAddr();
656- __local_mem__ yCopyDtype* yAddr = (__local_mem__ yCopyDtype*)yLocal.GetPhyAddr();655+ __ubuf__ yCopyDtype* yAddr = (__ubuf__ yCopyDtype*)yLocal.GetPhyAddr();
657- __local_mem__ xDtype* smoothAddr;656+ __ubuf__ xDtype* smoothAddr;
658 657 
659 if constexpr (hasSmooth) {658 if constexpr (hasSmooth) {
660 smoothLocal = smoothQueue.template DeQue<xDtype>();659 smoothLocal = smoothQueue.template DeQue<xDtype>();
661- smoothAddr = (__local_mem__ xDtype*)smoothLocal.GetPhyAddr();660+ smoothAddr = (__ubuf__ xDtype*)smoothLocal.GetPhyAddr();
662 }661 }
663 662 
664 ComputeYVF(xAddr, smoothAddr, yAddr, scaleAddr, offsetAddr, multiRow);663 ComputeYVF(xAddr, smoothAddr, yAddr, scaleAddr, offsetAddr, multiRow);
@@ -686,9 +685,8 @@ private:
686 outQueue.FreeTensor(yLocal);685 outQueue.FreeTensor(yLocal);
687 }686 }
688 687 
689- __aicore__ inline void ComputeYVF(__local_mem__ xDtype* xAddr, __local_mem__ xDtype* smoothAddr,688+ __aicore__ inline void ComputeYVF(__ubuf__ xDtype* xAddr, __ubuf__ xDtype* smoothAddr, __ubuf__ yCopyDtype* yAddr,
690- __local_mem__ yCopyDtype* yAddr, __local_mem__ float* scaleAddr,689+ __ubuf__ float* scaleAddr, __ubuf__ float* offsetAddr, int32_t multiRow)
691- __local_mem__ float* offsetAddr, int32_t multiRow)
692 {690 {
693 uint32_t dtypeSize = sizeof(float);691 uint32_t dtypeSize = sizeof(float);
694 uint16_t VL = AscendC::VECTOR_REG_WIDTH / dtypeSize;692 uint16_t VL = AscendC::VECTOR_REG_WIDTH / dtypeSize;
@@ -713,9 +711,9 @@ private:
713 AscendC::MicroAPI::MaskReg711 AscendC::MicroAPI::MaskReg
714 preg1 = AscendC::MicroAPI::CreateMask<float, AscendC::MicroAPI::MaskPattern::H>();712 preg1 = AscendC::MicroAPI::CreateMask<float, AscendC::MicroAPI::MaskPattern::H>();
715 713 
716- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_BRC_B32>(vreg_scale, scaleAddr);714+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_BRC_B32>(vreg_scale, scaleAddr);
717 if constexpr (isSymmertrical == false) {715 if constexpr (isSymmertrical == false) {
718- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_BRC_B32>(vreg_offset, offsetAddr);716+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_BRC_B32>(vreg_offset, offsetAddr);
719 }717 }
720 718 
721 for (uint16_t i = 0; i < static_cast<uint16_t>(multiRow); i++) {719 for (uint16_t i = 0; i < static_cast<uint16_t>(multiRow); i++) {
@@ -723,11 +721,11 @@ private:
723 for (uint16_t j = 0; j < vfLoop; j++) {721 for (uint16_t j = 0; j < vfLoop; j++) {
724 auto addr = yAddr + i * outAlignLen + j * VL;722 auto addr = yAddr + i * outAlignLen + j * VL;
725 preg0 = AscendC::MicroAPI::UpdateMask<float>(sreg0);723 preg0 = AscendC::MicroAPI::UpdateMask<float>(sreg0);
726- AscendC::MicroAPI::DataCopy<xDtype, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(724+ AscendC::MicroAPI::LoadAlign<xDtype, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(
727 vreg1, xAddr + i * rowCount + j * VL);725 vreg1, xAddr + i * rowCount + j * VL);
728 AscendC::MicroAPI::Cast<float, xDtype, castTraitB16ToB32>(vreg3, vreg1, preg0);726 AscendC::MicroAPI::Cast<float, xDtype, castTraitB16ToB32>(vreg3, vreg1, preg0);
729 if constexpr (hasSmooth) {727 if constexpr (hasSmooth) {
730- AscendC::MicroAPI::DataCopy<xDtype, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(728+ AscendC::MicroAPI::LoadAlign<xDtype, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(
731 vreg2, smoothAddr + j * VL);729 vreg2, smoothAddr + j * VL);
732 AscendC::MicroAPI::Cast<float, xDtype, castTraitB16ToB32>(vreg4, vreg2, preg0);730 AscendC::MicroAPI::Cast<float, xDtype, castTraitB16ToB32>(vreg4, vreg2, preg0);
733 AscendC::MicroAPI::Mul(vreg3, vreg3, vreg4, preg0);731 AscendC::MicroAPI::Mul(vreg3, vreg3, vreg4, preg0);
@@ -756,10 +754,10 @@ private:
756 addr = yAddr + (i * outAlignLen + j * VL) / 2;754 addr = yAddr + (i * outAlignLen + j * VL) / 2;
757 }755 }
758 if constexpr (IsSameType<yDtype, int4b_t>::value) {756 if constexpr (IsSameType<yDtype, int4b_t>::value) {
759- AscendC::MicroAPI::DataCopy<yCopyDtype, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(757+ AscendC::MicroAPI::StoreAlign<yCopyDtype, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(
760 addr, vreg8, preg1);758 addr, vreg8, preg1);
761 } else {759 } else {
762- AscendC::MicroAPI::DataCopy<yCopyDtype, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(760+ AscendC::MicroAPI::StoreAlign<yCopyDtype, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(
763 addr, vreg8, preg0);761 addr, vreg8, preg0);
764 }762 }
765 }763 }
Mquant/dynamic_quant/op_kernel/arch35/dynamic_quant_regbase_large_shape_db.h+61-61
@@ -63,19 +63,19 @@ private:
63 __aicore__ inline void SetNoSymMaxValue();63 __aicore__ inline void SetNoSymMaxValue();
64 __aicore__ inline void ProcessLoop(uint32_t i, uint32_t j);64 __aicore__ inline void ProcessLoop(uint32_t i, uint32_t j);
65 __aicore__ inline void CopyInByEle(int64_t offset, uint32_t loopIndex, uint32_t elementNum, uint8_t rightPadding);65 __aicore__ inline void CopyInByEle(int64_t offset, uint32_t loopIndex, uint32_t elementNum, uint8_t rightPadding);
66- __aicore__ inline void ComputeMaxScale(uint32_t elementNum, __local_mem__ float* scaleLocalAddr,66+ __aicore__ inline void ComputeMaxScale(uint32_t elementNum, __ubuf__ float* scaleLocalAddr,
67- __local_mem__ float* offsetLocalAddr);67+ __ubuf__ float* offsetLocalAddr);
68- __aicore__ inline void ComputeMaxScaleAndYTail(uint32_t elementNum, __local_mem__ float* scaleLocalAddr,68+ __aicore__ inline void ComputeMaxScaleAndYTail(uint32_t elementNum, __ubuf__ float* scaleLocalAddr,
69- __local_mem__ float* offsetLocalAddr);69+ __ubuf__ float* offsetLocalAddr);
70 template <bool isFinal = false>70 template <bool isFinal = false>
71- __aicore__ inline void ComputeMaxScaleVF(__local_mem__ T* inLocalAddr, __local_mem__ T* smoothLocalAddr,71+ __aicore__ inline void ComputeMaxScaleVF(__ubuf__ T* inLocalAddr, __ubuf__ T* smoothLocalAddr,
72- __local_mem__ float* maxLocalAddr, __local_mem__ float* minLocalAddr,72+ __ubuf__ float* maxLocalAddr, __ubuf__ float* minLocalAddr,
73 uint32_t elementNum);73 uint32_t elementNum);
74- __aicore__ inline void ComputeY(uint32_t elementNum, __local_mem__ float* scaleLocalAddr,74+ __aicore__ inline void ComputeY(uint32_t elementNum, __ubuf__ float* scaleLocalAddr,
75- __local_mem__ float* offsetLocalAddr);75+ __ubuf__ float* offsetLocalAddr);
76- __aicore__ inline void ComputeYVF(__local_mem__ T* inLocalAddr, __local_mem__ T* smoothLocalAddr,76+ __aicore__ inline void ComputeYVF(__ubuf__ T* inLocalAddr, __ubuf__ T* smoothLocalAddr,
77- __local_mem__ yCopyDtype* outAddr, __local_mem__ float* scaleLocalAddr,77+ __ubuf__ yCopyDtype* outAddr, __ubuf__ float* scaleLocalAddr,
78- __local_mem__ float* offsetLocalAddr, uint32_t elementNum);78+ __ubuf__ float* offsetLocalAddr, uint32_t elementNum);
79 __aicore__ inline void ParseTilingData(const DynamicQuantTilingDataArch35& tilingData);79 __aicore__ inline void ParseTilingData(const DynamicQuantTilingDataArch35& tilingData);
80 __aicore__ inline void CopyOutScale(int64_t offset);80 __aicore__ inline void CopyOutScale(int64_t offset);
81 __aicore__ inline void CopyOutY(int64_t offset, uint32_t element);81 __aicore__ inline void CopyOutY(int64_t offset, uint32_t element);
@@ -307,15 +307,15 @@ __aicore__ inline void DynamicQuantLargeShapeDb<T, yDtype, hasSmooth, isSymmetri
307 uint32_t j)307 uint32_t j)
308{308{
309 LocalTensor<float> scaleLocal = scaleQueue.AllocTensor<float>();309 LocalTensor<float> scaleLocal = scaleQueue.AllocTensor<float>();
310- __local_mem__ float* scaleLocalAddr = (__local_mem__ float*)scaleLocal.GetPhyAddr();310+ __ubuf__ float* scaleLocalAddr = (__ubuf__ float*)scaleLocal.GetPhyAddr();
311 LocalTensor<float> offsetLocal;311 LocalTensor<float> offsetLocal;
312- __local_mem__ float* offsetLocalAddr;312+ __ubuf__ float* offsetLocalAddr;
313 313 
314 if constexpr (isSymmetrical) {314 if constexpr (isSymmetrical) {
315 AscendC::Duplicate(scaleLocal, (float)0.0, 64, 1, 1, 8);315 AscendC::Duplicate(scaleLocal, (float)0.0, 64, 1, 1, 8);
316 } else {316 } else {
317 offsetLocal = offsetQueue.AllocTensor<float>();317 offsetLocal = offsetQueue.AllocTensor<float>();
318- offsetLocalAddr = (__local_mem__ float*)offsetLocal.GetPhyAddr();318+ offsetLocalAddr = (__ubuf__ float*)offsetLocal.GetPhyAddr();
319 AscendC::Duplicate(scaleLocal, MIN_FLOAT_VALUE, 64, 1, 1, 8);319 AscendC::Duplicate(scaleLocal, MIN_FLOAT_VALUE, 64, 1, 1, 8);
320 AscendC::Duplicate(offsetLocal, MAX_FLOAT_VALUE, 64, 1, 1, 8);320 AscendC::Duplicate(offsetLocal, MAX_FLOAT_VALUE, 64, 1, 1, 8);
321 }321 }
@@ -377,16 +377,16 @@ __aicore__ inline void DynamicQuantLargeShapeDb<T, yDtype, hasSmooth, isSymmetri
377 377 
378template <typename T, typename yDtype, int64_t hasSmooth, bool isSymmetrical>378template <typename T, typename yDtype, int64_t hasSmooth, bool isSymmetrical>
379__aicore__ inline void DynamicQuantLargeShapeDb<T, yDtype, hasSmooth, isSymmetrical>::ComputeMaxScale(379__aicore__ inline void DynamicQuantLargeShapeDb<T, yDtype, hasSmooth, isSymmetrical>::ComputeMaxScale(
380- uint32_t elementNum, __local_mem__ float* scaleLocalAddr, __local_mem__ float* offsetLocalAddr)380+ uint32_t elementNum, __ubuf__ float* scaleLocalAddr, __ubuf__ float* offsetLocalAddr)
381{381{
382 LocalTensor<T> inLocal = inQueue.DeQue<T>();382 LocalTensor<T> inLocal = inQueue.DeQue<T>();
383- __local_mem__ T* inLocalAddr = (__local_mem__ T*)inLocal.GetPhyAddr();383+ __ubuf__ T* inLocalAddr = (__ubuf__ T*)inLocal.GetPhyAddr();
384 LocalTensor<T> smoothLocal;384 LocalTensor<T> smoothLocal;
385- __local_mem__ T* smoothLocalAddr;385+ __ubuf__ T* smoothLocalAddr;
386 386 
387 if constexpr (hasSmooth == 1) {387 if constexpr (hasSmooth == 1) {
388 smoothLocal = smoothQueue.DeQue<T>();388 smoothLocal = smoothQueue.DeQue<T>();
389- smoothLocalAddr = (__local_mem__ T*)smoothLocal.GetPhyAddr();389+ smoothLocalAddr = (__ubuf__ T*)smoothLocal.GetPhyAddr();
390 }390 }
391 391 
392 ComputeMaxScaleVF(inLocalAddr, smoothLocalAddr, scaleLocalAddr, offsetLocalAddr, elementNum);392 ComputeMaxScaleVF(inLocalAddr, smoothLocalAddr, scaleLocalAddr, offsetLocalAddr, elementNum);
@@ -400,18 +400,18 @@ __aicore__ inline void DynamicQuantLargeShapeDb<T, yDtype, hasSmooth, isSymmetri
400 400 
401template <typename T, typename yDtype, int64_t hasSmooth, bool isSymmetrical>401template <typename T, typename yDtype, int64_t hasSmooth, bool isSymmetrical>
402__aicore__ inline void DynamicQuantLargeShapeDb<T, yDtype, hasSmooth, isSymmetrical>::ComputeMaxScaleAndYTail(402__aicore__ inline void DynamicQuantLargeShapeDb<T, yDtype, hasSmooth, isSymmetrical>::ComputeMaxScaleAndYTail(
403- uint32_t elementNum, __local_mem__ float* scaleLocalAddr, __local_mem__ float* offsetLocalAddr)403+ uint32_t elementNum, __ubuf__ float* scaleLocalAddr, __ubuf__ float* offsetLocalAddr)
404{404{
405 LocalTensor<T> inLocal = inQueue.DeQue<T>();405 LocalTensor<T> inLocal = inQueue.DeQue<T>();
406- __local_mem__ T* inLocalAddr = (__local_mem__ T*)inLocal.GetPhyAddr();406+ __ubuf__ T* inLocalAddr = (__ubuf__ T*)inLocal.GetPhyAddr();
407 LocalTensor<yCopyDtype> outLocal = outQueue.AllocTensor<yCopyDtype>();407 LocalTensor<yCopyDtype> outLocal = outQueue.AllocTensor<yCopyDtype>();
408- __local_mem__ yCopyDtype* outAddr = (__local_mem__ yCopyDtype*)outLocal.GetPhyAddr();408+ __ubuf__ yCopyDtype* outAddr = (__ubuf__ yCopyDtype*)outLocal.GetPhyAddr();
409 LocalTensor<T> smoothLocal;409 LocalTensor<T> smoothLocal;
410- __local_mem__ T* smoothLocalAddr;410+ __ubuf__ T* smoothLocalAddr;
411 411 
412 if constexpr (hasSmooth == 1) {412 if constexpr (hasSmooth == 1) {
413 smoothLocal = smoothQueue.DeQue<T>();413 smoothLocal = smoothQueue.DeQue<T>();
414- smoothLocalAddr = (__local_mem__ T*)smoothLocal.GetPhyAddr();414+ smoothLocalAddr = (__ubuf__ T*)smoothLocal.GetPhyAddr();
415 }415 }
416 416 
417 ComputeMaxScaleVF<true>(inLocalAddr, smoothLocalAddr, scaleLocalAddr, offsetLocalAddr, elementNum);417 ComputeMaxScaleVF<true>(inLocalAddr, smoothLocalAddr, scaleLocalAddr, offsetLocalAddr, elementNum);
@@ -428,8 +428,8 @@ __aicore__ inline void DynamicQuantLargeShapeDb<T, yDtype, hasSmooth, isSymmetri
428template <typename T, typename yDtype, int64_t hasSmooth, bool isSymmetrical>428template <typename T, typename yDtype, int64_t hasSmooth, bool isSymmetrical>
429template <bool isFinal>429template <bool isFinal>
430__aicore__ inline void DynamicQuantLargeShapeDb<T, yDtype, hasSmooth, isSymmetrical>::ComputeMaxScaleVF(430__aicore__ inline void DynamicQuantLargeShapeDb<T, yDtype, hasSmooth, isSymmetrical>::ComputeMaxScaleVF(
431- __local_mem__ T* inLocalAddr, __local_mem__ T* smoothLocalAddr, __local_mem__ float* maxLocalAddr,431+ __ubuf__ T* inLocalAddr, __ubuf__ T* smoothLocalAddr, __ubuf__ float* maxLocalAddr, __ubuf__ float* minLocalAddr,
432- __local_mem__ float* minLocalAddr, uint32_t elementNum)432+ uint32_t elementNum)
433{433{
434 uint32_t dtypeSize = sizeof(float);434 uint32_t dtypeSize = sizeof(float);
435 uint32_t VL = AscendC::VECTOR_REG_WIDTH / dtypeSize;435 uint32_t VL = AscendC::VECTOR_REG_WIDTH / dtypeSize;
@@ -452,9 +452,9 @@ __aicore__ inline void DynamicQuantLargeShapeDb<T, yDtype, hasSmooth, isSymmetri
452 AscendC::MicroAPI::RegTensor<float, MicroAPI::RegTraitNumOne> vregMinTail;452 AscendC::MicroAPI::RegTensor<float, MicroAPI::RegTraitNumOne> vregMinTail;
453 static constexpr AscendC::MicroAPI::DivSpecificMode mode = {AscendC::MicroAPI::MaskMergeMode::ZEROING, true};453 static constexpr AscendC::MicroAPI::DivSpecificMode mode = {AscendC::MicroAPI::MaskMergeMode::ZEROING, true};
454 454 
455- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_BRC_B32>(vreg6, maxLocalAddr);455+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_BRC_B32>(vreg6, maxLocalAddr);
456 if constexpr (!isSymmetrical) {456 if constexpr (!isSymmetrical) {
457- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_BRC_B32>(vreg8, minLocalAddr);457+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_BRC_B32>(vreg8, minLocalAddr);
458 }458 }
459 459 
460 AscendC::MicroAPI::MaskReg mask;460 AscendC::MicroAPI::MaskReg mask;
@@ -462,11 +462,11 @@ __aicore__ inline void DynamicQuantLargeShapeDb<T, yDtype, hasSmooth, isSymmetri
462 maskAll = AscendC::MicroAPI::CreateMask<float, AscendC::MicroAPI::MaskPattern::ALL>();462 maskAll = AscendC::MicroAPI::CreateMask<float, AscendC::MicroAPI::MaskPattern::ALL>();
463 463 
464 for (uint16_t i = 0; i < static_cast<uint16_t>(vfLoopNum - 1); i++) {464 for (uint16_t i = 0; i < static_cast<uint16_t>(vfLoopNum - 1); i++) {
465- AscendC::MicroAPI::DataCopy<T, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(vreg0, inLocalAddr + i * VL);465+ AscendC::MicroAPI::LoadAlign<T, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(vreg0, inLocalAddr + i * VL);
466 AscendC::MicroAPI::Cast<float, T, castTrait0>(vreg1, vreg0, maskAll);466 AscendC::MicroAPI::Cast<float, T, castTrait0>(vreg1, vreg0, maskAll);
467 if constexpr (hasSmooth == 1) {467 if constexpr (hasSmooth == 1) {
468- AscendC::MicroAPI::DataCopy<T, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(vreg2,468+ AscendC::MicroAPI::LoadAlign<T, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(vreg2,
469- smoothLocalAddr + i * VL);469+ smoothLocalAddr + i * VL);
470 AscendC::MicroAPI::Cast<float, T, castTrait0>(vreg3, vreg2, maskAll);470 AscendC::MicroAPI::Cast<float, T, castTrait0>(vreg3, vreg2, maskAll);
471 AscendC::MicroAPI::Mul(vreg1, vreg1, vreg3, maskAll);471 AscendC::MicroAPI::Mul(vreg1, vreg1, vreg3, maskAll);
472 }472 }
@@ -482,17 +482,17 @@ __aicore__ inline void DynamicQuantLargeShapeDb<T, yDtype, hasSmooth, isSymmetri
482 }482 }
483 {483 {
484 if constexpr (isSymmetrical) {484 if constexpr (isSymmetrical) {
485- AscendC::MicroAPI::ReduceMax<float>(vregMax, vreg6, maskAll);485+ AscendC::MicroAPI::Reduce<MicroAPI::ReduceType::MAX, float>(vregMax, vreg6, maskAll);
486 } else {486 } else {
487- AscendC::MicroAPI::ReduceMax<float>(vregMax, vreg6, maskAll);487+ AscendC::MicroAPI::Reduce<MicroAPI::ReduceType::MAX, float>(vregMax, vreg6, maskAll);
488- AscendC::MicroAPI::ReduceMin<float>(vregMin, vreg8, maskAll);488+ AscendC::MicroAPI::Reduce<MicroAPI::ReduceType::MIN, float>(vregMin, vreg8, maskAll);
489 }489 }
490 mask = AscendC::MicroAPI::UpdateMask<float>(tailNum);490 mask = AscendC::MicroAPI::UpdateMask<float>(tailNum);
491- AscendC::MicroAPI::DataCopy<T, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(491+ AscendC::MicroAPI::LoadAlign<T, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(
492 vreg0, inLocalAddr + (vfLoopNum - 1) * VL);492 vreg0, inLocalAddr + (vfLoopNum - 1) * VL);
493 AscendC::MicroAPI::Cast<float, T, castTrait0>(vreg1, vreg0, mask);493 AscendC::MicroAPI::Cast<float, T, castTrait0>(vreg1, vreg0, mask);
494 if constexpr (hasSmooth == 1) {494 if constexpr (hasSmooth == 1) {
495- AscendC::MicroAPI::DataCopy<T, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(495+ AscendC::MicroAPI::LoadAlign<T, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(
496 vreg2, smoothLocalAddr + (vfLoopNum - 1) * VL);496 vreg2, smoothLocalAddr + (vfLoopNum - 1) * VL);
497 AscendC::MicroAPI::Cast<float, T, castTrait0>(vreg3, vreg2, mask);497 AscendC::MicroAPI::Cast<float, T, castTrait0>(vreg3, vreg2, mask);
498 AscendC::MicroAPI::Mul(vreg1, vreg1, vreg3, mask);498 AscendC::MicroAPI::Mul(vreg1, vreg1, vreg3, mask);
@@ -501,21 +501,21 @@ __aicore__ inline void DynamicQuantLargeShapeDb<T, yDtype, hasSmooth, isSymmetri
501 AscendC::MicroAPI::Abs(vreg5, vreg1, mask);501 AscendC::MicroAPI::Abs(vreg5, vreg1, mask);
502 AscendC::MicroAPI::Muls(vreg5, vreg5, scaleMaxValue, mask);502 AscendC::MicroAPI::Muls(vreg5, vreg5, scaleMaxValue, mask);
503 AscendC::MicroAPI::Max(vreg6, vreg5, vreg6, mask);503 AscendC::MicroAPI::Max(vreg6, vreg5, vreg6, mask);
504- AscendC::MicroAPI::ReduceMax<float>(vregMaxTail, vreg6, mask);504+ AscendC::MicroAPI::Reduce<MicroAPI::ReduceType::MAX, float>(vregMaxTail, vreg6, mask);
505 AscendC::MicroAPI::Max(vregMax, vregMax, vregMaxTail, mask);505 AscendC::MicroAPI::Max(vregMax, vregMax, vregMaxTail, mask);
506 } else {506 } else {
507 AscendC::MicroAPI::Max(vreg6, vreg1, vreg6, mask);507 AscendC::MicroAPI::Max(vreg6, vreg1, vreg6, mask);
508 AscendC::MicroAPI::Min(vreg8, vreg1, vreg8, mask);508 AscendC::MicroAPI::Min(vreg8, vreg1, vreg8, mask);
509- AscendC::MicroAPI::ReduceMax<float>(vregMaxTail, vreg6, mask);509+ AscendC::MicroAPI::Reduce<MicroAPI::ReduceType::MAX, float>(vregMaxTail, vreg6, mask);
510- AscendC::MicroAPI::ReduceMin<float>(vregMinTail, vreg8, mask);510+ AscendC::MicroAPI::Reduce<MicroAPI::ReduceType::MIN, float>(vregMinTail, vreg8, mask);
511 AscendC::MicroAPI::Max(vregMax, vregMax, vregMaxTail, mask);511 AscendC::MicroAPI::Max(vregMax, vregMax, vregMaxTail, mask);
512 AscendC::MicroAPI::Min(vregMin, vregMin, vregMinTail, mask);512 AscendC::MicroAPI::Min(vregMin, vregMin, vregMinTail, mask);
513 }513 }
514 }514 }
515 515 
516 if constexpr (isSymmetrical) {516 if constexpr (isSymmetrical) {
517- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::StoreDist::DIST_FIRST_ELEMENT_B32>(maxLocalAddr,517+ AscendC::MicroAPI::StoreAlign<float, AscendC::MicroAPI::StoreDist::DIST_FIRST_ELEMENT_B32>(
518- vregMax, maskAll);518+ maxLocalAddr, vregMax, maskAll);
519 } else {519 } else {
520 if constexpr (isFinal) {520 if constexpr (isFinal) {
521 AscendC::MicroAPI::RegTensor<float, MicroAPI::RegTraitNumOne> vreg9;521 AscendC::MicroAPI::RegTensor<float, MicroAPI::RegTraitNumOne> vreg9;
@@ -530,14 +530,14 @@ __aicore__ inline void DynamicQuantLargeShapeDb<T, yDtype, hasSmooth, isSymmetri
530 AscendC::MicroAPI::Muls(vreg12, vreg11, -1, maskAll);530 AscendC::MicroAPI::Muls(vreg12, vreg11, -1, maskAll);
531 AscendC::MicroAPI::Adds(vreg13, vreg12, offsetMaxValue, maskAll); // offset531 AscendC::MicroAPI::Adds(vreg13, vreg12, offsetMaxValue, maskAll); // offset
532 532 
533- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::StoreDist::DIST_FIRST_ELEMENT_B32>(533+ AscendC::MicroAPI::StoreAlign<float, AscendC::MicroAPI::StoreDist::DIST_FIRST_ELEMENT_B32>(
534 maxLocalAddr, vreg10, maskAll);534 maxLocalAddr, vreg10, maskAll);
535- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::StoreDist::DIST_FIRST_ELEMENT_B32>(535+ AscendC::MicroAPI::StoreAlign<float, AscendC::MicroAPI::StoreDist::DIST_FIRST_ELEMENT_B32>(
536 minLocalAddr, vreg13, maskAll);536 minLocalAddr, vreg13, maskAll);
537 } else {537 } else {
538- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::StoreDist::DIST_FIRST_ELEMENT_B32>(538+ AscendC::MicroAPI::StoreAlign<float, AscendC::MicroAPI::StoreDist::DIST_FIRST_ELEMENT_B32>(
539 maxLocalAddr, vregMax, maskAll);539 maxLocalAddr, vregMax, maskAll);
540- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::StoreDist::DIST_FIRST_ELEMENT_B32>(540+ AscendC::MicroAPI::StoreAlign<float, AscendC::MicroAPI::StoreDist::DIST_FIRST_ELEMENT_B32>(
541 minLocalAddr, vregMin, maskAll);541 minLocalAddr, vregMin, maskAll);
542 }542 }
543 }543 }
@@ -546,18 +546,18 @@ __aicore__ inline void DynamicQuantLargeShapeDb<T, yDtype, hasSmooth, isSymmetri
546 546 
547template <typename T, typename yDtype, int64_t hasSmooth, bool isSymmetrical>547template <typename T, typename yDtype, int64_t hasSmooth, bool isSymmetrical>
548__aicore__ inline void DynamicQuantLargeShapeDb<T, yDtype, hasSmooth, isSymmetrical>::ComputeY(548__aicore__ inline void DynamicQuantLargeShapeDb<T, yDtype, hasSmooth, isSymmetrical>::ComputeY(
549- uint32_t elementNum, __local_mem__ float* scaleLocalAddr, __local_mem__ float* offsetLocalAddr)549+ uint32_t elementNum, __ubuf__ float* scaleLocalAddr, __ubuf__ float* offsetLocalAddr)
550{550{
551 LocalTensor<T> inLocal = inQueue.DeQue<T>();551 LocalTensor<T> inLocal = inQueue.DeQue<T>();
552- __local_mem__ T* inLocalAddr = (__local_mem__ T*)inLocal.GetPhyAddr();552+ __ubuf__ T* inLocalAddr = (__ubuf__ T*)inLocal.GetPhyAddr();
553 LocalTensor<yCopyDtype> outLocal = outQueue.AllocTensor<yCopyDtype>();553 LocalTensor<yCopyDtype> outLocal = outQueue.AllocTensor<yCopyDtype>();
554- __local_mem__ yCopyDtype* outAddr = (__local_mem__ yCopyDtype*)outLocal.GetPhyAddr();554+ __ubuf__ yCopyDtype* outAddr = (__ubuf__ yCopyDtype*)outLocal.GetPhyAddr();
555 LocalTensor<T> smoothLocal;555 LocalTensor<T> smoothLocal;
556- __local_mem__ T* smoothLocalAddr;556+ __ubuf__ T* smoothLocalAddr;
557 557 
558 if constexpr (hasSmooth == 1) {558 if constexpr (hasSmooth == 1) {
559 smoothLocal = smoothQueue.DeQue<T>();559 smoothLocal = smoothQueue.DeQue<T>();
560- smoothLocalAddr = (__local_mem__ T*)smoothLocal.GetPhyAddr();560+ smoothLocalAddr = (__ubuf__ T*)smoothLocal.GetPhyAddr();
561 }561 }
562 562 
563 ComputeYVF(inLocalAddr, smoothLocalAddr, outAddr, scaleLocalAddr, offsetLocalAddr, elementNum);563 ComputeYVF(inLocalAddr, smoothLocalAddr, outAddr, scaleLocalAddr, offsetLocalAddr, elementNum);
@@ -572,8 +572,8 @@ __aicore__ inline void DynamicQuantLargeShapeDb<T, yDtype, hasSmooth, isSymmetri
572 572 
573template <typename T, typename yDtype, int64_t hasSmooth, bool isSymmetrical>573template <typename T, typename yDtype, int64_t hasSmooth, bool isSymmetrical>
574__aicore__ inline void DynamicQuantLargeShapeDb<T, yDtype, hasSmooth, isSymmetrical>::ComputeYVF(574__aicore__ inline void DynamicQuantLargeShapeDb<T, yDtype, hasSmooth, isSymmetrical>::ComputeYVF(
575- __local_mem__ T* inLocalAddr, __local_mem__ T* smoothLocalAddr, __local_mem__ yCopyDtype* outAddr,575+ __ubuf__ T* inLocalAddr, __ubuf__ T* smoothLocalAddr, __ubuf__ yCopyDtype* outAddr, __ubuf__ float* scaleLocalAddr,
576- __local_mem__ float* scaleLocalAddr, __local_mem__ float* offsetLocalAddr, uint32_t elementNum)576+ __ubuf__ float* offsetLocalAddr, uint32_t elementNum)
577{577{
578 uint32_t dtypeSize = sizeof(float);578 uint32_t dtypeSize = sizeof(float);
579 uint32_t VL = AscendC::VECTOR_REG_WIDTH / dtypeSize;579 uint32_t VL = AscendC::VECTOR_REG_WIDTH / dtypeSize;
@@ -592,18 +592,18 @@ __aicore__ inline void DynamicQuantLargeShapeDb<T, yDtype, hasSmooth, isSymmetri
592 AscendC::MicroAPI::RegTensor<yCopyDtype, MicroAPI::RegTraitNumOne> vreg8;592 AscendC::MicroAPI::RegTensor<yCopyDtype, MicroAPI::RegTraitNumOne> vreg8;
593 AscendC::MicroAPI::RegTensor<float, MicroAPI::RegTraitNumOne> vreg9;593 AscendC::MicroAPI::RegTensor<float, MicroAPI::RegTraitNumOne> vreg9;
594 594 
595- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_BRC_B32>(vreg9, scaleLocalAddr);595+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_BRC_B32>(vreg9, scaleLocalAddr);
596 596 
597 AscendC::MicroAPI::MaskReg mask = AscendC::MicroAPI::CreateMask<float, AscendC::MicroAPI::MaskPattern::ALL>();597 AscendC::MicroAPI::MaskReg mask = AscendC::MicroAPI::CreateMask<float, AscendC::MicroAPI::MaskPattern::ALL>();
598 AscendC::MicroAPI::MaskReg mask2 = AscendC::MicroAPI::CreateMask<float, AscendC::MicroAPI::MaskPattern::H>();598 AscendC::MicroAPI::MaskReg mask2 = AscendC::MicroAPI::CreateMask<float, AscendC::MicroAPI::MaskPattern::H>();
599 for (uint16_t i = 0; i < vfLoopNum; i++) {599 for (uint16_t i = 0; i < vfLoopNum; i++) {
600 auto addr = outAddr + i * VL;600 auto addr = outAddr + i * VL;
601- AscendC::MicroAPI::DataCopy<T, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(vreg0, inLocalAddr + i * VL);601+ AscendC::MicroAPI::LoadAlign<T, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(vreg0, inLocalAddr + i * VL);
602 AscendC::MicroAPI::Cast<float, T, castTrait0>(vreg1, vreg0, mask);602 AscendC::MicroAPI::Cast<float, T, castTrait0>(vreg1, vreg0, mask);
603 603 
604 if constexpr (hasSmooth == 1) {604 if constexpr (hasSmooth == 1) {
605- AscendC::MicroAPI::DataCopy<T, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(vreg2,605+ AscendC::MicroAPI::LoadAlign<T, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(vreg2,
606- smoothLocalAddr + i * VL);606+ smoothLocalAddr + i * VL);
607 AscendC::MicroAPI::Cast<float, T, castTrait0>(vreg3, vreg2, mask);607 AscendC::MicroAPI::Cast<float, T, castTrait0>(vreg3, vreg2, mask);
608 AscendC::MicroAPI::Mul(vreg4, vreg1, vreg3, mask);608 AscendC::MicroAPI::Mul(vreg4, vreg1, vreg3, mask);
609 AscendC::MicroAPI::Div(vreg5, vreg4, vreg9, mask);609 AscendC::MicroAPI::Div(vreg5, vreg4, vreg9, mask);
@@ -612,8 +612,8 @@ __aicore__ inline void DynamicQuantLargeShapeDb<T, yDtype, hasSmooth, isSymmetri
612 }612 }
613 if constexpr (!isSymmetrical) {613 if constexpr (!isSymmetrical) {
614 AscendC::MicroAPI::RegTensor<float, MicroAPI::RegTraitNumOne> vregOffset;614 AscendC::MicroAPI::RegTensor<float, MicroAPI::RegTraitNumOne> vregOffset;
615- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_BRC_B32>(vregOffset,615+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_BRC_B32>(vregOffset,
616- offsetLocalAddr);616+ offsetLocalAddr);
617 AscendC::MicroAPI::Add(vreg5, vreg5, vregOffset, mask);617 AscendC::MicroAPI::Add(vreg5, vreg5, vregOffset, mask);
618 }618 }
619 619 
@@ -636,11 +636,11 @@ __aicore__ inline void DynamicQuantLargeShapeDb<T, yDtype, hasSmooth, isSymmetri
636 }636 }
637 637 
638 if constexpr (IsSameType<yDtype, int4b_t>::value) {638 if constexpr (IsSameType<yDtype, int4b_t>::value) {
639- AscendC::MicroAPI::DataCopy<yCopyDtype, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(addr, vreg8,639+ AscendC::MicroAPI::StoreAlign<yCopyDtype, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(addr, vreg8,
640- mask2);640+ mask2);
641 } else {641 } else {
642- AscendC::MicroAPI::DataCopy<yCopyDtype, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(addr, vreg8,642+ AscendC::MicroAPI::StoreAlign<yCopyDtype, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(addr, vreg8,
643- mask);643+ mask);
644 }644 }
645 }645 }
646 }646 }
Mquant/dynamic_quant/op_kernel/arch35/dynamic_quant_regbase_large_shape_db_pertensor.h+124-130
@@ -61,31 +61,27 @@ private:
61 __aicore__ inline void ProcessScaleRowLoop(uint32_t i, uint32_t j);61 __aicore__ inline void ProcessScaleRowLoop(uint32_t i, uint32_t j);
62 __aicore__ inline void ProcessScaleRow();62 __aicore__ inline void ProcessScaleRow();
63 __aicore__ inline void ProcessScaleCol();63 __aicore__ inline void ProcessScaleCol();
64- __aicore__ inline void ProcessYRow(uint32_t i, uint32_t j, __local_mem__ float* scaleAddr,64+ __aicore__ inline void ProcessYRow(uint32_t i, uint32_t j, __ubuf__ float* scaleAddr, __ubuf__ float* offsetAddr);
65- __local_mem__ float* offsetAddr);
66 __aicore__ inline void CopyInByEle(int64_t offset, uint32_t loopIndex, uint32_t elementNum, uint8_t rightPadding);65 __aicore__ inline void CopyInByEle(int64_t offset, uint32_t loopIndex, uint32_t elementNum, uint8_t rightPadding);
67 __aicore__ inline void CopyInScaleByEle(int64_t offset, uint32_t elementNum);66 __aicore__ inline void CopyInScaleByEle(int64_t offset, uint32_t elementNum);
68 __aicore__ inline void ComputeMaxRowScale(uint32_t elementNum);67 __aicore__ inline void ComputeMaxRowScale(uint32_t elementNum);
69- __aicore__ inline void ComputeMaxColScale(uint32_t elementNum, __local_mem__ float* maxAddr,68+ __aicore__ inline void ComputeMaxColScale(uint32_t elementNum, __ubuf__ float* maxAddr, __ubuf__ float* minAddr,
70- __local_mem__ float* minAddr, __local_mem__ float* scaleAddr,69+ __ubuf__ float* scaleAddr, __ubuf__ float* offsetAddr);
71- __local_mem__ float* offsetAddr);70+ __aicore__ inline void ComputeMaxRowScaleVF(__ubuf__ T* inLocalAddr, __ubuf__ T* smoothLocalAddr,
72- __aicore__ inline void ComputeMaxRowScaleVF(__local_mem__ T* inLocalAddr, __local_mem__ T* smoothLocalAddr,71+ __ubuf__ float* scaleLocalAddr, __ubuf__ float* maxLocalAddr,
73- __local_mem__ float* scaleLocalAddr, __local_mem__ float* maxLocalAddr,72+ __ubuf__ float* minLocalAddr, uint32_t elementNum);
74- __local_mem__ float* minLocalAddr, uint32_t elementNum);73+ __aicore__ inline void ComputeMaxColScaleVF(__ubuf__ float* scaleLocalAddr, __ubuf__ float* scaleOutLocalAddr,
75- __aicore__ inline void ComputeMaxColScaleVF(__local_mem__ float* scaleLocalAddr,74+ __ubuf__ float* maxLocalAddr, __ubuf__ float* maxOutLocalAddr,
76- __local_mem__ float* scaleOutLocalAddr,75+ __ubuf__ float* minLocalAddr, __ubuf__ float* minOutLocalAddr,
77- __local_mem__ float* maxLocalAddr, __local_mem__ float* maxOutLocalAddr,
78- __local_mem__ float* minLocalAddr, __local_mem__ float* minOutLocalAddr,
79 uint32_t elementNum);76 uint32_t elementNum);
80- __aicore__ inline void ComputeY(uint32_t elementNum, __local_mem__ float* scaleAddr,77+ __aicore__ inline void ComputeY(uint32_t elementNum, __ubuf__ float* scaleAddr, __ubuf__ float* offsetAddr);
81- __local_mem__ float* offsetAddr);78+ __aicore__ inline void ComputeScaleSymVF(__ubuf__ float* maxLocalAddr, __ubuf__ float* minLocalAddr,
82- __aicore__ inline void ComputeScaleSymVF(__local_mem__ float* maxLocalAddr, __local_mem__ float* minLocalAddr,79+ __ubuf__ float* scaleLocalAddr, uint32_t elementNum);
83- __local_mem__ float* scaleLocalAddr, uint32_t elementNum);80+ __aicore__ inline void ComputeOffsetSymVF(__ubuf__ float* maxLocalAddr, __ubuf__ float* scaleLocalAddr,
84- __aicore__ inline void ComputeOffsetSymVF(__local_mem__ float* maxLocalAddr, __local_mem__ float* scaleLocalAddr,81+ __ubuf__ float* offsetLocalAddr, uint32_t elementNum);
85- __local_mem__ float* offsetLocalAddr, uint32_t elementNum);82+ __aicore__ inline void ComputeYVF(__ubuf__ T* inLocalAddr, __ubuf__ T* smoothLocalAddr,
86- __aicore__ inline void ComputeYVF(__local_mem__ T* inLocalAddr, __local_mem__ T* smoothLocalAddr,83+ __ubuf__ yCopyDtype* outAddr, __ubuf__ float* scaleLocalAddr,
87- __local_mem__ yCopyDtype* outAddr, __local_mem__ float* scaleLocalAddr,84+ __ubuf__ float* offsetLocalAddr, uint32_t elementNum);
88- __local_mem__ float* offsetLocalAddr, uint32_t elementNum);
89 __aicore__ inline void ParseTilingData(const DynamicQuantTilingDataArch35& tilingData);85 __aicore__ inline void ParseTilingData(const DynamicQuantTilingDataArch35& tilingData);
90 __aicore__ inline void CopyOutY(int64_t offset, uint32_t element);86 __aicore__ inline void CopyOutY(int64_t offset, uint32_t element);
91 __aicore__ inline void CopyUB2Workspace(int64_t size);87 __aicore__ inline void CopyUB2Workspace(int64_t size);
@@ -298,14 +294,14 @@ template <typename T, typename yDtype, int64_t hasSmooth, bool isSymmertrical>
298__aicore__ inline void DynamicQuantLargeShapeDbPertensor<T, yDtype, hasSmooth, isSymmertrical>::ProcessY()294__aicore__ inline void DynamicQuantLargeShapeDbPertensor<T, yDtype, hasSmooth, isSymmertrical>::ProcessY()
299{295{
300 LocalTensor<float> scaleOutLocal = scaleOutQueue.DeQue<float>();296 LocalTensor<float> scaleOutLocal = scaleOutQueue.DeQue<float>();
301- __local_mem__ float* scaleOutLocalAddr = (__local_mem__ float*)scaleOutLocal.GetPhyAddr();297+ __ubuf__ float* scaleOutLocalAddr = (__ubuf__ float*)scaleOutLocal.GetPhyAddr();
302 298 
303 LocalTensor<float> offsetLocal;299 LocalTensor<float> offsetLocal;
304- __local_mem__ float* offsetLocalAddr;300+ __ubuf__ float* offsetLocalAddr;
305 301 
306 if constexpr (isSymmertrical == false) {302 if constexpr (isSymmertrical == false) {
307 offsetLocal = offsetQueue.DeQue<float>();303 offsetLocal = offsetQueue.DeQue<float>();
308- offsetLocalAddr = (__local_mem__ float*)offsetLocal.GetPhyAddr();304+ offsetLocalAddr = (__ubuf__ float*)offsetLocal.GetPhyAddr();
309 }305 }
310 306 
311 for (uint32_t i = 0; i < loopCntHead; i++) {307 for (uint32_t i = 0; i < loopCntHead; i++) {
@@ -397,31 +393,31 @@ __aicore__ inline void DynamicQuantLargeShapeDbPertensor<T, yDtype, hasSmooth, i
397 LocalTensor<float> MinOutLocal;393 LocalTensor<float> MinOutLocal;
398 LocalTensor<float> scaleOutLocal;394 LocalTensor<float> scaleOutLocal;
399 LocalTensor<float> offsetLocal;395 LocalTensor<float> offsetLocal;
400- __local_mem__ float* MaxOutLocalAddr;396+ __ubuf__ float* MaxOutLocalAddr;
401- __local_mem__ float* MinOutLocalAddr;397+ __ubuf__ float* MinOutLocalAddr;
402- __local_mem__ float* scaleOutLocalAddr;398+ __ubuf__ float* scaleOutLocalAddr;
403- __local_mem__ float* offsetLocalAddr;399+ __ubuf__ float* offsetLocalAddr;
404 400 
405 if constexpr (isSymmertrical == false) {401 if constexpr (isSymmertrical == false) {
406 MaxOutLocal = MaxOutQueue.AllocTensor<float>();402 MaxOutLocal = MaxOutQueue.AllocTensor<float>();
407 AscendC::Duplicate(MaxOutLocal, MIN_FLOAT_VALUE, 64, 1, 1, 8);403 AscendC::Duplicate(MaxOutLocal, MIN_FLOAT_VALUE, 64, 1, 1, 8);
408- MaxOutLocalAddr = (__local_mem__ float*)MaxOutLocal.GetPhyAddr();404+ MaxOutLocalAddr = (__ubuf__ float*)MaxOutLocal.GetPhyAddr();
409 405 
410 MinOutLocal = MinOutQueue.AllocTensor<float>();406 MinOutLocal = MinOutQueue.AllocTensor<float>();
411 AscendC::Duplicate(MinOutLocal, MAX_FLOAT_VALUE, 64, 1, 1, 8);407 AscendC::Duplicate(MinOutLocal, MAX_FLOAT_VALUE, 64, 1, 1, 8);
412- MinOutLocalAddr = (__local_mem__ float*)MinOutLocal.GetPhyAddr();408+ MinOutLocalAddr = (__ubuf__ float*)MinOutLocal.GetPhyAddr();
413 409 
414 scaleOutLocal = scaleOutQueue.AllocTensor<float>();410 scaleOutLocal = scaleOutQueue.AllocTensor<float>();
415 AscendC::Duplicate(scaleOutLocal, (float)0.0, 64, 1, 1, 8);411 AscendC::Duplicate(scaleOutLocal, (float)0.0, 64, 1, 1, 8);
416- scaleOutLocalAddr = (__local_mem__ float*)scaleOutLocal.GetPhyAddr();412+ scaleOutLocalAddr = (__ubuf__ float*)scaleOutLocal.GetPhyAddr();
417 413 
418 offsetLocal = offsetQueue.AllocTensor<float>();414 offsetLocal = offsetQueue.AllocTensor<float>();
419 AscendC::Duplicate(offsetLocal, (float)0.0, 64, 1, 1, 8);415 AscendC::Duplicate(offsetLocal, (float)0.0, 64, 1, 1, 8);
420- offsetLocalAddr = (__local_mem__ float*)offsetLocal.GetPhyAddr();416+ offsetLocalAddr = (__ubuf__ float*)offsetLocal.GetPhyAddr();
421 } else {417 } else {
422 scaleOutLocal = scaleOutQueue.AllocTensor<float>();418 scaleOutLocal = scaleOutQueue.AllocTensor<float>();
423 AscendC::Duplicate(scaleOutLocal, (float)0.0, 64, 1, 1, 8);419 AscendC::Duplicate(scaleOutLocal, (float)0.0, 64, 1, 1, 8);
424- scaleOutLocalAddr = (__local_mem__ float*)scaleOutLocal.GetPhyAddr();420+ scaleOutLocalAddr = (__ubuf__ float*)scaleOutLocal.GetPhyAddr();
425 }421 }
426 scaleOffset = 0;422 scaleOffset = 0;
427 CopyInScaleByEle(scaleOffset, tilingData_.coreNum);423 CopyInScaleByEle(scaleOffset, tilingData_.coreNum);
@@ -438,7 +434,7 @@ __aicore__ inline void DynamicQuantLargeShapeDbPertensor<T, yDtype, hasSmooth, i
438 434 
439template <typename T, typename yDtype, int64_t hasSmooth, bool isSymmertrical>435template <typename T, typename yDtype, int64_t hasSmooth, bool isSymmertrical>
440__aicore__ inline void DynamicQuantLargeShapeDbPertensor<T, yDtype, hasSmooth, isSymmertrical>::ProcessYRow(436__aicore__ inline void DynamicQuantLargeShapeDbPertensor<T, yDtype, hasSmooth, isSymmertrical>::ProcessYRow(
441- uint32_t i, uint32_t j, __local_mem__ float* scaleAddr, __local_mem__ float* offsetAddr)437+ uint32_t i, uint32_t j, __ubuf__ float* scaleAddr, __ubuf__ float* offsetAddr)
442{438{
443 offsetBase = i * THIRTY_TWO + j;439 offsetBase = i * THIRTY_TWO + j;
444 srcOffset = offsetBase * tilingData_.rowLen;440 srcOffset = offsetBase * tilingData_.rowLen;
@@ -503,29 +499,29 @@ __aicore__ inline void DynamicQuantLargeShapeDbPertensor<T, yDtype, hasSmooth, i
503 uint32_t elementNum)499 uint32_t elementNum)
504{500{
505 LocalTensor<T> inLocal = inQueue.DeQue<T>();501 LocalTensor<T> inLocal = inQueue.DeQue<T>();
506- __local_mem__ T* inLocalAddr = (__local_mem__ T*)inLocal.GetPhyAddr();502+ __ubuf__ T* inLocalAddr = (__ubuf__ T*)inLocal.GetPhyAddr();
507 LocalTensor<T> smoothLocal;503 LocalTensor<T> smoothLocal;
508- __local_mem__ T* smoothLocalAddr;504+ __ubuf__ T* smoothLocalAddr;
509 LocalTensor<float> maxToWorkSpaceLocal;505 LocalTensor<float> maxToWorkSpaceLocal;
510- __local_mem__ float* maxToWorkSpaceLocalAddr;506+ __ubuf__ float* maxToWorkSpaceLocalAddr;
511 LocalTensor<float> minToWorkSpaceLocal;507 LocalTensor<float> minToWorkSpaceLocal;
512- __local_mem__ float* minToWorkSpaceLocalAddr;508+ __ubuf__ float* minToWorkSpaceLocalAddr;
513 LocalTensor<float> scaleToWorkSpaceLocal;509 LocalTensor<float> scaleToWorkSpaceLocal;
514- __local_mem__ float* scaleToWorkSpaceLocalAddr;510+ __ubuf__ float* scaleToWorkSpaceLocalAddr;
515 511 
516 if constexpr (hasSmooth == 1) {512 if constexpr (hasSmooth == 1) {
517 smoothLocal = smoothQueue.DeQue<T>();513 smoothLocal = smoothQueue.DeQue<T>();
518- smoothLocalAddr = (__local_mem__ T*)smoothLocal.GetPhyAddr();514+ smoothLocalAddr = (__ubuf__ T*)smoothLocal.GetPhyAddr();
519 }515 }
520 if constexpr (isSymmertrical == false) {516 if constexpr (isSymmertrical == false) {
521 maxToWorkSpaceLocal = MaxToWorkSpaceQueue.DeQue<float>();517 maxToWorkSpaceLocal = MaxToWorkSpaceQueue.DeQue<float>();
522- maxToWorkSpaceLocalAddr = (__local_mem__ float*)maxToWorkSpaceLocal.GetPhyAddr();518+ maxToWorkSpaceLocalAddr = (__ubuf__ float*)maxToWorkSpaceLocal.GetPhyAddr();
523 519 
524 minToWorkSpaceLocal = MinToWorkSpaceQueue.DeQue<float>();520 minToWorkSpaceLocal = MinToWorkSpaceQueue.DeQue<float>();
525- minToWorkSpaceLocalAddr = (__local_mem__ float*)minToWorkSpaceLocal.GetPhyAddr();521+ minToWorkSpaceLocalAddr = (__ubuf__ float*)minToWorkSpaceLocal.GetPhyAddr();
526 } else {522 } else {
527 scaleToWorkSpaceLocal = scaleToWorkSpaceQueue.DeQue<float>();523 scaleToWorkSpaceLocal = scaleToWorkSpaceQueue.DeQue<float>();
528- scaleToWorkSpaceLocalAddr = (__local_mem__ float*)scaleToWorkSpaceLocal.GetPhyAddr();524+ scaleToWorkSpaceLocalAddr = (__ubuf__ float*)scaleToWorkSpaceLocal.GetPhyAddr();
529 }525 }
530 ComputeMaxRowScaleVF(inLocalAddr, smoothLocalAddr, scaleToWorkSpaceLocalAddr, maxToWorkSpaceLocalAddr,526 ComputeMaxRowScaleVF(inLocalAddr, smoothLocalAddr, scaleToWorkSpaceLocalAddr, maxToWorkSpaceLocalAddr,
531 minToWorkSpaceLocalAddr, elementNum);527 minToWorkSpaceLocalAddr, elementNum);
@@ -543,22 +539,22 @@ __aicore__ inline void DynamicQuantLargeShapeDbPertensor<T, yDtype, hasSmooth, i
543 539 
544template <typename T, typename yDtype, int64_t hasSmooth, bool isSymmertrical>540template <typename T, typename yDtype, int64_t hasSmooth, bool isSymmertrical>
545__aicore__ inline void DynamicQuantLargeShapeDbPertensor<T, yDtype, hasSmooth, isSymmertrical>::ComputeMaxColScale(541__aicore__ inline void DynamicQuantLargeShapeDbPertensor<T, yDtype, hasSmooth, isSymmertrical>::ComputeMaxColScale(
546- uint32_t elementNum, __local_mem__ float* maxAddr, __local_mem__ float* minAddr, __local_mem__ float* scaleAddr,542+ uint32_t elementNum, __ubuf__ float* maxAddr, __ubuf__ float* minAddr, __ubuf__ float* scaleAddr,
547- __local_mem__ float* offsetAddr)543+ __ubuf__ float* offsetAddr)
548{544{
549 LocalTensor<float> scaleFromWorkSpaceLocal;545 LocalTensor<float> scaleFromWorkSpaceLocal;
550- __local_mem__ float* scaleFromWorkSpaceLocalAddr;546+ __ubuf__ float* scaleFromWorkSpaceLocalAddr;
551 547 
552 LocalTensor<float> maxFromWorkSpaceLocal;548 LocalTensor<float> maxFromWorkSpaceLocal;
553- __local_mem__ float* maxFromWorkSpaceLocalAddr;549+ __ubuf__ float* maxFromWorkSpaceLocalAddr;
554 LocalTensor<float> minFromWorkSpaceLocal;550 LocalTensor<float> minFromWorkSpaceLocal;
555- __local_mem__ float* minFromWorkSpaceLocalAddr;551+ __ubuf__ float* minFromWorkSpaceLocalAddr;
556 552 
557 if constexpr (isSymmertrical == false) {553 if constexpr (isSymmertrical == false) {
558 maxFromWorkSpaceLocal = MaxFromWorkSpaceQueue.DeQue<float>();554 maxFromWorkSpaceLocal = MaxFromWorkSpaceQueue.DeQue<float>();
559- maxFromWorkSpaceLocalAddr = (__local_mem__ float*)maxFromWorkSpaceLocal.GetPhyAddr();555+ maxFromWorkSpaceLocalAddr = (__ubuf__ float*)maxFromWorkSpaceLocal.GetPhyAddr();
560 minFromWorkSpaceLocal = MinFromWorkSpaceQueue.DeQue<float>();556 minFromWorkSpaceLocal = MinFromWorkSpaceQueue.DeQue<float>();
561- minFromWorkSpaceLocalAddr = (__local_mem__ float*)minFromWorkSpaceLocal.GetPhyAddr();557+ minFromWorkSpaceLocalAddr = (__ubuf__ float*)minFromWorkSpaceLocal.GetPhyAddr();
562 ComputeMaxColScaleVF(scaleFromWorkSpaceLocalAddr, scaleAddr, maxFromWorkSpaceLocalAddr, maxAddr,558 ComputeMaxColScaleVF(scaleFromWorkSpaceLocalAddr, scaleAddr, maxFromWorkSpaceLocalAddr, maxAddr,
563 minFromWorkSpaceLocalAddr, minAddr, elementNum);559 minFromWorkSpaceLocalAddr, minAddr, elementNum);
564 ComputeScaleSymVF(maxAddr, minAddr, scaleAddr, 1);560 ComputeScaleSymVF(maxAddr, minAddr, scaleAddr, 1);
@@ -568,7 +564,7 @@ __aicore__ inline void DynamicQuantLargeShapeDbPertensor<T, yDtype, hasSmooth, i
568 MinFromWorkSpaceQueue.FreeTensor(minFromWorkSpaceLocal);564 MinFromWorkSpaceQueue.FreeTensor(minFromWorkSpaceLocal);
569 } else {565 } else {
570 scaleFromWorkSpaceLocal = scaleFromWorkSpaceQueue.DeQue<float>();566 scaleFromWorkSpaceLocal = scaleFromWorkSpaceQueue.DeQue<float>();
571- scaleFromWorkSpaceLocalAddr = (__local_mem__ float*)scaleFromWorkSpaceLocal.GetPhyAddr();567+ scaleFromWorkSpaceLocalAddr = (__ubuf__ float*)scaleFromWorkSpaceLocal.GetPhyAddr();
572 ComputeMaxColScaleVF(scaleFromWorkSpaceLocalAddr, scaleAddr, maxFromWorkSpaceLocalAddr, maxAddr,568 ComputeMaxColScaleVF(scaleFromWorkSpaceLocalAddr, scaleAddr, maxFromWorkSpaceLocalAddr, maxAddr,
573 minFromWorkSpaceLocalAddr, minAddr, elementNum);569 minFromWorkSpaceLocalAddr, minAddr, elementNum);
574 scaleFromWorkSpaceQueue.FreeTensor(scaleFromWorkSpaceLocal);570 scaleFromWorkSpaceQueue.FreeTensor(scaleFromWorkSpaceLocal);
@@ -577,8 +573,8 @@ __aicore__ inline void DynamicQuantLargeShapeDbPertensor<T, yDtype, hasSmooth, i
577 573 
578template <typename T, typename yDtype, int64_t hasSmooth, bool isSymmertrical>574template <typename T, typename yDtype, int64_t hasSmooth, bool isSymmertrical>
579__aicore__ inline void DynamicQuantLargeShapeDbPertensor<T, yDtype, hasSmooth, isSymmertrical>::ComputeMaxRowScaleVF(575__aicore__ inline void DynamicQuantLargeShapeDbPertensor<T, yDtype, hasSmooth, isSymmertrical>::ComputeMaxRowScaleVF(
580- __local_mem__ T* inLocalAddr, __local_mem__ T* smoothLocalAddr, __local_mem__ float* scaleLocalAddr,576+ __ubuf__ T* inLocalAddr, __ubuf__ T* smoothLocalAddr, __ubuf__ float* scaleLocalAddr, __ubuf__ float* maxLocalAddr,
581- __local_mem__ float* maxLocalAddr, __local_mem__ float* minLocalAddr, uint32_t elementNum)577+ __ubuf__ float* minLocalAddr, uint32_t elementNum)
582{578{
583 uint32_t dtypeSize = sizeof(float);579 uint32_t dtypeSize = sizeof(float);
584 uint32_t VL = AscendC::VECTOR_REG_WIDTH / dtypeSize;580 uint32_t VL = AscendC::VECTOR_REG_WIDTH / dtypeSize;
@@ -600,21 +596,21 @@ __aicore__ inline void DynamicQuantLargeShapeDbPertensor<T, yDtype, hasSmooth, i
600 AscendC::MicroAPI::RegTensor<float, MicroAPI::RegTraitNumOne> vreg8_2;596 AscendC::MicroAPI::RegTensor<float, MicroAPI::RegTraitNumOne> vreg8_2;
601 597 
602 if constexpr (isSymmertrical == false) {598 if constexpr (isSymmertrical == false) {
603- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_BRC_B32>(vreg6_1, maxLocalAddr);599+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_BRC_B32>(vreg6_1, maxLocalAddr);
604- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_BRC_B32>(vreg6_2, minLocalAddr);600+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_BRC_B32>(vreg6_2, minLocalAddr);
605 } else {601 } else {
606- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_BRC_B32>(vreg6_1, scaleLocalAddr);602+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_BRC_B32>(vreg6_1, scaleLocalAddr);
607 }603 }
608 AscendC::MicroAPI::MaskReg mask;604 AscendC::MicroAPI::MaskReg mask;
609 605 
610 for (uint16_t i = 0; i < static_cast<uint16_t>(vfLoopNum - 1); i++) {606 for (uint16_t i = 0; i < static_cast<uint16_t>(vfLoopNum - 1); i++) {
611 maskNum = elementNum - i * VL;607 maskNum = elementNum - i * VL;
612 mask = AscendC::MicroAPI::UpdateMask<float>(maskNum);608 mask = AscendC::MicroAPI::UpdateMask<float>(maskNum);
613- AscendC::MicroAPI::DataCopy<T, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(vreg0, inLocalAddr + i * VL);609+ AscendC::MicroAPI::LoadAlign<T, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(vreg0, inLocalAddr + i * VL);
614 AscendC::MicroAPI::Cast<float, T, castTrait0>(vreg1, vreg0, mask);610 AscendC::MicroAPI::Cast<float, T, castTrait0>(vreg1, vreg0, mask);
615 if constexpr (hasSmooth == 1) {611 if constexpr (hasSmooth == 1) {
616- AscendC::MicroAPI::DataCopy<T, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(vreg2,612+ AscendC::MicroAPI::LoadAlign<T, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(vreg2,
617- smoothLocalAddr + i * VL);613+ smoothLocalAddr + i * VL);
618 AscendC::MicroAPI::Cast<float, T, castTrait0>(vreg3, vreg2, mask);614 AscendC::MicroAPI::Cast<float, T, castTrait0>(vreg3, vreg2, mask);
619 AscendC::MicroAPI::Mul(vreg1, vreg1, vreg3, mask);615 AscendC::MicroAPI::Mul(vreg1, vreg1, vreg3, mask);
620 }616 }
@@ -629,19 +625,19 @@ __aicore__ inline void DynamicQuantLargeShapeDbPertensor<T, yDtype, hasSmooth, i
629 }625 }
630 }626 }
631 627 
632- AscendC::MicroAPI::ReduceMax<float>(vreg7_1, vreg6_1, mask);628+ AscendC::MicroAPI::Reduce<MicroAPI::ReduceType::MAX, float>(vreg7_1, vreg6_1, mask);
633 if constexpr (isSymmertrical == false) {629 if constexpr (isSymmertrical == false) {
634- AscendC::MicroAPI::ReduceMin<float>(vreg7_2, vreg6_2, mask);630+ AscendC::MicroAPI::Reduce<MicroAPI::ReduceType::MIN, float>(vreg7_2, vreg6_2, mask);
635 }631 }
636 632 
637 for (uint16_t i = vfLoopNum - 1; i < vfLoopNum; i++) {633 for (uint16_t i = vfLoopNum - 1; i < vfLoopNum; i++) {
638 maskNum = elementNum - i * VL;634 maskNum = elementNum - i * VL;
639 mask = AscendC::MicroAPI::UpdateMask<float>(maskNum);635 mask = AscendC::MicroAPI::UpdateMask<float>(maskNum);
640- AscendC::MicroAPI::DataCopy<T, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(vreg0, inLocalAddr + i * VL);636+ AscendC::MicroAPI::LoadAlign<T, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(vreg0, inLocalAddr + i * VL);
641 AscendC::MicroAPI::Cast<float, T, castTrait0>(vreg1, vreg0, mask);637 AscendC::MicroAPI::Cast<float, T, castTrait0>(vreg1, vreg0, mask);
642 if constexpr (hasSmooth == 1) {638 if constexpr (hasSmooth == 1) {
643- AscendC::MicroAPI::DataCopy<T, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(vreg2,639+ AscendC::MicroAPI::LoadAlign<T, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(vreg2,
644- smoothLocalAddr + i * VL);640+ smoothLocalAddr + i * VL);
645 AscendC::MicroAPI::Cast<float, T, castTrait0>(vreg3, vreg2, mask);641 AscendC::MicroAPI::Cast<float, T, castTrait0>(vreg3, vreg2, mask);
646 AscendC::MicroAPI::Mul(vreg1, vreg1, vreg3, mask);642 AscendC::MicroAPI::Mul(vreg1, vreg1, vreg3, mask);
647 }643 }
@@ -653,9 +649,9 @@ __aicore__ inline void DynamicQuantLargeShapeDbPertensor<T, yDtype, hasSmooth, i
653 AscendC::MicroAPI::Muls(vreg5, vreg5, float(1.0) / maxValue, mask);649 AscendC::MicroAPI::Muls(vreg5, vreg5, float(1.0) / maxValue, mask);
654 AscendC::MicroAPI::Max(vreg6_1, vreg5, vreg6_1, mask);650 AscendC::MicroAPI::Max(vreg6_1, vreg5, vreg6_1, mask);
655 }651 }
656- AscendC::MicroAPI::ReduceMax<float>(vreg8_1, vreg6_1, mask);652+ AscendC::MicroAPI::Reduce<MicroAPI::ReduceType::MAX, float>(vreg8_1, vreg6_1, mask);
657 if constexpr (isSymmertrical == false) {653 if constexpr (isSymmertrical == false) {
658- AscendC::MicroAPI::ReduceMin<float>(vreg8_2, vreg6_2, mask);654+ AscendC::MicroAPI::Reduce<MicroAPI::ReduceType::MIN, float>(vreg8_2, vreg6_2, mask);
659 }655 }
660 656 
661 AscendC::MicroAPI::Max(vreg8_1, vreg7_1, vreg8_1, mask);657 AscendC::MicroAPI::Max(vreg8_1, vreg7_1, vreg8_1, mask);
@@ -664,22 +660,21 @@ __aicore__ inline void DynamicQuantLargeShapeDbPertensor<T, yDtype, hasSmooth, i
664 }660 }
665 }661 }
666 if constexpr (isSymmertrical == false) {662 if constexpr (isSymmertrical == false) {
667- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::StoreDist::DIST_FIRST_ELEMENT_B32>(maxLocalAddr,663+ AscendC::MicroAPI::StoreAlign<float, AscendC::MicroAPI::StoreDist::DIST_FIRST_ELEMENT_B32>(maxLocalAddr,
668- vreg8_1, mask);664+ vreg8_1, mask);
669- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::StoreDist::DIST_FIRST_ELEMENT_B32>(minLocalAddr,665+ AscendC::MicroAPI::StoreAlign<float, AscendC::MicroAPI::StoreDist::DIST_FIRST_ELEMENT_B32>(minLocalAddr,
670- vreg8_2, mask);666+ vreg8_2, mask);
671 } else {667 } else {
672- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::StoreDist::DIST_FIRST_ELEMENT_B32>(scaleLocalAddr,668+ AscendC::MicroAPI::StoreAlign<float, AscendC::MicroAPI::StoreDist::DIST_FIRST_ELEMENT_B32>(scaleLocalAddr,
673- vreg8_1, mask);669+ vreg8_1, mask);
674 }670 }
675 }671 }
676}672}
677 673 
678template <typename T, typename yDtype, int64_t hasSmooth, bool isSymmertrical>674template <typename T, typename yDtype, int64_t hasSmooth, bool isSymmertrical>
679__aicore__ inline void DynamicQuantLargeShapeDbPertensor<T, yDtype, hasSmooth, isSymmertrical>::ComputeMaxColScaleVF(675__aicore__ inline void DynamicQuantLargeShapeDbPertensor<T, yDtype, hasSmooth, isSymmertrical>::ComputeMaxColScaleVF(
680- __local_mem__ float* scaleLocalAddr, __local_mem__ float* scaleOutLocalAddr, __local_mem__ float* maxLocalAddr,676+ __ubuf__ float* scaleLocalAddr, __ubuf__ float* scaleOutLocalAddr, __ubuf__ float* maxLocalAddr,
681- __local_mem__ float* maxOutLocalAddr, __local_mem__ float* minLocalAddr, __local_mem__ float* minOutLocalAddr,677+ __ubuf__ float* maxOutLocalAddr, __ubuf__ float* minLocalAddr, __ubuf__ float* minOutLocalAddr, uint32_t elementNum)
682- uint32_t elementNum)
683{678{
684 uint32_t dtypeSize = sizeof(float);679 uint32_t dtypeSize = sizeof(float);
685 uint32_t VL = AscendC::VECTOR_REG_WIDTH / dtypeSize;680 uint32_t VL = AscendC::VECTOR_REG_WIDTH / dtypeSize;
@@ -698,10 +693,10 @@ __aicore__ inline void DynamicQuantLargeShapeDbPertensor<T, yDtype, hasSmooth, i
698 AscendC::MicroAPI::RegTensor<float, MicroAPI::RegTraitNumOne> vreg3_2;693 AscendC::MicroAPI::RegTensor<float, MicroAPI::RegTraitNumOne> vreg3_2;
699 694 
700 if constexpr (isSymmertrical == false) {695 if constexpr (isSymmertrical == false) {
701- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_BRC_B32>(vreg1_1, maxOutLocalAddr);696+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_BRC_B32>(vreg1_1, maxOutLocalAddr);
702- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_BRC_B32>(vreg1_2, minOutLocalAddr);697+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_BRC_B32>(vreg1_2, minOutLocalAddr);
703 } else {698 } else {
704- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_BRC_B32>(vreg1_1, scaleOutLocalAddr);699+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_BRC_B32>(vreg1_1, scaleOutLocalAddr);
705 }700 }
706 701 
707 AscendC::MicroAPI::MaskReg mask = AscendC::MicroAPI::CreateMask<float, AscendC::MicroAPI::MaskPattern::ALL>();702 AscendC::MicroAPI::MaskReg mask = AscendC::MicroAPI::CreateMask<float, AscendC::MicroAPI::MaskPattern::ALL>();
@@ -711,74 +706,74 @@ __aicore__ inline void DynamicQuantLargeShapeDbPertensor<T, yDtype, hasSmooth, i
711 mask = AscendC::MicroAPI::UpdateMask<float>(maskNum);706 mask = AscendC::MicroAPI::UpdateMask<float>(maskNum);
712 707 
713 if constexpr (isSymmertrical == false) {708 if constexpr (isSymmertrical == false) {
714- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(vreg0_1,709+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(vreg0_1,
715- maxLocalAddr + i * VL);710+ maxLocalAddr + i * VL);
716 AscendC::MicroAPI::Max(vreg1_1, vreg0_1, vreg1_1, mask);711 AscendC::MicroAPI::Max(vreg1_1, vreg0_1, vreg1_1, mask);
717- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(vreg0_2,712+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(vreg0_2,
718- minLocalAddr + i * VL);713+ minLocalAddr + i * VL);
719 AscendC::MicroAPI::Min(vreg1_2, vreg0_2, vreg1_2, mask);714 AscendC::MicroAPI::Min(vreg1_2, vreg0_2, vreg1_2, mask);
720 } else {715 } else {
721- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(vreg0_1,716+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(vreg0_1,
722- scaleLocalAddr + i * VL);717+ scaleLocalAddr + i * VL);
723 AscendC::MicroAPI::Max(vreg1_1, vreg0_1, vreg1_1, mask);718 AscendC::MicroAPI::Max(vreg1_1, vreg0_1, vreg1_1, mask);
724 }719 }
725 }720 }
726 if constexpr (isSymmertrical == false) {721 if constexpr (isSymmertrical == false) {
727- AscendC::MicroAPI::ReduceMax<float>(vreg2_1, vreg1_1, mask);722+ AscendC::MicroAPI::Reduce<MicroAPI::ReduceType::MAX, float>(vreg2_1, vreg1_1, mask);
728- AscendC::MicroAPI::ReduceMin<float>(vreg2_2, vreg1_2, mask);723+ AscendC::MicroAPI::Reduce<MicroAPI::ReduceType::MIN, float>(vreg2_2, vreg1_2, mask);
729 } else {724 } else {
730- AscendC::MicroAPI::ReduceMax<float>(vreg2_1, vreg1_1, mask);725+ AscendC::MicroAPI::Reduce<MicroAPI::ReduceType::MAX, float>(vreg2_1, vreg1_1, mask);
731 }726 }
732 727 
733 for (uint16_t i = vfLoopNum - 1; i < vfLoopNum; i++) {728 for (uint16_t i = vfLoopNum - 1; i < vfLoopNum; i++) {
734 maskNum = elementNum - i * VL;729 maskNum = elementNum - i * VL;
735 mask = AscendC::MicroAPI::UpdateMask<float>(maskNum);730 mask = AscendC::MicroAPI::UpdateMask<float>(maskNum);
736 if constexpr (isSymmertrical == false) {731 if constexpr (isSymmertrical == false) {
737- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(vreg0_1,732+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(vreg0_1,
738- maxLocalAddr + i * VL);733+ maxLocalAddr + i * VL);
739 AscendC::MicroAPI::Max(vreg1_1, vreg0_1, vreg1_1, mask);734 AscendC::MicroAPI::Max(vreg1_1, vreg0_1, vreg1_1, mask);
740- AscendC::MicroAPI::ReduceMax<float>(vreg3_1, vreg1_1, mask);735+ AscendC::MicroAPI::Reduce<MicroAPI::ReduceType::MAX, float>(vreg3_1, vreg1_1, mask);
741 AscendC::MicroAPI::Max(vreg3_1, vreg2_1, vreg3_1, mask);736 AscendC::MicroAPI::Max(vreg3_1, vreg2_1, vreg3_1, mask);
742 737 
743- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(vreg0_2,738+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(vreg0_2,
744- minLocalAddr + i * VL);739+ minLocalAddr + i * VL);
745 AscendC::MicroAPI::Min(vreg1_2, vreg0_2, vreg1_2, mask);740 AscendC::MicroAPI::Min(vreg1_2, vreg0_2, vreg1_2, mask);
746- AscendC::MicroAPI::ReduceMin<float>(vreg3_2, vreg1_2, mask);741+ AscendC::MicroAPI::Reduce<MicroAPI::ReduceType::MIN, float>(vreg3_2, vreg1_2, mask);
747 AscendC::MicroAPI::Min(vreg3_2, vreg2_2, vreg3_2, mask);742 AscendC::MicroAPI::Min(vreg3_2, vreg2_2, vreg3_2, mask);
748 } else {743 } else {
749- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(vreg0_1,744+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(vreg0_1,
750- scaleLocalAddr + i * VL);745+ scaleLocalAddr + i * VL);
751 AscendC::MicroAPI::Max(vreg1_1, vreg0_1, vreg1_1, mask);746 AscendC::MicroAPI::Max(vreg1_1, vreg0_1, vreg1_1, mask);
752- AscendC::MicroAPI::ReduceMax<float>(vreg3_1, vreg1_1, mask);747+ AscendC::MicroAPI::Reduce<MicroAPI::ReduceType::MAX, float>(vreg3_1, vreg1_1, mask);
753 AscendC::MicroAPI::Max(vreg3_1, vreg2_1, vreg3_1, mask);748 AscendC::MicroAPI::Max(vreg3_1, vreg2_1, vreg3_1, mask);
754 }749 }
755 }750 }
756 if constexpr (isSymmertrical == false) {751 if constexpr (isSymmertrical == false) {
757- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::StoreDist::DIST_FIRST_ELEMENT_B32>(maxOutLocalAddr,752+ AscendC::MicroAPI::StoreAlign<float, AscendC::MicroAPI::StoreDist::DIST_FIRST_ELEMENT_B32>(maxOutLocalAddr,
758- vreg3_1, mask);753+ vreg3_1, mask);
759- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::StoreDist::DIST_FIRST_ELEMENT_B32>(minOutLocalAddr,754+ AscendC::MicroAPI::StoreAlign<float, AscendC::MicroAPI::StoreDist::DIST_FIRST_ELEMENT_B32>(minOutLocalAddr,
760- vreg3_2, mask);755+ vreg3_2, mask);
761 } else {756 } else {
762- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::StoreDist::DIST_FIRST_ELEMENT_B32>(scaleOutLocalAddr,757+ AscendC::MicroAPI::StoreAlign<float, AscendC::MicroAPI::StoreDist::DIST_FIRST_ELEMENT_B32>(
763- vreg3_1, mask);758+ scaleOutLocalAddr, vreg3_1, mask);
764 }759 }
765 }760 }
766}761}
767 762 
768template <typename T, typename yDtype, int64_t hasSmooth, bool isSymmertrical>763template <typename T, typename yDtype, int64_t hasSmooth, bool isSymmertrical>
769__aicore__ inline void DynamicQuantLargeShapeDbPertensor<T, yDtype, hasSmooth, isSymmertrical>::ComputeY(764__aicore__ inline void DynamicQuantLargeShapeDbPertensor<T, yDtype, hasSmooth, isSymmertrical>::ComputeY(
770- uint32_t elementNum, __local_mem__ float* scaleAddr, __local_mem__ float* offsetAddr)765+ uint32_t elementNum, __ubuf__ float* scaleAddr, __ubuf__ float* offsetAddr)
771{766{
772 LocalTensor<T> inLocal = inQueue.DeQue<T>();767 LocalTensor<T> inLocal = inQueue.DeQue<T>();
773- __local_mem__ T* inLocalAddr = (__local_mem__ T*)inLocal.GetPhyAddr();768+ __ubuf__ T* inLocalAddr = (__ubuf__ T*)inLocal.GetPhyAddr();
774 LocalTensor<yCopyDtype> outLocal = outQueue.AllocTensor<yCopyDtype>();769 LocalTensor<yCopyDtype> outLocal = outQueue.AllocTensor<yCopyDtype>();
775- __local_mem__ yCopyDtype* outAddr = (__local_mem__ yCopyDtype*)outLocal.GetPhyAddr();770+ __ubuf__ yCopyDtype* outAddr = (__ubuf__ yCopyDtype*)outLocal.GetPhyAddr();
776 LocalTensor<T> smoothLocal;771 LocalTensor<T> smoothLocal;
777- __local_mem__ T* smoothLocalAddr;772+ __ubuf__ T* smoothLocalAddr;
778 773 
779 if constexpr (hasSmooth == 1) {774 if constexpr (hasSmooth == 1) {
780 smoothLocal = smoothQueue.DeQue<T>();775 smoothLocal = smoothQueue.DeQue<T>();
781- smoothLocalAddr = (__local_mem__ T*)smoothLocal.GetPhyAddr();776+ smoothLocalAddr = (__ubuf__ T*)smoothLocal.GetPhyAddr();
782 }777 }
783 778 
784 ComputeYVF(inLocalAddr, smoothLocalAddr, outAddr, scaleAddr, offsetAddr, elementNum);779 ComputeYVF(inLocalAddr, smoothLocalAddr, outAddr, scaleAddr, offsetAddr, elementNum);
@@ -792,8 +787,7 @@ __aicore__ inline void DynamicQuantLargeShapeDbPertensor<T, yDtype, hasSmooth, i
792 787 
793template <typename T, typename yDtype, int64_t hasSmooth, bool isSymmertrical>788template <typename T, typename yDtype, int64_t hasSmooth, bool isSymmertrical>
794__aicore__ inline void DynamicQuantLargeShapeDbPertensor<T, yDtype, hasSmooth, isSymmertrical>::ComputeScaleSymVF(789__aicore__ inline void DynamicQuantLargeShapeDbPertensor<T, yDtype, hasSmooth, isSymmertrical>::ComputeScaleSymVF(
795- __local_mem__ float* maxLocalAddr, __local_mem__ float* minLocalAddr, __local_mem__ float* scaleLocalAddr,790+ __ubuf__ float* maxLocalAddr, __ubuf__ float* minLocalAddr, __ubuf__ float* scaleLocalAddr, uint32_t elementNum)
796- uint32_t elementNum)
797{791{
798 uint32_t dtypeSize = sizeof(float);792 uint32_t dtypeSize = sizeof(float);
799 uint32_t VL = AscendC::VECTOR_REG_WIDTH / dtypeSize;793 uint32_t VL = AscendC::VECTOR_REG_WIDTH / dtypeSize;
@@ -811,20 +805,19 @@ __aicore__ inline void DynamicQuantLargeShapeDbPertensor<T, yDtype, hasSmooth, i
811 maskNum = elementNum - i * VL;805 maskNum = elementNum - i * VL;
812 mask = AscendC::MicroAPI::UpdateMask<float>(maskNum);806 mask = AscendC::MicroAPI::UpdateMask<float>(maskNum);
813 807 
814- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(vreg0, maxLocalAddr + i * VL);808+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(vreg0, maxLocalAddr + i * VL);
815- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(vreg1, minLocalAddr + i * VL);809+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(vreg1, minLocalAddr + i * VL);
816 AscendC::MicroAPI::Sub(vreg1, vreg0, vreg1, mask);810 AscendC::MicroAPI::Sub(vreg1, vreg0, vreg1, mask);
817 AscendC::MicroAPI::Muls(vreg1, vreg1, float(1.0) / maxValueNoSym, mask);811 AscendC::MicroAPI::Muls(vreg1, vreg1, float(1.0) / maxValueNoSym, mask);
818- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::StoreDist::DIST_FIRST_ELEMENT_B32>(scaleLocalAddr,812+ AscendC::MicroAPI::StoreAlign<float, AscendC::MicroAPI::StoreDist::DIST_FIRST_ELEMENT_B32>(scaleLocalAddr,
819- vreg1, mask);813+ vreg1, mask);
820 }814 }
821 }815 }
822}816}
823 817 
824template <typename T, typename yDtype, int64_t hasSmooth, bool isSymmertrical>818template <typename T, typename yDtype, int64_t hasSmooth, bool isSymmertrical>
825__aicore__ inline void DynamicQuantLargeShapeDbPertensor<T, yDtype, hasSmooth, isSymmertrical>::ComputeOffsetSymVF(819__aicore__ inline void DynamicQuantLargeShapeDbPertensor<T, yDtype, hasSmooth, isSymmertrical>::ComputeOffsetSymVF(
826- __local_mem__ float* maxLocalAddr, __local_mem__ float* scaleLocalAddr, __local_mem__ float* offsetLocalAddr,820+ __ubuf__ float* maxLocalAddr, __ubuf__ float* scaleLocalAddr, __ubuf__ float* offsetLocalAddr, uint32_t elementNum)
827- uint32_t elementNum)
828{821{
829 uint32_t dtypeSize = sizeof(float);822 uint32_t dtypeSize = sizeof(float);
830 uint32_t VL = AscendC::VECTOR_REG_WIDTH / dtypeSize;823 uint32_t VL = AscendC::VECTOR_REG_WIDTH / dtypeSize;
@@ -843,22 +836,22 @@ __aicore__ inline void DynamicQuantLargeShapeDbPertensor<T, yDtype, hasSmooth, i
843 maskNum = elementNum - i * VL;836 maskNum = elementNum - i * VL;
844 mask = AscendC::MicroAPI::UpdateMask<float>(maskNum);837 mask = AscendC::MicroAPI::UpdateMask<float>(maskNum);
845 838 
846- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(vreg0, maxLocalAddr + i * VL);839+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(vreg0, maxLocalAddr + i * VL);
847- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(vreg1, scaleLocalAddr + i * VL);840+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(vreg1, scaleLocalAddr + i * VL);
848 AscendC::MicroAPI::Div<float, &mode>(vreg1, vreg0, vreg1, mask);841 AscendC::MicroAPI::Div<float, &mode>(vreg1, vreg0, vreg1, mask);
849 AscendC::MicroAPI::Muls(vreg1, vreg1, float(-1.0), mask);842 AscendC::MicroAPI::Muls(vreg1, vreg1, float(-1.0), mask);
850 AscendC::MicroAPI::Adds(vreg1, vreg1, maxValue, mask);843 AscendC::MicroAPI::Adds(vreg1, vreg1, maxValue, mask);
851 844 
852- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::StoreDist::DIST_FIRST_ELEMENT_B32>(offsetLocalAddr,845+ AscendC::MicroAPI::StoreAlign<float, AscendC::MicroAPI::StoreDist::DIST_FIRST_ELEMENT_B32>(offsetLocalAddr,
853- vreg1, mask);846+ vreg1, mask);
854 }847 }
855 }848 }
856}849}
857 850 
858template <typename T, typename yDtype, int64_t hasSmooth, bool isSymmertrical>851template <typename T, typename yDtype, int64_t hasSmooth, bool isSymmertrical>
859__aicore__ inline void DynamicQuantLargeShapeDbPertensor<T, yDtype, hasSmooth, isSymmertrical>::ComputeYVF(852__aicore__ inline void DynamicQuantLargeShapeDbPertensor<T, yDtype, hasSmooth, isSymmertrical>::ComputeYVF(
860- __local_mem__ T* inLocalAddr, __local_mem__ T* smoothLocalAddr, __local_mem__ yCopyDtype* outAddr,853+ __ubuf__ T* inLocalAddr, __ubuf__ T* smoothLocalAddr, __ubuf__ yCopyDtype* outAddr, __ubuf__ float* scaleLocalAddr,
861- __local_mem__ float* scaleLocalAddr, __local_mem__ float* offsetLocalAddr, uint32_t elementNum)854+ __ubuf__ float* offsetLocalAddr, uint32_t elementNum)
862{855{
863 uint32_t dtypeSize = sizeof(float);856 uint32_t dtypeSize = sizeof(float);
864 uint32_t VL = AscendC::VECTOR_REG_WIDTH / dtypeSize;857 uint32_t VL = AscendC::VECTOR_REG_WIDTH / dtypeSize;
@@ -878,20 +871,21 @@ __aicore__ inline void DynamicQuantLargeShapeDbPertensor<T, yDtype, hasSmooth, i
878 AscendC::MicroAPI::RegTensor<float, MicroAPI::RegTraitNumOne> vreg9;871 AscendC::MicroAPI::RegTensor<float, MicroAPI::RegTraitNumOne> vreg9;
879 AscendC::MicroAPI::RegTensor<float, MicroAPI::RegTraitNumOne> vreg_offset;872 AscendC::MicroAPI::RegTensor<float, MicroAPI::RegTraitNumOne> vreg_offset;
880 873 
881- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_BRC_B32>(vreg9, scaleLocalAddr);874+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_BRC_B32>(vreg9, scaleLocalAddr);
882 if constexpr (isSymmertrical == false) {875 if constexpr (isSymmertrical == false) {
883- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_BRC_B32>(vreg_offset, offsetLocalAddr);876+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_BRC_B32>(vreg_offset,
877+ offsetLocalAddr);
884 }878 }
885 AscendC::MicroAPI::MaskReg mask = AscendC::MicroAPI::CreateMask<float, AscendC::MicroAPI::MaskPattern::ALL>();879 AscendC::MicroAPI::MaskReg mask = AscendC::MicroAPI::CreateMask<float, AscendC::MicroAPI::MaskPattern::ALL>();
886 AscendC::MicroAPI::MaskReg mask2 = AscendC::MicroAPI::CreateMask<float, AscendC::MicroAPI::MaskPattern::H>();880 AscendC::MicroAPI::MaskReg mask2 = AscendC::MicroAPI::CreateMask<float, AscendC::MicroAPI::MaskPattern::H>();
887 for (uint16_t i = 0; i < vfLoopNum; i++) {881 for (uint16_t i = 0; i < vfLoopNum; i++) {
888 auto addr = outAddr + i * VL;882 auto addr = outAddr + i * VL;
889- AscendC::MicroAPI::DataCopy<T, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(vreg0, inLocalAddr + i * VL);883+ AscendC::MicroAPI::LoadAlign<T, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(vreg0, inLocalAddr + i * VL);
890 AscendC::MicroAPI::Cast<float, T, castTrait0>(vreg1, vreg0, mask);884 AscendC::MicroAPI::Cast<float, T, castTrait0>(vreg1, vreg0, mask);
891 885 
892 if constexpr (hasSmooth == 1) {886 if constexpr (hasSmooth == 1) {
893- AscendC::MicroAPI::DataCopy<T, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(vreg2,887+ AscendC::MicroAPI::LoadAlign<T, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(vreg2,
894- smoothLocalAddr + i * VL);888+ smoothLocalAddr + i * VL);
895 AscendC::MicroAPI::Cast<float, T, castTrait0>(vreg3, vreg2, mask);889 AscendC::MicroAPI::Cast<float, T, castTrait0>(vreg3, vreg2, mask);
896 AscendC::MicroAPI::Mul(vreg4, vreg1, vreg3, mask);890 AscendC::MicroAPI::Mul(vreg4, vreg1, vreg3, mask);
897 AscendC::MicroAPI::Div(vreg5, vreg4, vreg9, mask);891 AscendC::MicroAPI::Div(vreg5, vreg4, vreg9, mask);
@@ -921,11 +915,11 @@ __aicore__ inline void DynamicQuantLargeShapeDbPertensor<T, yDtype, hasSmooth, i
921 }915 }
922 916 
923 if constexpr (IsSameType<yDtype, int4b_t>::value) {917 if constexpr (IsSameType<yDtype, int4b_t>::value) {
924- AscendC::MicroAPI::DataCopy<yCopyDtype, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(addr, vreg8,918+ AscendC::MicroAPI::StoreAlign<yCopyDtype, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(addr, vreg8,
925- mask2);919+ mask2);
926 } else {920 } else {
927- AscendC::MicroAPI::DataCopy<yCopyDtype, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(addr, vreg8,921+ AscendC::MicroAPI::StoreAlign<yCopyDtype, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(addr, vreg8,
928- mask);922+ mask);
929 }923 }
930 }924 }
931 }925 }
Mquant/dynamic_quant/op_kernel/arch35/dynamic_quant_regbase_moe_full_load.h+36-37
@@ -201,19 +201,19 @@ private:
201 LocalTensor<float> scaleLocal = scaleQueue.template AllocTensor<float>();201 LocalTensor<float> scaleLocal = scaleQueue.template AllocTensor<float>();
202 LocalTensor<float> offsetLocal;202 LocalTensor<float> offsetLocal;
203 203 
204- __local_mem__ xDtype* xAddr = (__local_mem__ xDtype*)xLocal.GetPhyAddr();204+ __ubuf__ xDtype* xAddr = (__ubuf__ xDtype*)xLocal.GetPhyAddr();
205- __local_mem__ xDtype* smoothAddr;205+ __ubuf__ xDtype* smoothAddr;
206 206 
207- __local_mem__ yCopyDtype* yAddr = (__local_mem__ yCopyDtype*)yLocal.GetPhyAddr();207+ __ubuf__ yCopyDtype* yAddr = (__ubuf__ yCopyDtype*)yLocal.GetPhyAddr();
208- __local_mem__ float* scaleAddr = (__local_mem__ float*)scaleLocal.GetPhyAddr();208+ __ubuf__ float* scaleAddr = (__ubuf__ float*)scaleLocal.GetPhyAddr();
209- __local_mem__ float* offsetAddr;209+ __ubuf__ float* offsetAddr;
210 210 
211 if constexpr (isSymmetrical == false) {211 if constexpr (isSymmetrical == false) {
212 offsetLocal = offsetQueue.template AllocTensor<float>();212 offsetLocal = offsetQueue.template AllocTensor<float>();
213- offsetAddr = (__local_mem__ float*)offsetLocal.GetPhyAddr();213+ offsetAddr = (__ubuf__ float*)offsetLocal.GetPhyAddr();
214 }214 }
215 215 
216- smoothAddr = (__local_mem__ xDtype*)smoothLocal.GetPhyAddr();216+ smoothAddr = (__ubuf__ xDtype*)smoothLocal.GetPhyAddr();
217 217 
218 for (int32_t i = 0; i < multiRow; i++) {218 for (int32_t i = 0; i < multiRow; i++) {
219 realRowNum = offsetRow + i + 1;219 realRowNum = offsetRow + i + 1;
@@ -224,7 +224,7 @@ private:
224 smoothOffset = smoothIndex * tilingData_.rowLen;224 smoothOffset = smoothIndex * tilingData_.rowLen;
225 SmoothCopyIn(smoothOffset);225 SmoothCopyIn(smoothOffset);
226 smoothLocal = smoothQueue.template DeQue<xDtype>();226 smoothLocal = smoothQueue.template DeQue<xDtype>();
227- smoothAddr = (__local_mem__ xDtype*)smoothLocal.GetPhyAddr();227+ smoothAddr = (__ubuf__ xDtype*)smoothLocal.GetPhyAddr();
228 }228 }
229 ComputeVF(xAddr, smoothAddr, yAddr, scaleAddr + i, offsetAddr + i, i);229 ComputeVF(xAddr, smoothAddr, yAddr, scaleAddr + i, offsetAddr + i, i);
230 }230 }
@@ -267,9 +267,8 @@ private:
267 scaleQueue.FreeTensor(scaleLocal);267 scaleQueue.FreeTensor(scaleLocal);
268 }268 }
269 269 
270- __aicore__ inline void ComputeVF(__local_mem__ xDtype* xAddr, __local_mem__ xDtype* smoothAddr,270+ __aicore__ inline void ComputeVF(__ubuf__ xDtype* xAddr, __ubuf__ xDtype* smoothAddr, __ubuf__ yCopyDtype* yAddr,
271- __local_mem__ yCopyDtype* yAddr, __local_mem__ float* scaleAddr,271+ __ubuf__ float* scaleAddr, __ubuf__ float* offsetAddr, int32_t multiRow)
272- __local_mem__ float* offsetAddr, int32_t multiRow)
273 {272 {
274 uint32_t dtypeSize = sizeof(float);273 uint32_t dtypeSize = sizeof(float);
275 uint16_t VL = AscendC::VECTOR_REG_WIDTH / dtypeSize;274 uint16_t VL = AscendC::VECTOR_REG_WIDTH / dtypeSize;
@@ -324,28 +323,28 @@ private:
324 AscendC::MicroAPI::MaskReg323 AscendC::MicroAPI::MaskReg
325 preg5 = AscendC::MicroAPI::CreateMask<float, AscendC::MicroAPI::MaskPattern::VL1>();324 preg5 = AscendC::MicroAPI::CreateMask<float, AscendC::MicroAPI::MaskPattern::VL1>();
326 325 
327- AscendC::MicroAPI::UnalignReg ureg0;326+ AscendC::MicroAPI::UnalignRegForStore ureg0;
328- AscendC::MicroAPI::UnalignReg ureg1;327+ AscendC::MicroAPI::UnalignRegForStore ureg1;
329 uint32_t sreg0 = rowCount;328 uint32_t sreg0 = rowCount;
330 AscendC::MicroAPI::Duplicate(vreg7, NEG_INFINITY);329 AscendC::MicroAPI::Duplicate(vreg7, NEG_INFINITY);
331 330 
332 if constexpr (isSymmetrical == true) {331 if constexpr (isSymmetrical == true) {
333 for (uint16_t j = 0; j < vfLoop; j++) {332 for (uint16_t j = 0; j < vfLoop; j++) {
334 preg0 = AscendC::MicroAPI::UpdateMask<float>(sreg0);333 preg0 = AscendC::MicroAPI::UpdateMask<float>(sreg0);
335- AscendC::MicroAPI::DataCopy<xDtype, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(334+ AscendC::MicroAPI::LoadAlign<xDtype, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(
336 vreg1, xAddr + multiRow * rowCount + j * VL);335 vreg1, xAddr + multiRow * rowCount + j * VL);
337 AscendC::MicroAPI::Cast<float, xDtype, castTraitB16ToB32>(vreg3, vreg1, preg0);336 AscendC::MicroAPI::Cast<float, xDtype, castTraitB16ToB32>(vreg3, vreg1, preg0);
338- AscendC::MicroAPI::DataCopy<xDtype, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(337+ AscendC::MicroAPI::LoadAlign<xDtype, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(
339 vreg2, smoothAddr + j * VL);338 vreg2, smoothAddr + j * VL);
340 AscendC::MicroAPI::Cast<float, xDtype, castTraitB16ToB32>(vreg4, vreg2, preg0);339 AscendC::MicroAPI::Cast<float, xDtype, castTraitB16ToB32>(vreg4, vreg2, preg0);
341 AscendC::MicroAPI::Mul(vreg3, vreg3, vreg4, preg0);340 AscendC::MicroAPI::Mul(vreg3, vreg3, vreg4, preg0);
342 AscendC::MicroAPI::Abs(vreg6, vreg3, preg0);341 AscendC::MicroAPI::Abs(vreg6, vreg3, preg0);
343 AscendC::MicroAPI::Max(vreg7, vreg6, vreg7, preg1);342 AscendC::MicroAPI::Max(vreg7, vreg6, vreg7, preg1);
344 }343 }
345- AscendC::MicroAPI::ReduceMax(vreg8, vreg7, preg1);344+ AscendC::MicroAPI::Reduce<MicroAPI::ReduceType::MAX>(vreg8, vreg7, preg1);
346 AscendC::MicroAPI::Muls(vreg9, vreg8, maxValue, preg1);345 AscendC::MicroAPI::Muls(vreg9, vreg8, maxValue, preg1);
347 AscendC::MicroAPI::Duplicate(vreg10, vreg9, preg1);346 AscendC::MicroAPI::Duplicate(vreg10, vreg9, preg1);
348- AscendC::MicroAPI::DataCopyUnAlign<float, AscendC::MicroAPI::PostLiteral::POST_MODE_UPDATE>(347+ AscendC::MicroAPI::StoreUnAlign<float, AscendC::MicroAPI::PostLiteral::POST_MODE_UPDATE>(
349 scaleAddr, vreg9, ureg0, 1);348 scaleAddr, vreg9, ureg0, 1);
350 }349 }
351 350 
@@ -353,31 +352,31 @@ private:
353 AscendC::MicroAPI::Duplicate(vregMinX, POS_INFINITY);352 AscendC::MicroAPI::Duplicate(vregMinX, POS_INFINITY);
354 for (uint16_t j = 0; j < vfLoop - 1; j++) {353 for (uint16_t j = 0; j < vfLoop - 1; j++) {
355 preg0 = AscendC::MicroAPI::UpdateMask<float>(sreg0);354 preg0 = AscendC::MicroAPI::UpdateMask<float>(sreg0);
356- AscendC::MicroAPI::DataCopy<xDtype, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(355+ AscendC::MicroAPI::LoadAlign<xDtype, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(
357 vreg1, xAddr + multiRow * rowCount + j * VL);356 vreg1, xAddr + multiRow * rowCount + j * VL);
358 AscendC::MicroAPI::Cast<float, xDtype, castTraitB16ToB32>(vreg3, vreg1, preg0);357 AscendC::MicroAPI::Cast<float, xDtype, castTraitB16ToB32>(vreg3, vreg1, preg0);
359- AscendC::MicroAPI::DataCopy<xDtype, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(358+ AscendC::MicroAPI::LoadAlign<xDtype, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(
360 vreg2, smoothAddr + j * VL);359 vreg2, smoothAddr + j * VL);
361 AscendC::MicroAPI::Cast<float, xDtype, castTraitB16ToB32>(vreg4, vreg2, preg0);360 AscendC::MicroAPI::Cast<float, xDtype, castTraitB16ToB32>(vreg4, vreg2, preg0);
362 AscendC::MicroAPI::Mul(vreg3, vreg3, vreg4, preg0);361 AscendC::MicroAPI::Mul(vreg3, vreg3, vreg4, preg0);
363 AscendC::MicroAPI::Max(vreg7, vreg3, vreg7, preg1);362 AscendC::MicroAPI::Max(vreg7, vreg3, vreg7, preg1);
364 AscendC::MicroAPI::Min(vregMinX, vreg3, vregMinX, preg1);363 AscendC::MicroAPI::Min(vregMinX, vreg3, vregMinX, preg1);
365 }364 }
366- AscendC::MicroAPI::ReduceMax(vregReduceMaxX, vreg7, preg1);365+ AscendC::MicroAPI::Reduce<MicroAPI::ReduceType::MAX>(vregReduceMaxX, vreg7, preg1);
367- AscendC::MicroAPI::ReduceMin(vregReduceMinX, vregMinX, preg1);366+ AscendC::MicroAPI::Reduce<MicroAPI::ReduceType::MIN>(vregReduceMinX, vregMinX, preg1);
368 367 
369 preg4 = AscendC::MicroAPI::UpdateMask<float>(tailNum);368 preg4 = AscendC::MicroAPI::UpdateMask<float>(tailNum);
370- AscendC::MicroAPI::DataCopy<xDtype, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(369+ AscendC::MicroAPI::LoadAlign<xDtype, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(
371 vreg1, xAddr + multiRow * rowCount + (vfLoop - 1) * VL);370 vreg1, xAddr + multiRow * rowCount + (vfLoop - 1) * VL);
372 AscendC::MicroAPI::Cast<float, xDtype, castTraitB16ToB32>(vreg3, vreg1, preg4);371 AscendC::MicroAPI::Cast<float, xDtype, castTraitB16ToB32>(vreg3, vreg1, preg4);
373- AscendC::MicroAPI::DataCopy<xDtype, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(372+ AscendC::MicroAPI::LoadAlign<xDtype, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(
374 vreg2, smoothAddr + (vfLoop - 1) * VL);373 vreg2, smoothAddr + (vfLoop - 1) * VL);
375 AscendC::MicroAPI::Cast<float, xDtype, castTraitB16ToB32>(vreg4, vreg2, preg4);374 AscendC::MicroAPI::Cast<float, xDtype, castTraitB16ToB32>(vreg4, vreg2, preg4);
376 AscendC::MicroAPI::Mul(vreg3, vreg3, vreg4, preg4);375 AscendC::MicroAPI::Mul(vreg3, vreg3, vreg4, preg4);
377 AscendC::MicroAPI::Max(vreg7, vreg3, vreg7, preg4);376 AscendC::MicroAPI::Max(vreg7, vreg3, vreg7, preg4);
378 AscendC::MicroAPI::Min(vregMinX, vreg3, vregMinX, preg4);377 AscendC::MicroAPI::Min(vregMinX, vreg3, vregMinX, preg4);
379- AscendC::MicroAPI::ReduceMax(vregReduceMaxXTail, vreg7, preg4);378+ AscendC::MicroAPI::Reduce<MicroAPI::ReduceType::MAX>(vregReduceMaxXTail, vreg7, preg4);
380- AscendC::MicroAPI::ReduceMin(vregReduceMinXTail, vregMinX, preg4);379+ AscendC::MicroAPI::Reduce<MicroAPI::ReduceType::MIN>(vregReduceMinXTail, vregMinX, preg4);
381 380 
382 AscendC::MicroAPI::Max(vregFinalMax, vregReduceMaxX, vregReduceMaxXTail, preg5);381 AscendC::MicroAPI::Max(vregFinalMax, vregReduceMaxX, vregReduceMaxXTail, preg5);
383 AscendC::MicroAPI::Min(vregFinalMin, vregReduceMinX, vregReduceMinXTail, preg5);382 AscendC::MicroAPI::Min(vregFinalMin, vregReduceMinX, vregReduceMinXTail, preg5);
@@ -385,13 +384,13 @@ private:
385 AscendC::MicroAPI::Sub(vregMaxSubMin, vregFinalMax, vregFinalMin, preg5);384 AscendC::MicroAPI::Sub(vregMaxSubMin, vregFinalMax, vregFinalMin, preg5);
386 AscendC::MicroAPI::Muls(vreg9, vregMaxSubMin, offsetDivValue, preg5);385 AscendC::MicroAPI::Muls(vreg9, vregMaxSubMin, offsetDivValue, preg5);
387 AscendC::MicroAPI::Duplicate(vreg10, vreg9, preg1);386 AscendC::MicroAPI::Duplicate(vreg10, vreg9, preg1);
388- AscendC::MicroAPI::DataCopyUnAlign<float, AscendC::MicroAPI::PostLiteral::POST_MODE_UPDATE>(387+ AscendC::MicroAPI::StoreUnAlign<float, AscendC::MicroAPI::PostLiteral::POST_MODE_UPDATE>(
389 scaleAddr, vreg9, ureg0, 1);388 scaleAddr, vreg9, ureg0, 1);
390 AscendC::MicroAPI::Div<float, &mode>(vregMaxDivScale, vregFinalMax, vreg9, preg5);389 AscendC::MicroAPI::Div<float, &mode>(vregMaxDivScale, vregFinalMax, vreg9, preg5);
391 AscendC::MicroAPI::Muls(vregNegMaxDivScale, vregMaxDivScale, NEGATIVE_ONE, preg5);390 AscendC::MicroAPI::Muls(vregNegMaxDivScale, vregMaxDivScale, NEGATIVE_ONE, preg5);
392 AscendC::MicroAPI::Adds(vregOffset, vregNegMaxDivScale, offsetValue, preg5); //391 AscendC::MicroAPI::Adds(vregOffset, vregNegMaxDivScale, offsetValue, preg5); //
393 AscendC::MicroAPI::Duplicate(vregDupOffset, vregOffset, preg1);392 AscendC::MicroAPI::Duplicate(vregDupOffset, vregOffset, preg1);
394- AscendC::MicroAPI::DataCopyUnAlign<float, AscendC::MicroAPI::PostLiteral::POST_MODE_UPDATE>(393+ AscendC::MicroAPI::StoreUnAlign<float, AscendC::MicroAPI::PostLiteral::POST_MODE_UPDATE>(
395 offsetAddr, vregOffset, ureg1, 1);394 offsetAddr, vregOffset, ureg1, 1);
396 }395 }
397 396 
@@ -399,11 +398,11 @@ private:
399 for (uint16_t j = 0; j < vfLoop; j++) {398 for (uint16_t j = 0; j < vfLoop; j++) {
400 auto addr = yAddr + multiRow * outAlignLen + j * VL;399 auto addr = yAddr + multiRow * outAlignLen + j * VL;
401 preg2 = AscendC::MicroAPI::UpdateMask<float>(sreg1);400 preg2 = AscendC::MicroAPI::UpdateMask<float>(sreg1);
402- AscendC::MicroAPI::DataCopy<xDtype, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(401+ AscendC::MicroAPI::LoadAlign<xDtype, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(
403 vreg1, xAddr + multiRow * rowCount + j * VL);402 vreg1, xAddr + multiRow * rowCount + j * VL);
404 AscendC::MicroAPI::Cast<float, xDtype, castTraitB16ToB32>(vreg3, vreg1, preg2);403 AscendC::MicroAPI::Cast<float, xDtype, castTraitB16ToB32>(vreg3, vreg1, preg2);
405- AscendC::MicroAPI::DataCopy<xDtype, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(vreg2,404+ AscendC::MicroAPI::LoadAlign<xDtype, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(vreg2,
406- smoothAddr + j * VL);405+ smoothAddr + j * VL);
407 AscendC::MicroAPI::Cast<float, xDtype, castTraitB16ToB32>(vreg4, vreg2, preg2);406 AscendC::MicroAPI::Cast<float, xDtype, castTraitB16ToB32>(vreg4, vreg2, preg2);
408 AscendC::MicroAPI::Mul(vreg3, vreg3, vreg4, preg2);407 AscendC::MicroAPI::Mul(vreg3, vreg3, vreg4, preg2);
409 if constexpr (isSymmetrical == true) {408 if constexpr (isSymmetrical == true) {
@@ -431,16 +430,16 @@ private:
431 addr = yAddr + (multiRow * outAlignLen + j * VL) / 2;430 addr = yAddr + (multiRow * outAlignLen + j * VL) / 2;
432 }431 }
433 if constexpr (IsSameType<yDtype, int4b_t>::value) {432 if constexpr (IsSameType<yDtype, int4b_t>::value) {
434- AscendC::MicroAPI::DataCopy<yCopyDtype, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(addr, vreg19,433+ AscendC::MicroAPI::StoreAlign<yCopyDtype, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(
435- preg3);434+ addr, vreg19, preg3);
436 } else {435 } else {
437- AscendC::MicroAPI::DataCopy<yCopyDtype, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(addr, vreg19,436+ AscendC::MicroAPI::StoreAlign<yCopyDtype, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(
438- preg2);437+ addr, vreg19, preg2);
439 }438 }
440 }439 }
441- AscendC::MicroAPI::DataCopyUnAlignPost(scaleAddr, ureg0, 0);440+ AscendC::MicroAPI::StoreUnAlignPost(scaleAddr, ureg0, 0);
442 if constexpr (isSymmetrical == false) {441 if constexpr (isSymmetrical == false) {
443- AscendC::MicroAPI::DataCopyUnAlignPost(offsetAddr, ureg1, 0);442+ AscendC::MicroAPI::StoreUnAlignPost(offsetAddr, ureg1, 0);
444 }443 }
445 }444 }
446 }445 }
@@ -513,4 +512,4 @@ private:
513 RoundMode::CAST_ROUND};512 RoundMode::CAST_ROUND};
514};513};
515} // namespace DynamicQuantV2Op514} // namespace DynamicQuantV2Op
516-#endif515+#endif
Mquant/dynamic_quant/op_kernel/arch35/dynamic_quant_regbase_moe_full_load_pertensor.h+104-107
@@ -100,31 +100,31 @@ public:
100 LocalTensor<float> MinOutLocal;100 LocalTensor<float> MinOutLocal;
101 LocalTensor<float> scaleOutLocal;101 LocalTensor<float> scaleOutLocal;
102 LocalTensor<float> offsetLocal;102 LocalTensor<float> offsetLocal;
103- __local_mem__ float* MaxOutLocalAddr = nullptr;103+ __ubuf__ float* MaxOutLocalAddr = nullptr;
104- __local_mem__ float* MinOutLocalAddr = nullptr;104+ __ubuf__ float* MinOutLocalAddr = nullptr;
105- __local_mem__ float* scaleOutLocalAddr = nullptr;105+ __ubuf__ float* scaleOutLocalAddr = nullptr;
106- __local_mem__ float* offsetLocalAddr = nullptr;106+ __ubuf__ float* offsetLocalAddr = nullptr;
107 107 
108 if constexpr (isSymmetrical == false) {108 if constexpr (isSymmetrical == false) {
109 MaxOutLocal = MaxOutQueue.template AllocTensor<float>();109 MaxOutLocal = MaxOutQueue.template AllocTensor<float>();
110 AscendC::Duplicate(MaxOutLocal, MIN_FLOAT_VALUE, SIXTY_FOUR, ONE, ONE, EIGHT);110 AscendC::Duplicate(MaxOutLocal, MIN_FLOAT_VALUE, SIXTY_FOUR, ONE, ONE, EIGHT);
111- MaxOutLocalAddr = (__local_mem__ float*)MaxOutLocal.GetPhyAddr();111+ MaxOutLocalAddr = (__ubuf__ float*)MaxOutLocal.GetPhyAddr();
112 112 
113 MinOutLocal = MinOutQueue.template AllocTensor<float>();113 MinOutLocal = MinOutQueue.template AllocTensor<float>();
114 AscendC::Duplicate(MinOutLocal, MAX_FLOAT_VALUE, SIXTY_FOUR, ONE, ONE, EIGHT);114 AscendC::Duplicate(MinOutLocal, MAX_FLOAT_VALUE, SIXTY_FOUR, ONE, ONE, EIGHT);
115- MinOutLocalAddr = (__local_mem__ float*)MinOutLocal.GetPhyAddr();115+ MinOutLocalAddr = (__ubuf__ float*)MinOutLocal.GetPhyAddr();
116 116 
117 scaleOutLocal = scaleOutQueue.template AllocTensor<float>();117 scaleOutLocal = scaleOutQueue.template AllocTensor<float>();
118 AscendC::Duplicate(scaleOutLocal, (float)0.0, SIXTY_FOUR, ONE, ONE, EIGHT);118 AscendC::Duplicate(scaleOutLocal, (float)0.0, SIXTY_FOUR, ONE, ONE, EIGHT);
119- scaleOutLocalAddr = (__local_mem__ float*)scaleOutLocal.GetPhyAddr();119+ scaleOutLocalAddr = (__ubuf__ float*)scaleOutLocal.GetPhyAddr();
120 120 
121 offsetLocal = offsetQueue.template AllocTensor<float>();121 offsetLocal = offsetQueue.template AllocTensor<float>();
122 AscendC::Duplicate(offsetLocal, (float)0.0, SIXTY_FOUR, ONE, ONE, EIGHT);122 AscendC::Duplicate(offsetLocal, (float)0.0, SIXTY_FOUR, ONE, ONE, EIGHT);
123- offsetLocalAddr = (__local_mem__ float*)offsetLocal.GetPhyAddr();123+ offsetLocalAddr = (__ubuf__ float*)offsetLocal.GetPhyAddr();
124 } else {124 } else {
125 scaleOutLocal = scaleOutQueue.template AllocTensor<float>();125 scaleOutLocal = scaleOutQueue.template AllocTensor<float>();
126 AscendC::Duplicate(scaleOutLocal, (float)0.0, SIXTY_FOUR, ONE, ONE, EIGHT);126 AscendC::Duplicate(scaleOutLocal, (float)0.0, SIXTY_FOUR, ONE, ONE, EIGHT);
127- scaleOutLocalAddr = (__local_mem__ float*)scaleOutLocal.GetPhyAddr();127+ scaleOutLocalAddr = (__ubuf__ float*)scaleOutLocal.GetPhyAddr();
128 }128 }
129 scaleOffset = 0;129 scaleOffset = 0;
130 CopyInScaleByEle(scaleOffset, tilingData_.coreNum);130 CopyInScaleByEle(scaleOffset, tilingData_.coreNum);
@@ -149,13 +149,13 @@ public:
149 __aicore__ inline void ProcessY()149 __aicore__ inline void ProcessY()
150 {150 {
151 LocalTensor<float> scaleOutLocal = scaleOutQueue.template DeQue<float>();151 LocalTensor<float> scaleOutLocal = scaleOutQueue.template DeQue<float>();
152- __local_mem__ float* scaleOutLocalAddr = (__local_mem__ float*)scaleOutLocal.GetPhyAddr();152+ __ubuf__ float* scaleOutLocalAddr = (__ubuf__ float*)scaleOutLocal.GetPhyAddr();
153 LocalTensor<float> offsetLocal;153 LocalTensor<float> offsetLocal;
154- __local_mem__ float* offsetLocalAddr = nullptr;154+ __ubuf__ float* offsetLocalAddr = nullptr;
155 155 
156 if constexpr (isSymmetrical == false) {156 if constexpr (isSymmetrical == false) {
157 offsetLocal = offsetQueue.template DeQue<float>();157 offsetLocal = offsetQueue.template DeQue<float>();
158- offsetLocalAddr = (__local_mem__ float*)offsetLocal.GetPhyAddr();158+ offsetLocalAddr = (__ubuf__ float*)offsetLocal.GetPhyAddr();
159 }159 }
160 160 
161 smoothIndex = 0;161 smoothIndex = 0;
@@ -280,12 +280,12 @@ private:
280 curSmoothIdx = smoothIdx;280 curSmoothIdx = smoothIdx;
281 }281 }
282 282 
283- __aicore__ inline __local_mem__ xDtype* GetSmoothRowAddr(uint32_t smoothIdx)283+ __aicore__ inline __ubuf__ xDtype* GetSmoothRowAddr(uint32_t smoothIdx)
284 {284 {
285 if (smoothIdx != curSmoothIdx) {285 if (smoothIdx != curSmoothIdx) {
286 LoadSmooth(smoothIdx);286 LoadSmooth(smoothIdx);
287 }287 }
288- return (__local_mem__ xDtype*)smoothLocal.GetPhyAddr();288+ return (__ubuf__ xDtype*)smoothLocal.GetPhyAddr();
289 }289 }
290 290 
291 __aicore__ inline uint32_t GetSmoothIndexLocal(int32_t realRowNum, int32_t& groupVal, uint32_t startIndex)291 __aicore__ inline uint32_t GetSmoothIndexLocal(int32_t realRowNum, int32_t& groupVal, uint32_t startIndex)
@@ -347,10 +347,9 @@ private:
347 }347 }
348 }348 }
349 349 
350- __aicore__ inline void ComputeMaxRowScaleVF(__local_mem__ xDtype* inLocalAddr,350+ __aicore__ inline void ComputeMaxRowScaleVF(__ubuf__ xDtype* inLocalAddr, __ubuf__ xDtype* smoothLocalAddr,
351- __local_mem__ xDtype* smoothLocalAddr,351+ __ubuf__ float* scaleLocalAddr, __ubuf__ float* maxLocalAddr,
352- __local_mem__ float* scaleLocalAddr, __local_mem__ float* maxLocalAddr,352+ __ubuf__ float* minLocalAddr, uint32_t multiRow)
353- __local_mem__ float* minLocalAddr, uint32_t multiRow)
354 {353 {
355 uint32_t dtypeSize = sizeof(float);354 uint32_t dtypeSize = sizeof(float);
356 uint16_t VL = AscendC::VECTOR_REG_WIDTH / dtypeSize;355 uint16_t VL = AscendC::VECTOR_REG_WIDTH / dtypeSize;
@@ -378,10 +377,10 @@ private:
378 preg1 = AscendC::MicroAPI::CreateMask<float, AscendC::MicroAPI::MaskPattern::ALL>();377 preg1 = AscendC::MicroAPI::CreateMask<float, AscendC::MicroAPI::MaskPattern::ALL>();
379 378 
380 if constexpr (isSymmetrical == false) {379 if constexpr (isSymmetrical == false) {
381- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_BRC_B32>(vreg9_1, maxLocalAddr);380+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_BRC_B32>(vreg9_1, maxLocalAddr);
382- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_BRC_B32>(vreg9_2, minLocalAddr);381+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_BRC_B32>(vreg9_2, minLocalAddr);
383 } else {382 } else {
384- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_BRC_B32>(vreg9_1, scaleLocalAddr);383+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_BRC_B32>(vreg9_1, scaleLocalAddr);
385 }384 }
386 385 
387 for (uint16_t i = 0; i < static_cast<uint16_t>(multiRow); i++) {386 for (uint16_t i = 0; i < static_cast<uint16_t>(multiRow); i++) {
@@ -394,10 +393,10 @@ private:
394 }393 }
395 for (uint16_t j = 0; j < static_cast<uint16_t>(vfLoop - 1); j++) {394 for (uint16_t j = 0; j < static_cast<uint16_t>(vfLoop - 1); j++) {
396 preg0 = AscendC::MicroAPI::UpdateMask<float>(sreg0);395 preg0 = AscendC::MicroAPI::UpdateMask<float>(sreg0);
397- AscendC::MicroAPI::DataCopy<xDtype, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(396+ AscendC::MicroAPI::LoadAlign<xDtype, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(
398 vreg1, inLocalAddr + i * rowCount + j * VL);397 vreg1, inLocalAddr + i * rowCount + j * VL);
399 AscendC::MicroAPI::Cast<float, xDtype, castTraitB16ToB32>(vreg3, vreg1, preg0);398 AscendC::MicroAPI::Cast<float, xDtype, castTraitB16ToB32>(vreg3, vreg1, preg0);
400- AscendC::MicroAPI::DataCopy<xDtype, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(399+ AscendC::MicroAPI::LoadAlign<xDtype, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(
401 vreg2, smoothLocalAddr + j * VL);400 vreg2, smoothLocalAddr + j * VL);
402 AscendC::MicroAPI::Cast<float, xDtype, castTraitB16ToB32>(vreg4, vreg2, preg0);401 AscendC::MicroAPI::Cast<float, xDtype, castTraitB16ToB32>(vreg4, vreg2, preg0);
403 AscendC::MicroAPI::Mul(vreg3, vreg3, vreg4, preg0);402 AscendC::MicroAPI::Mul(vreg3, vreg3, vreg4, preg0);
@@ -411,29 +410,29 @@ private:
411 }410 }
412 }411 }
413 if constexpr (isSymmetrical == false) {412 if constexpr (isSymmetrical == false) {
414- AscendC::MicroAPI::ReduceMax(vreg8_1, vreg7_1, preg1);413+ AscendC::MicroAPI::Reduce<MicroAPI::ReduceType::MAX>(vreg8_1, vreg7_1, preg1);
415- AscendC::MicroAPI::ReduceMin(vreg8_2, vreg7_2, preg1);414+ AscendC::MicroAPI::Reduce<MicroAPI::ReduceType::MIN>(vreg8_2, vreg7_2, preg1);
416 } else {415 } else {
417- AscendC::MicroAPI::ReduceMax(vreg8_1, vreg7_1, preg1);416+ AscendC::MicroAPI::Reduce<MicroAPI::ReduceType::MAX>(vreg8_1, vreg7_1, preg1);
418 }417 }
419 for (uint16_t j = static_cast<uint16_t>(vfLoop - 1); j < vfLoop; j++) {418 for (uint16_t j = static_cast<uint16_t>(vfLoop - 1); j < vfLoop; j++) {
420 preg0 = AscendC::MicroAPI::UpdateMask<float>(sreg0);419 preg0 = AscendC::MicroAPI::UpdateMask<float>(sreg0);
421- AscendC::MicroAPI::DataCopy<xDtype, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(420+ AscendC::MicroAPI::LoadAlign<xDtype, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(
422 vreg1, inLocalAddr + i * rowCount + j * VL);421 vreg1, inLocalAddr + i * rowCount + j * VL);
423 AscendC::MicroAPI::Cast<float, xDtype, castTraitB16ToB32>(vreg3, vreg1, preg0);422 AscendC::MicroAPI::Cast<float, xDtype, castTraitB16ToB32>(vreg3, vreg1, preg0);
424- AscendC::MicroAPI::DataCopy<xDtype, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(423+ AscendC::MicroAPI::LoadAlign<xDtype, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(
425 vreg2, smoothLocalAddr + j * VL);424 vreg2, smoothLocalAddr + j * VL);
426 AscendC::MicroAPI::Cast<float, xDtype, castTraitB16ToB32>(vreg4, vreg2, preg0);425 AscendC::MicroAPI::Cast<float, xDtype, castTraitB16ToB32>(vreg4, vreg2, preg0);
427 AscendC::MicroAPI::Mul(vreg3, vreg3, vreg4, preg0);426 AscendC::MicroAPI::Mul(vreg3, vreg3, vreg4, preg0);
428 if constexpr (isSymmetrical == false) {427 if constexpr (isSymmetrical == false) {
429- AscendC::MicroAPI::ReduceMax(vreg7_1, vreg3, preg0);428+ AscendC::MicroAPI::Reduce<MicroAPI::ReduceType::MAX>(vreg7_1, vreg3, preg0);
430- AscendC::MicroAPI::ReduceMin(vreg7_2, vreg3, preg0);429+ AscendC::MicroAPI::Reduce<MicroAPI::ReduceType::MIN>(vreg7_2, vreg3, preg0);
431 AscendC::MicroAPI::Max(vreg8_1, vreg8_1, vreg7_1, preg0);430 AscendC::MicroAPI::Max(vreg8_1, vreg8_1, vreg7_1, preg0);
432 AscendC::MicroAPI::Min(vreg8_2, vreg8_2, vreg7_2, preg0);431 AscendC::MicroAPI::Min(vreg8_2, vreg8_2, vreg7_2, preg0);
433 } else {432 } else {
434 AscendC::MicroAPI::Abs(vreg6_1, vreg3, preg0);433 AscendC::MicroAPI::Abs(vreg6_1, vreg3, preg0);
435 AscendC::MicroAPI::Muls(vreg6_1, vreg6_1, float(1.0) / maxValue, preg0);434 AscendC::MicroAPI::Muls(vreg6_1, vreg6_1, float(1.0) / maxValue, preg0);
436- AscendC::MicroAPI::ReduceMax(vreg7_1, vreg6_1, preg0);435+ AscendC::MicroAPI::Reduce<MicroAPI::ReduceType::MAX>(vreg7_1, vreg6_1, preg0);
437 AscendC::MicroAPI::Max(vreg8_1, vreg8_1, vreg7_1, preg0);436 AscendC::MicroAPI::Max(vreg8_1, vreg8_1, vreg7_1, preg0);
438 }437 }
439 }438 }
@@ -445,21 +444,20 @@ private:
445 }444 }
446 }445 }
447 if constexpr (isSymmetrical == false) {446 if constexpr (isSymmetrical == false) {
448- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::StoreDist::DIST_FIRST_ELEMENT_B32>(447+ AscendC::MicroAPI::StoreAlign<float, AscendC::MicroAPI::StoreDist::DIST_FIRST_ELEMENT_B32>(
449 maxLocalAddr, vreg9_1, preg0);448 maxLocalAddr, vreg9_1, preg0);
450- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::StoreDist::DIST_FIRST_ELEMENT_B32>(449+ AscendC::MicroAPI::StoreAlign<float, AscendC::MicroAPI::StoreDist::DIST_FIRST_ELEMENT_B32>(
451 minLocalAddr, vreg9_2, preg0);450 minLocalAddr, vreg9_2, preg0);
452 } else {451 } else {
453- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::StoreDist::DIST_FIRST_ELEMENT_B32>(452+ AscendC::MicroAPI::StoreAlign<float, AscendC::MicroAPI::StoreDist::DIST_FIRST_ELEMENT_B32>(
454 scaleLocalAddr, vreg9_1, preg0);453 scaleLocalAddr, vreg9_1, preg0);
455 }454 }
456 }455 }
457 }456 }
458 457 
459- __aicore__ inline void ComputeMaxColScaleVF(__local_mem__ float* scaleLocalAddr,458+ __aicore__ inline void ComputeMaxColScaleVF(__ubuf__ float* scaleLocalAddr, __ubuf__ float* scaleOutLocalAddr,
460- __local_mem__ float* scaleOutLocalAddr,459+ __ubuf__ float* maxLocalAddr, __ubuf__ float* maxOutLocalAddr,
461- __local_mem__ float* maxLocalAddr, __local_mem__ float* maxOutLocalAddr,460+ __ubuf__ float* minLocalAddr, __ubuf__ float* minOutLocalAddr,
462- __local_mem__ float* minLocalAddr, __local_mem__ float* minOutLocalAddr,
463 uint32_t elementNum)461 uint32_t elementNum)
464 {462 {
465 uint32_t dtypeSize = sizeof(float);463 uint32_t dtypeSize = sizeof(float);
@@ -479,11 +477,13 @@ private:
479 AscendC::MicroAPI::RegTensor<float, MicroAPI::RegTraitNumOne> vreg3_2;477 AscendC::MicroAPI::RegTensor<float, MicroAPI::RegTraitNumOne> vreg3_2;
480 478 
481 if constexpr (isSymmetrical == false) {479 if constexpr (isSymmetrical == false) {
482- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_BRC_B32>(vreg1_1, maxOutLocalAddr);480+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_BRC_B32>(vreg1_1,
483- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_BRC_B32>(vreg1_2, minOutLocalAddr);481+ maxOutLocalAddr);
482+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_BRC_B32>(vreg1_2,
483+ minOutLocalAddr);
484 } else {484 } else {
485- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_BRC_B32>(vreg1_1,485+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_BRC_B32>(vreg1_1,
486- scaleOutLocalAddr);486+ scaleOutLocalAddr);
487 }487 }
488 488 
489 AscendC::MicroAPI::MaskReg489 AscendC::MicroAPI::MaskReg
@@ -492,60 +492,60 @@ private:
492 maskNum = elementNum - i * VL;492 maskNum = elementNum - i * VL;
493 mask = AscendC::MicroAPI::UpdateMask<float>(maskNum);493 mask = AscendC::MicroAPI::UpdateMask<float>(maskNum);
494 if constexpr (isSymmetrical == false) {494 if constexpr (isSymmetrical == false) {
495- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(vreg0_1,495+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(vreg0_1,
496- maxLocalAddr + i * VL);496+ maxLocalAddr + i * VL);
497 AscendC::MicroAPI::Max(vreg1_1, vreg0_1, vreg1_1, mask);497 AscendC::MicroAPI::Max(vreg1_1, vreg0_1, vreg1_1, mask);
498- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(vreg0_2,498+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(vreg0_2,
499- minLocalAddr + i * VL);499+ minLocalAddr + i * VL);
500 AscendC::MicroAPI::Min(vreg1_2, vreg0_2, vreg1_2, mask);500 AscendC::MicroAPI::Min(vreg1_2, vreg0_2, vreg1_2, mask);
501 } else {501 } else {
502- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(vreg0_1,502+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(
503- scaleLocalAddr + i * VL);503+ vreg0_1, scaleLocalAddr + i * VL);
504 AscendC::MicroAPI::Max(vreg1_1, vreg0_1, vreg1_1, mask);504 AscendC::MicroAPI::Max(vreg1_1, vreg0_1, vreg1_1, mask);
505 }505 }
506 }506 }
507 if constexpr (isSymmetrical == false) {507 if constexpr (isSymmetrical == false) {
508- AscendC::MicroAPI::ReduceMax<float>(vreg2_1, vreg1_1, mask);508+ AscendC::MicroAPI::Reduce<MicroAPI::ReduceType::MAX, float>(vreg2_1, vreg1_1, mask);
509- AscendC::MicroAPI::ReduceMin<float>(vreg2_2, vreg1_2, mask);509+ AscendC::MicroAPI::Reduce<MicroAPI::ReduceType::MIN, float>(vreg2_2, vreg1_2, mask);
510 } else {510 } else {
511- AscendC::MicroAPI::ReduceMax<float>(vreg2_1, vreg1_1, mask);511+ AscendC::MicroAPI::Reduce<MicroAPI::ReduceType::MAX, float>(vreg2_1, vreg1_1, mask);
512 }512 }
513 for (uint16_t i = vfLoopNum - 1; i < vfLoopNum; i++) {513 for (uint16_t i = vfLoopNum - 1; i < vfLoopNum; i++) {
514 maskNum = elementNum - i * VL;514 maskNum = elementNum - i * VL;
515 mask = AscendC::MicroAPI::UpdateMask<float>(maskNum);515 mask = AscendC::MicroAPI::UpdateMask<float>(maskNum);
516 if constexpr (isSymmetrical == false) {516 if constexpr (isSymmetrical == false) {
517- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(vreg0_1,517+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(vreg0_1,
518- maxLocalAddr + i * VL);518+ maxLocalAddr + i * VL);
519 AscendC::MicroAPI::Max(vreg1_1, vreg0_1, vreg1_1, mask);519 AscendC::MicroAPI::Max(vreg1_1, vreg0_1, vreg1_1, mask);
520- AscendC::MicroAPI::ReduceMax<float>(vreg3_1, vreg1_1, mask);520+ AscendC::MicroAPI::Reduce<MicroAPI::ReduceType::MAX, float>(vreg3_1, vreg1_1, mask);
521 AscendC::MicroAPI::Max(vreg3_1, vreg2_1, vreg3_1, mask);521 AscendC::MicroAPI::Max(vreg3_1, vreg2_1, vreg3_1, mask);
522- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(vreg0_2,522+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(vreg0_2,
523- minLocalAddr + i * VL);523+ minLocalAddr + i * VL);
524 AscendC::MicroAPI::Min(vreg1_2, vreg0_2, vreg1_2, mask);524 AscendC::MicroAPI::Min(vreg1_2, vreg0_2, vreg1_2, mask);
525- AscendC::MicroAPI::ReduceMin<float>(vreg3_2, vreg1_2, mask);525+ AscendC::MicroAPI::Reduce<MicroAPI::ReduceType::MIN, float>(vreg3_2, vreg1_2, mask);
526 AscendC::MicroAPI::Min(vreg3_2, vreg2_2, vreg3_2, mask);526 AscendC::MicroAPI::Min(vreg3_2, vreg2_2, vreg3_2, mask);
527 } else {527 } else {
528- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(vreg0_1,528+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(
529- scaleLocalAddr + i * VL);529+ vreg0_1, scaleLocalAddr + i * VL);
530 AscendC::MicroAPI::Max(vreg1_1, vreg0_1, vreg1_1, mask);530 AscendC::MicroAPI::Max(vreg1_1, vreg0_1, vreg1_1, mask);
531- AscendC::MicroAPI::ReduceMax<float>(vreg3_1, vreg1_1, mask);531+ AscendC::MicroAPI::Reduce<MicroAPI::ReduceType::MAX, float>(vreg3_1, vreg1_1, mask);
532 AscendC::MicroAPI::Max(vreg3_1, vreg2_1, vreg3_1, mask);532 AscendC::MicroAPI::Max(vreg3_1, vreg2_1, vreg3_1, mask);
533 }533 }
534 }534 }
535 if constexpr (isSymmetrical == false) {535 if constexpr (isSymmetrical == false) {
536- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::StoreDist::DIST_FIRST_ELEMENT_B32>(536+ AscendC::MicroAPI::StoreAlign<float, AscendC::MicroAPI::StoreDist::DIST_FIRST_ELEMENT_B32>(
537 maxOutLocalAddr, vreg3_1, mask);537 maxOutLocalAddr, vreg3_1, mask);
538- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::StoreDist::DIST_FIRST_ELEMENT_B32>(538+ AscendC::MicroAPI::StoreAlign<float, AscendC::MicroAPI::StoreDist::DIST_FIRST_ELEMENT_B32>(
539 minOutLocalAddr, vreg3_2, mask);539 minOutLocalAddr, vreg3_2, mask);
540 } else {540 } else {
541- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::StoreDist::DIST_FIRST_ELEMENT_B32>(541+ AscendC::MicroAPI::StoreAlign<float, AscendC::MicroAPI::StoreDist::DIST_FIRST_ELEMENT_B32>(
542 scaleOutLocalAddr, vreg3_1, mask);542 scaleOutLocalAddr, vreg3_1, mask);
543 }543 }
544 }544 }
545 }545 }
546 546 
547- __aicore__ inline void ComputeScaleSymVF(__local_mem__ float* maxLocalAddr, __local_mem__ float* minLocalAddr,547+ __aicore__ inline void ComputeScaleSymVF(__ubuf__ float* maxLocalAddr, __ubuf__ float* minLocalAddr,
548- __local_mem__ float* scaleLocalAddr, uint32_t elementNum)548+ __ubuf__ float* scaleLocalAddr, uint32_t elementNum)
549 {549 {
550 uint32_t dtypeSize = sizeof(float);550 uint32_t dtypeSize = sizeof(float);
551 uint32_t VL = AscendC::VECTOR_REG_WIDTH / dtypeSize;551 uint32_t VL = AscendC::VECTOR_REG_WIDTH / dtypeSize;
@@ -559,20 +559,20 @@ private:
559 for (uint16_t i = 0; i < vfLoopNum; i++) {559 for (uint16_t i = 0; i < vfLoopNum; i++) {
560 maskNum = elementNum - i * VL;560 maskNum = elementNum - i * VL;
561 mask = AscendC::MicroAPI::UpdateMask<float>(maskNum);561 mask = AscendC::MicroAPI::UpdateMask<float>(maskNum);
562- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(vreg0,562+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(vreg0,
563- maxLocalAddr + i * VL);563+ maxLocalAddr + i * VL);
564- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(vreg1,564+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(vreg1,
565- minLocalAddr + i * VL);565+ minLocalAddr + i * VL);
566 AscendC::MicroAPI::Sub(vreg1, vreg0, vreg1, mask);566 AscendC::MicroAPI::Sub(vreg1, vreg0, vreg1, mask);
567 AscendC::MicroAPI::Muls(vreg1, vreg1, float(1.0) / maxValueNoSym, mask);567 AscendC::MicroAPI::Muls(vreg1, vreg1, float(1.0) / maxValueNoSym, mask);
568- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::StoreDist::DIST_FIRST_ELEMENT_B32>(scaleLocalAddr,568+ AscendC::MicroAPI::StoreAlign<float, AscendC::MicroAPI::StoreDist::DIST_FIRST_ELEMENT_B32>(
569- vreg1, mask);569+ scaleLocalAddr, vreg1, mask);
570 }570 }
571 }571 }
572 }572 }
573 573 
574- __aicore__ inline void ComputeOffsetSymVF(__local_mem__ float* maxLocalAddr, __local_mem__ float* scaleLocalAddr,574+ __aicore__ inline void ComputeOffsetSymVF(__ubuf__ float* maxLocalAddr, __ubuf__ float* scaleLocalAddr,
575- __local_mem__ float* offsetLocalAddr, uint32_t elementNum)575+ __ubuf__ float* offsetLocalAddr, uint32_t elementNum)
576 {576 {
577 uint32_t dtypeSize = sizeof(float);577 uint32_t dtypeSize = sizeof(float);
578 uint32_t VL = AscendC::VECTOR_REG_WIDTH / dtypeSize;578 uint32_t VL = AscendC::VECTOR_REG_WIDTH / dtypeSize;
@@ -588,14 +588,14 @@ private:
588 for (uint16_t i = 0; i < vfLoopNum; i++) {588 for (uint16_t i = 0; i < vfLoopNum; i++) {
589 maskNum = elementNum - i * VL;589 maskNum = elementNum - i * VL;
590 mask = AscendC::MicroAPI::UpdateMask<float>(maskNum);590 mask = AscendC::MicroAPI::UpdateMask<float>(maskNum);
591- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(vreg0,591+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(vreg0,
592- maxLocalAddr + i * VL);592+ maxLocalAddr + i * VL);
593- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(vreg1,593+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(vreg1,
594- scaleLocalAddr + i * VL);594+ scaleLocalAddr + i * VL);
595 AscendC::MicroAPI::Div<float, &mode>(vreg1, vreg0, vreg1, mask);595 AscendC::MicroAPI::Div<float, &mode>(vreg1, vreg0, vreg1, mask);
596 AscendC::MicroAPI::Muls(vreg1, vreg1, float(-1.0), mask);596 AscendC::MicroAPI::Muls(vreg1, vreg1, float(-1.0), mask);
597 AscendC::MicroAPI::Adds(vreg1, vreg1, maxValue, mask);597 AscendC::MicroAPI::Adds(vreg1, vreg1, maxValue, mask);
598- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::StoreDist::DIST_FIRST_ELEMENT_B32>(598+ AscendC::MicroAPI::StoreAlign<float, AscendC::MicroAPI::StoreDist::DIST_FIRST_ELEMENT_B32>(
599 offsetLocalAddr, vreg1, mask);599 offsetLocalAddr, vreg1, mask);
600 }600 }
601 }601 }
@@ -604,23 +604,23 @@ private:
604 __aicore__ inline void ComputeMaxRowScaleMoE(int32_t multiRow, int32_t offsetRow)604 __aicore__ inline void ComputeMaxRowScaleMoE(int32_t multiRow, int32_t offsetRow)
605 {605 {
606 LocalTensor<xDtype> inLocal = inQueue.template DeQue<xDtype>();606 LocalTensor<xDtype> inLocal = inQueue.template DeQue<xDtype>();
607- __local_mem__ xDtype* inLocalAddr = (__local_mem__ xDtype*)inLocal.GetPhyAddr();607+ __ubuf__ xDtype* inLocalAddr = (__ubuf__ xDtype*)inLocal.GetPhyAddr();
608 608 
609 LocalTensor<float> maxToWorkSpaceLocal;609 LocalTensor<float> maxToWorkSpaceLocal;
610- __local_mem__ float* maxToWorkSpaceLocalAddr = nullptr;610+ __ubuf__ float* maxToWorkSpaceLocalAddr = nullptr;
611 LocalTensor<float> minToWorkSpaceLocal;611 LocalTensor<float> minToWorkSpaceLocal;
612- __local_mem__ float* minToWorkSpaceLocalAddr = nullptr;612+ __ubuf__ float* minToWorkSpaceLocalAddr = nullptr;
613 LocalTensor<float> scaleToWorkSpaceLocal;613 LocalTensor<float> scaleToWorkSpaceLocal;
614- __local_mem__ float* scaleToWorkSpaceLocalAddr = nullptr;614+ __ubuf__ float* scaleToWorkSpaceLocalAddr = nullptr;
615 615 
616 if constexpr (isSymmetrical == false) {616 if constexpr (isSymmetrical == false) {
617 maxToWorkSpaceLocal = MaxToWorkSpaceQueue.template DeQue<float>();617 maxToWorkSpaceLocal = MaxToWorkSpaceQueue.template DeQue<float>();
618- maxToWorkSpaceLocalAddr = (__local_mem__ float*)maxToWorkSpaceLocal.GetPhyAddr();618+ maxToWorkSpaceLocalAddr = (__ubuf__ float*)maxToWorkSpaceLocal.GetPhyAddr();
619 minToWorkSpaceLocal = MinToWorkSpaceQueue.template DeQue<float>();619 minToWorkSpaceLocal = MinToWorkSpaceQueue.template DeQue<float>();
620- minToWorkSpaceLocalAddr = (__local_mem__ float*)minToWorkSpaceLocal.GetPhyAddr();620+ minToWorkSpaceLocalAddr = (__ubuf__ float*)minToWorkSpaceLocal.GetPhyAddr();
621 } else {621 } else {
622 scaleToWorkSpaceLocal = scaleToWorkSpaceQueue.template DeQue<float>();622 scaleToWorkSpaceLocal = scaleToWorkSpaceQueue.template DeQue<float>();
623- scaleToWorkSpaceLocalAddr = (__local_mem__ float*)scaleToWorkSpaceLocal.GetPhyAddr();623+ scaleToWorkSpaceLocalAddr = (__ubuf__ float*)scaleToWorkSpaceLocal.GetPhyAddr();
624 }624 }
625 625 
626 for (int32_t segStart = 0; segStart < multiRow;) {626 for (int32_t segStart = 0; segStart < multiRow;) {
@@ -628,7 +628,7 @@ private:
628 if (groupValue < realRowStart) {628 if (groupValue < realRowStart) {
629 smoothIndex = GetSmoothIndexLocal(realRowStart, groupValue, smoothIndex + 1);629 smoothIndex = GetSmoothIndexLocal(realRowStart, groupValue, smoothIndex + 1);
630 }630 }
631- __local_mem__ xDtype* smoothRowAddr = GetSmoothRowAddr(smoothIndex);631+ __ubuf__ xDtype* smoothRowAddr = GetSmoothRowAddr(smoothIndex);
632 632 
633 int32_t boundary = groupValue - offsetRow;633 int32_t boundary = groupValue - offsetRow;
634 int32_t segEnd = boundary < multiRow ? boundary : multiRow;634 int32_t segEnd = boundary < multiRow ? boundary : multiRow;
@@ -649,15 +649,14 @@ private:
649 inQueue.FreeTensor(inLocal);649 inQueue.FreeTensor(inLocal);
650 }650 }
651 651 
652- __aicore__ inline void ComputeMaxColScale(uint32_t elementNum, __local_mem__ float* maxAddr,652+ __aicore__ inline void ComputeMaxColScale(uint32_t elementNum, __ubuf__ float* maxAddr, __ubuf__ float* minAddr,
653- __local_mem__ float* minAddr, __local_mem__ float* scaleAddr,653+ __ubuf__ float* scaleAddr, __ubuf__ float* offsetAddr)
654- __local_mem__ float* offsetAddr)
655 {654 {
656 if constexpr (isSymmetrical == false) {655 if constexpr (isSymmetrical == false) {
657 LocalTensor<float> maxFromWorkSpaceLocal = MaxFromWorkSpaceQueue.template DeQue<float>();656 LocalTensor<float> maxFromWorkSpaceLocal = MaxFromWorkSpaceQueue.template DeQue<float>();
658- __local_mem__ float* maxFromWorkSpaceLocalAddr = (__local_mem__ float*)maxFromWorkSpaceLocal.GetPhyAddr();657+ __ubuf__ float* maxFromWorkSpaceLocalAddr = (__ubuf__ float*)maxFromWorkSpaceLocal.GetPhyAddr();
659 LocalTensor<float> minFromWorkSpaceLocal = MinFromWorkSpaceQueue.template DeQue<float>();658 LocalTensor<float> minFromWorkSpaceLocal = MinFromWorkSpaceQueue.template DeQue<float>();
660- __local_mem__ float* minFromWorkSpaceLocalAddr = (__local_mem__ float*)minFromWorkSpaceLocal.GetPhyAddr();659+ __ubuf__ float* minFromWorkSpaceLocalAddr = (__ubuf__ float*)minFromWorkSpaceLocal.GetPhyAddr();
661 ComputeMaxColScaleVF(nullptr, scaleAddr, maxFromWorkSpaceLocalAddr, maxAddr, minFromWorkSpaceLocalAddr,660 ComputeMaxColScaleVF(nullptr, scaleAddr, maxFromWorkSpaceLocalAddr, maxAddr, minFromWorkSpaceLocalAddr,
662 minAddr, elementNum);661 minAddr, elementNum);
663 ComputeScaleSymVF(maxAddr, minAddr, scaleAddr, 1);662 ComputeScaleSymVF(maxAddr, minAddr, scaleAddr, 1);
@@ -666,29 +665,28 @@ private:
666 MinFromWorkSpaceQueue.FreeTensor(minFromWorkSpaceLocal);665 MinFromWorkSpaceQueue.FreeTensor(minFromWorkSpaceLocal);
667 } else {666 } else {
668 LocalTensor<float> scaleFromWorkSpaceLocal = scaleFromWorkSpaceQueue.template DeQue<float>();667 LocalTensor<float> scaleFromWorkSpaceLocal = scaleFromWorkSpaceQueue.template DeQue<float>();
669- __local_mem__ float* scaleFromWorkSpaceLocalAddr = (__local_mem__ float*)668+ __ubuf__ float* scaleFromWorkSpaceLocalAddr = (__ubuf__ float*)scaleFromWorkSpaceLocal.GetPhyAddr();
670- scaleFromWorkSpaceLocal.GetPhyAddr();
671 ComputeMaxColScaleVF(scaleFromWorkSpaceLocalAddr, scaleAddr, nullptr, maxAddr, nullptr, minAddr,669 ComputeMaxColScaleVF(scaleFromWorkSpaceLocalAddr, scaleAddr, nullptr, maxAddr, nullptr, minAddr,
672 elementNum);670 elementNum);
673 scaleFromWorkSpaceQueue.FreeTensor(scaleFromWorkSpaceLocal);671 scaleFromWorkSpaceQueue.FreeTensor(scaleFromWorkSpaceLocal);
674 }672 }
675 }673 }
676 674 
677- __aicore__ inline void ComputeYMoE(int32_t multiRow, int32_t offsetRow, __local_mem__ float* scaleAddr,675+ __aicore__ inline void ComputeYMoE(int32_t multiRow, int32_t offsetRow, __ubuf__ float* scaleAddr,
678- __local_mem__ float* offsetAddr)676+ __ubuf__ float* offsetAddr)
679 {677 {
680 LocalTensor<yCopyDtype> yLocal = outQueue.template AllocTensor<yCopyDtype>();678 LocalTensor<yCopyDtype> yLocal = outQueue.template AllocTensor<yCopyDtype>();
681 LocalTensor<xDtype> xLocal = inQueue.template DeQue<xDtype>();679 LocalTensor<xDtype> xLocal = inQueue.template DeQue<xDtype>();
682 680 
683- __local_mem__ xDtype* xAddr = (__local_mem__ xDtype*)xLocal.GetPhyAddr();681+ __ubuf__ xDtype* xAddr = (__ubuf__ xDtype*)xLocal.GetPhyAddr();
684- __local_mem__ yCopyDtype* yAddr = (__local_mem__ yCopyDtype*)yLocal.GetPhyAddr();682+ __ubuf__ yCopyDtype* yAddr = (__ubuf__ yCopyDtype*)yLocal.GetPhyAddr();
685 683 
686 for (int32_t segStart = 0; segStart < multiRow;) {684 for (int32_t segStart = 0; segStart < multiRow;) {
687 int32_t realRowStart = offsetRow + segStart + 1;685 int32_t realRowStart = offsetRow + segStart + 1;
688 if (groupValue < realRowStart) {686 if (groupValue < realRowStart) {
689 smoothIndex = GetSmoothIndexLocal(realRowStart, groupValue, smoothIndex + 1);687 smoothIndex = GetSmoothIndexLocal(realRowStart, groupValue, smoothIndex + 1);
690 }688 }
691- __local_mem__ xDtype* smoothRowAddr = GetSmoothRowAddr(smoothIndex);689+ __ubuf__ xDtype* smoothRowAddr = GetSmoothRowAddr(smoothIndex);
692 690 
693 int32_t boundary = groupValue - offsetRow;691 int32_t boundary = groupValue - offsetRow;
694 int32_t segEnd = boundary < multiRow ? boundary : multiRow;692 int32_t segEnd = boundary < multiRow ? boundary : multiRow;
@@ -718,9 +716,8 @@ private:
718 outQueue.FreeTensor(yLocal);716 outQueue.FreeTensor(yLocal);
719 }717 }
720 718 
721- __aicore__ inline void ComputeYVF(__local_mem__ xDtype* xAddr, __local_mem__ xDtype* smoothAddr,719+ __aicore__ inline void ComputeYVF(__ubuf__ xDtype* xAddr, __ubuf__ xDtype* smoothAddr, __ubuf__ yCopyDtype* yAddr,
722- __local_mem__ yCopyDtype* yAddr, __local_mem__ float* scaleAddr,720+ __ubuf__ float* scaleAddr, __ubuf__ float* offsetAddr, int32_t multiRow)
723- __local_mem__ float* offsetAddr, int32_t multiRow)
724 {721 {
725 uint32_t dtypeSize = sizeof(float);722 uint32_t dtypeSize = sizeof(float);
726 uint16_t VL = AscendC::VECTOR_REG_WIDTH / dtypeSize;723 uint16_t VL = AscendC::VECTOR_REG_WIDTH / dtypeSize;
@@ -744,9 +741,9 @@ private:
744 AscendC::MicroAPI::MaskReg741 AscendC::MicroAPI::MaskReg
745 preg1 = AscendC::MicroAPI::CreateMask<float, AscendC::MicroAPI::MaskPattern::H>();742 preg1 = AscendC::MicroAPI::CreateMask<float, AscendC::MicroAPI::MaskPattern::H>();
746 743 
747- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_BRC_B32>(vreg_scale, scaleAddr);744+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_BRC_B32>(vreg_scale, scaleAddr);
748 if constexpr (isSymmetrical == false) {745 if constexpr (isSymmetrical == false) {
749- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_BRC_B32>(vreg_offset, offsetAddr);746+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_BRC_B32>(vreg_offset, offsetAddr);
750 }747 }
751 748 
752 for (uint16_t i = 0; i < static_cast<uint16_t>(multiRow); i++) {749 for (uint16_t i = 0; i < static_cast<uint16_t>(multiRow); i++) {
@@ -754,10 +751,10 @@ private:
754 for (uint16_t j = 0; j < vfLoop; j++) {751 for (uint16_t j = 0; j < vfLoop; j++) {
755 auto addr = yAddr + i * outAlignLen + j * VL;752 auto addr = yAddr + i * outAlignLen + j * VL;
756 preg0 = AscendC::MicroAPI::UpdateMask<float>(sreg0);753 preg0 = AscendC::MicroAPI::UpdateMask<float>(sreg0);
757- AscendC::MicroAPI::DataCopy<xDtype, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(754+ AscendC::MicroAPI::LoadAlign<xDtype, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(
758 vreg1, xAddr + i * rowCount + j * VL);755 vreg1, xAddr + i * rowCount + j * VL);
759 AscendC::MicroAPI::Cast<float, xDtype, castTraitB16ToB32>(vreg3, vreg1, preg0);756 AscendC::MicroAPI::Cast<float, xDtype, castTraitB16ToB32>(vreg3, vreg1, preg0);
760- AscendC::MicroAPI::DataCopy<xDtype, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(757+ AscendC::MicroAPI::LoadAlign<xDtype, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(
761 vreg2, smoothAddr + j * VL);758 vreg2, smoothAddr + j * VL);
762 AscendC::MicroAPI::Cast<float, xDtype, castTraitB16ToB32>(vreg4, vreg2, preg0);759 AscendC::MicroAPI::Cast<float, xDtype, castTraitB16ToB32>(vreg4, vreg2, preg0);
763 AscendC::MicroAPI::Mul(vreg3, vreg3, vreg4, preg0);760 AscendC::MicroAPI::Mul(vreg3, vreg3, vreg4, preg0);
@@ -785,10 +782,10 @@ private:
785 addr = yAddr + (i * outAlignLen + j * VL) / 2;782 addr = yAddr + (i * outAlignLen + j * VL) / 2;
786 }783 }
787 if constexpr (IsSameType<yDtype, int4b_t>::value) {784 if constexpr (IsSameType<yDtype, int4b_t>::value) {
788- AscendC::MicroAPI::DataCopy<yCopyDtype, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(785+ AscendC::MicroAPI::StoreAlign<yCopyDtype, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(
789 addr, vreg8, preg1);786 addr, vreg8, preg1);
790 } else {787 } else {
791- AscendC::MicroAPI::DataCopy<yCopyDtype, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(788+ AscendC::MicroAPI::StoreAlign<yCopyDtype, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(
792 addr, vreg8, preg0);789 addr, vreg8, preg0);
793 }790 }
794 }791 }
Mquant/dynamic_quant/op_kernel/arch35/dynamic_quant_regbase_moe_large_shape.h+62-62
@@ -64,19 +64,19 @@ private:
64 __aicore__ inline void ProcessLoop(int64_t offsetBase, int32_t& groupNum);64 __aicore__ inline void ProcessLoop(int64_t offsetBase, int32_t& groupNum);
65 __aicore__ inline void CopyInByEle(int64_t xOffset, int64_t smoothOffset, uint32_t elementNum,65 __aicore__ inline void CopyInByEle(int64_t xOffset, int64_t smoothOffset, uint32_t elementNum,
66 uint8_t rightPadding);66 uint8_t rightPadding);
67- __aicore__ inline void ComputeMaxScale(uint32_t elementNum, __local_mem__ float* scaleLocalAddr,67+ __aicore__ inline void ComputeMaxScale(uint32_t elementNum, __ubuf__ float* scaleLocalAddr,
68- __local_mem__ float* offsetLocalAddr);68+ __ubuf__ float* offsetLocalAddr);
69- __aicore__ inline void ComputeMaxScaleAndYTail(uint32_t elementNum, __local_mem__ float* scaleLocalAddr,69+ __aicore__ inline void ComputeMaxScaleAndYTail(uint32_t elementNum, __ubuf__ float* scaleLocalAddr,
70- __local_mem__ float* offsetLocalAddr);70+ __ubuf__ float* offsetLocalAddr);
71 template <bool isFinal = false>71 template <bool isFinal = false>
72- __aicore__ inline void ComputeMaxScaleVF(__local_mem__ T* inLocalAddr, __local_mem__ T* smoothLocalAddr,72+ __aicore__ inline void ComputeMaxScaleVF(__ubuf__ T* inLocalAddr, __ubuf__ T* smoothLocalAddr,
73- __local_mem__ float* maxLocalAddr, __local_mem__ float* minLocalAddr,73+ __ubuf__ float* maxLocalAddr, __ubuf__ float* minLocalAddr,
74 uint32_t elementNum);74 uint32_t elementNum);
75- __aicore__ inline void ComputeY(uint32_t elementNum, __local_mem__ float* scaleLocalAddr,75+ __aicore__ inline void ComputeY(uint32_t elementNum, __ubuf__ float* scaleLocalAddr,
76- __local_mem__ float* offsetLocalAddr);76+ __ubuf__ float* offsetLocalAddr);
77- __aicore__ inline void ComputeYVF(__local_mem__ T* inLocalAddr, __local_mem__ T* smoothLocalAddr,77+ __aicore__ inline void ComputeYVF(__ubuf__ T* inLocalAddr, __ubuf__ T* smoothLocalAddr,
78- __local_mem__ yCopyDtype* outAddr, __local_mem__ float* scaleLocalAddr,78+ __ubuf__ yCopyDtype* outAddr, __ubuf__ float* scaleLocalAddr,
79- __local_mem__ float* offsetLocalAddr, uint32_t elementNum);79+ __ubuf__ float* offsetLocalAddr, uint32_t elementNum);
80 __aicore__ inline void ParseTilingData(const DynamicQuantTilingDataArch35& tilingData);80 __aicore__ inline void ParseTilingData(const DynamicQuantTilingDataArch35& tilingData);
81 __aicore__ inline void CopyOutScale(int64_t offset);81 __aicore__ inline void CopyOutScale(int64_t offset);
82 __aicore__ inline void CopyOutY(int64_t offset, uint32_t element);82 __aicore__ inline void CopyOutY(int64_t offset, uint32_t element);
@@ -305,16 +305,16 @@ __aicore__ inline void DynamicQuantLargeShapeMOE<T, yDtype, hasSmooth, isSymmetr
305 int32_t& groupNum)305 int32_t& groupNum)
306{306{
307 LocalTensor<float> scaleLocal = scaleQueue.AllocTensor<float>();307 LocalTensor<float> scaleLocal = scaleQueue.AllocTensor<float>();
308- __local_mem__ float* scaleLocalAddr = (__local_mem__ float*)scaleLocal.GetPhyAddr();308+ __ubuf__ float* scaleLocalAddr = (__ubuf__ float*)scaleLocal.GetPhyAddr();
309 309 
310 LocalTensor<float> offsetLocal;310 LocalTensor<float> offsetLocal;
311- __local_mem__ float* offsetLocalAddr;311+ __ubuf__ float* offsetLocalAddr;
312 312 
313 if constexpr (isSymmetrical) {313 if constexpr (isSymmetrical) {
314 AscendC::Duplicate(scaleLocal, (float)0.0, 64, 1, 1, 8);314 AscendC::Duplicate(scaleLocal, (float)0.0, 64, 1, 1, 8);
315 } else {315 } else {
316 offsetLocal = offsetQueue.AllocTensor<float>();316 offsetLocal = offsetQueue.AllocTensor<float>();
317- offsetLocalAddr = (__local_mem__ float*)offsetLocal.GetPhyAddr();317+ offsetLocalAddr = (__ubuf__ float*)offsetLocal.GetPhyAddr();
318 AscendC::Duplicate(scaleLocal, MIN_FLOAT_VALUE, 64, 1, 1, 8);318 AscendC::Duplicate(scaleLocal, MIN_FLOAT_VALUE, 64, 1, 1, 8);
319 AscendC::Duplicate(offsetLocal, MAX_FLOAT_VALUE, 64, 1, 1, 8);319 AscendC::Duplicate(offsetLocal, MAX_FLOAT_VALUE, 64, 1, 1, 8);
320 }320 }
@@ -380,16 +380,16 @@ __aicore__ inline void DynamicQuantLargeShapeMOE<T, yDtype, hasSmooth, isSymmetr
380 380 
381template <typename T, typename yDtype, int64_t hasSmooth, bool isSymmetrical>381template <typename T, typename yDtype, int64_t hasSmooth, bool isSymmetrical>
382__aicore__ inline void DynamicQuantLargeShapeMOE<T, yDtype, hasSmooth, isSymmetrical>::ComputeMaxScale(382__aicore__ inline void DynamicQuantLargeShapeMOE<T, yDtype, hasSmooth, isSymmetrical>::ComputeMaxScale(
383- uint32_t elementNum, __local_mem__ float* scaleLocalAddr, __local_mem__ float* offsetLocalAddr)383+ uint32_t elementNum, __ubuf__ float* scaleLocalAddr, __ubuf__ float* offsetLocalAddr)
384{384{
385 LocalTensor<T> inLocal = inQueue.DeQue<T>();385 LocalTensor<T> inLocal = inQueue.DeQue<T>();
386- __local_mem__ T* inLocalAddr = (__local_mem__ T*)inLocal.GetPhyAddr();386+ __ubuf__ T* inLocalAddr = (__ubuf__ T*)inLocal.GetPhyAddr();
387 LocalTensor<T> smoothLocal;387 LocalTensor<T> smoothLocal;
388- __local_mem__ T* smoothLocalAddr;388+ __ubuf__ T* smoothLocalAddr;
389 389 
390 if constexpr (hasSmooth == 1) {390 if constexpr (hasSmooth == 1) {
391 smoothLocal = smoothQueue.DeQue<T>();391 smoothLocal = smoothQueue.DeQue<T>();
392- smoothLocalAddr = (__local_mem__ T*)smoothLocal.GetPhyAddr();392+ smoothLocalAddr = (__ubuf__ T*)smoothLocal.GetPhyAddr();
393 }393 }
394 394 
395 ComputeMaxScaleVF(inLocalAddr, smoothLocalAddr, scaleLocalAddr, offsetLocalAddr, elementNum);395 ComputeMaxScaleVF(inLocalAddr, smoothLocalAddr, scaleLocalAddr, offsetLocalAddr, elementNum);
@@ -403,18 +403,18 @@ __aicore__ inline void DynamicQuantLargeShapeMOE<T, yDtype, hasSmooth, isSymmetr
403 403 
404template <typename T, typename yDtype, int64_t hasSmooth, bool isSymmetrical>404template <typename T, typename yDtype, int64_t hasSmooth, bool isSymmetrical>
405__aicore__ inline void DynamicQuantLargeShapeMOE<T, yDtype, hasSmooth, isSymmetrical>::ComputeMaxScaleAndYTail(405__aicore__ inline void DynamicQuantLargeShapeMOE<T, yDtype, hasSmooth, isSymmetrical>::ComputeMaxScaleAndYTail(
406- uint32_t elementNum, __local_mem__ float* scaleLocalAddr, __local_mem__ float* offsetLocalAddr)406+ uint32_t elementNum, __ubuf__ float* scaleLocalAddr, __ubuf__ float* offsetLocalAddr)
407{407{
408 LocalTensor<T> inLocal = inQueue.DeQue<T>();408 LocalTensor<T> inLocal = inQueue.DeQue<T>();
409- __local_mem__ T* inLocalAddr = (__local_mem__ T*)inLocal.GetPhyAddr();409+ __ubuf__ T* inLocalAddr = (__ubuf__ T*)inLocal.GetPhyAddr();
410 LocalTensor<yCopyDtype> outLocal = outQueue.AllocTensor<yCopyDtype>();410 LocalTensor<yCopyDtype> outLocal = outQueue.AllocTensor<yCopyDtype>();
411- __local_mem__ yCopyDtype* outAddr = (__local_mem__ yCopyDtype*)outLocal.GetPhyAddr();411+ __ubuf__ yCopyDtype* outAddr = (__ubuf__ yCopyDtype*)outLocal.GetPhyAddr();
412 LocalTensor<T> smoothLocal;412 LocalTensor<T> smoothLocal;
413- __local_mem__ T* smoothLocalAddr;413+ __ubuf__ T* smoothLocalAddr;
414 414 
415 if constexpr (hasSmooth == 1) {415 if constexpr (hasSmooth == 1) {
416 smoothLocal = smoothQueue.DeQue<T>();416 smoothLocal = smoothQueue.DeQue<T>();
417- smoothLocalAddr = (__local_mem__ T*)smoothLocal.GetPhyAddr();417+ smoothLocalAddr = (__ubuf__ T*)smoothLocal.GetPhyAddr();
418 }418 }
419 419 
420 ComputeMaxScaleVF<true>(inLocalAddr, smoothLocalAddr, scaleLocalAddr, offsetLocalAddr, elementNum);420 ComputeMaxScaleVF<true>(inLocalAddr, smoothLocalAddr, scaleLocalAddr, offsetLocalAddr, elementNum);
@@ -431,8 +431,8 @@ __aicore__ inline void DynamicQuantLargeShapeMOE<T, yDtype, hasSmooth, isSymmetr
431template <typename T, typename yDtype, int64_t hasSmooth, bool isSymmetrical>431template <typename T, typename yDtype, int64_t hasSmooth, bool isSymmetrical>
432template <bool isFinal>432template <bool isFinal>
433__aicore__ inline void DynamicQuantLargeShapeMOE<T, yDtype, hasSmooth, isSymmetrical>::ComputeMaxScaleVF(433__aicore__ inline void DynamicQuantLargeShapeMOE<T, yDtype, hasSmooth, isSymmetrical>::ComputeMaxScaleVF(
434- __local_mem__ T* inLocalAddr, __local_mem__ T* smoothLocalAddr, __local_mem__ float* maxLocalAddr,434+ __ubuf__ T* inLocalAddr, __ubuf__ T* smoothLocalAddr, __ubuf__ float* maxLocalAddr, __ubuf__ float* minLocalAddr,
435- __local_mem__ float* minLocalAddr, uint32_t elementNum)435+ uint32_t elementNum)
436{436{
437 uint32_t dtypeSize = sizeof(float);437 uint32_t dtypeSize = sizeof(float);
438 uint32_t VL = AscendC::VECTOR_REG_WIDTH / dtypeSize;438 uint32_t VL = AscendC::VECTOR_REG_WIDTH / dtypeSize;
@@ -455,9 +455,9 @@ __aicore__ inline void DynamicQuantLargeShapeMOE<T, yDtype, hasSmooth, isSymmetr
455 AscendC::MicroAPI::RegTensor<float, MicroAPI::RegTraitNumOne> vregMinTail;455 AscendC::MicroAPI::RegTensor<float, MicroAPI::RegTraitNumOne> vregMinTail;
456 static constexpr AscendC::MicroAPI::DivSpecificMode mode = {AscendC::MicroAPI::MaskMergeMode::ZEROING, true};456 static constexpr AscendC::MicroAPI::DivSpecificMode mode = {AscendC::MicroAPI::MaskMergeMode::ZEROING, true};
457 457 
458- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_BRC_B32>(vreg6, maxLocalAddr);458+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_BRC_B32>(vreg6, maxLocalAddr);
459 if constexpr (!isSymmetrical) {459 if constexpr (!isSymmetrical) {
460- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_BRC_B32>(vreg8, minLocalAddr);460+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_BRC_B32>(vreg8, minLocalAddr);
461 }461 }
462 462 
463 AscendC::MicroAPI::MaskReg mask;463 AscendC::MicroAPI::MaskReg mask;
@@ -465,10 +465,10 @@ __aicore__ inline void DynamicQuantLargeShapeMOE<T, yDtype, hasSmooth, isSymmetr
465 maskAll = AscendC::MicroAPI::CreateMask<float, AscendC::MicroAPI::MaskPattern::ALL>();465 maskAll = AscendC::MicroAPI::CreateMask<float, AscendC::MicroAPI::MaskPattern::ALL>();
466 466 
467 for (uint16_t i = 0; i < static_cast<uint16_t>(vfLoopNum - 1); i++) {467 for (uint16_t i = 0; i < static_cast<uint16_t>(vfLoopNum - 1); i++) {
468- AscendC::MicroAPI::DataCopy<T, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(vreg0, inLocalAddr + i * VL);468+ AscendC::MicroAPI::LoadAlign<T, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(vreg0, inLocalAddr + i * VL);
469 AscendC::MicroAPI::Cast<float, T, castTrait0>(vreg1, vreg0, maskAll);469 AscendC::MicroAPI::Cast<float, T, castTrait0>(vreg1, vreg0, maskAll);
470- AscendC::MicroAPI::DataCopy<T, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(vreg2,470+ AscendC::MicroAPI::LoadAlign<T, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(vreg2,
471- smoothLocalAddr + i * VL);471+ smoothLocalAddr + i * VL);
472 AscendC::MicroAPI::Cast<float, T, castTrait0>(vreg3, vreg2, maskAll);472 AscendC::MicroAPI::Cast<float, T, castTrait0>(vreg3, vreg2, maskAll);
473 AscendC::MicroAPI::Mul(vreg1, vreg1, vreg3, maskAll);473 AscendC::MicroAPI::Mul(vreg1, vreg1, vreg3, maskAll);
474 if constexpr (isSymmetrical) {474 if constexpr (isSymmetrical) {
@@ -482,16 +482,16 @@ __aicore__ inline void DynamicQuantLargeShapeMOE<T, yDtype, hasSmooth, isSymmetr
482 }482 }
483 {483 {
484 if constexpr (isSymmetrical) {484 if constexpr (isSymmetrical) {
485- AscendC::MicroAPI::ReduceMax<float>(vregMax, vreg6, maskAll);485+ AscendC::MicroAPI::Reduce<MicroAPI::ReduceType::MAX, float>(vregMax, vreg6, maskAll);
486 } else {486 } else {
487- AscendC::MicroAPI::ReduceMax<float>(vregMax, vreg6, maskAll);487+ AscendC::MicroAPI::Reduce<MicroAPI::ReduceType::MAX, float>(vregMax, vreg6, maskAll);
488- AscendC::MicroAPI::ReduceMin<float>(vregMin, vreg8, maskAll);488+ AscendC::MicroAPI::Reduce<MicroAPI::ReduceType::MIN, float>(vregMin, vreg8, maskAll);
489 }489 }
490 mask = AscendC::MicroAPI::UpdateMask<float>(tailNum);490 mask = AscendC::MicroAPI::UpdateMask<float>(tailNum);
491- AscendC::MicroAPI::DataCopy<T, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(491+ AscendC::MicroAPI::LoadAlign<T, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(
492 vreg0, inLocalAddr + (vfLoopNum - 1) * VL);492 vreg0, inLocalAddr + (vfLoopNum - 1) * VL);
493 AscendC::MicroAPI::Cast<float, T, castTrait0>(vreg1, vreg0, mask);493 AscendC::MicroAPI::Cast<float, T, castTrait0>(vreg1, vreg0, mask);
494- AscendC::MicroAPI::DataCopy<T, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(494+ AscendC::MicroAPI::LoadAlign<T, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(
495 vreg2, smoothLocalAddr + (vfLoopNum - 1) * VL);495 vreg2, smoothLocalAddr + (vfLoopNum - 1) * VL);
496 AscendC::MicroAPI::Cast<float, T, castTrait0>(vreg3, vreg2, mask);496 AscendC::MicroAPI::Cast<float, T, castTrait0>(vreg3, vreg2, mask);
497 AscendC::MicroAPI::Mul(vreg1, vreg1, vreg3, mask);497 AscendC::MicroAPI::Mul(vreg1, vreg1, vreg3, mask);
@@ -499,21 +499,21 @@ __aicore__ inline void DynamicQuantLargeShapeMOE<T, yDtype, hasSmooth, isSymmetr
499 AscendC::MicroAPI::Abs(vreg5, vreg1, mask);499 AscendC::MicroAPI::Abs(vreg5, vreg1, mask);
500 AscendC::MicroAPI::Muls(vreg5, vreg5, scaleMaxValue, mask);500 AscendC::MicroAPI::Muls(vreg5, vreg5, scaleMaxValue, mask);
501 AscendC::MicroAPI::Max(vreg6, vreg5, vreg6, mask);501 AscendC::MicroAPI::Max(vreg6, vreg5, vreg6, mask);
502- AscendC::MicroAPI::ReduceMax<float>(vregMaxTail, vreg6, mask);502+ AscendC::MicroAPI::Reduce<MicroAPI::ReduceType::MAX, float>(vregMaxTail, vreg6, mask);
503 AscendC::MicroAPI::Max(vregMax, vregMax, vregMaxTail, mask);503 AscendC::MicroAPI::Max(vregMax, vregMax, vregMaxTail, mask);
504 } else {504 } else {
505 AscendC::MicroAPI::Max(vreg6, vreg1, vreg6, mask);505 AscendC::MicroAPI::Max(vreg6, vreg1, vreg6, mask);
506 AscendC::MicroAPI::Min(vreg8, vreg1, vreg8, mask);506 AscendC::MicroAPI::Min(vreg8, vreg1, vreg8, mask);
507- AscendC::MicroAPI::ReduceMax<float>(vregMaxTail, vreg6, mask);507+ AscendC::MicroAPI::Reduce<MicroAPI::ReduceType::MAX, float>(vregMaxTail, vreg6, mask);
508- AscendC::MicroAPI::ReduceMin<float>(vregMinTail, vreg8, mask);508+ AscendC::MicroAPI::Reduce<MicroAPI::ReduceType::MIN, float>(vregMinTail, vreg8, mask);
509 AscendC::MicroAPI::Max(vregMax, vregMax, vregMaxTail, mask);509 AscendC::MicroAPI::Max(vregMax, vregMax, vregMaxTail, mask);
510 AscendC::MicroAPI::Min(vregMin, vregMin, vregMinTail, mask);510 AscendC::MicroAPI::Min(vregMin, vregMin, vregMinTail, mask);
511 }511 }
512 }512 }
513 513 
514 if constexpr (isSymmetrical) {514 if constexpr (isSymmetrical) {
515- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::StoreDist::DIST_FIRST_ELEMENT_B32>(maxLocalAddr,515+ AscendC::MicroAPI::StoreAlign<float, AscendC::MicroAPI::StoreDist::DIST_FIRST_ELEMENT_B32>(
516- vregMax, maskAll);516+ maxLocalAddr, vregMax, maskAll);
517 } else {517 } else {
518 if constexpr (isFinal) {518 if constexpr (isFinal) {
519 AscendC::MicroAPI::RegTensor<float, MicroAPI::RegTraitNumOne> vreg9;519 AscendC::MicroAPI::RegTensor<float, MicroAPI::RegTraitNumOne> vreg9;
@@ -528,14 +528,14 @@ __aicore__ inline void DynamicQuantLargeShapeMOE<T, yDtype, hasSmooth, isSymmetr
528 AscendC::MicroAPI::Muls(vreg12, vreg11, -1, maskAll);528 AscendC::MicroAPI::Muls(vreg12, vreg11, -1, maskAll);
529 AscendC::MicroAPI::Adds(vreg13, vreg12, offsetMaxValue, maskAll); // offset529 AscendC::MicroAPI::Adds(vreg13, vreg12, offsetMaxValue, maskAll); // offset
530 530 
531- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::StoreDist::DIST_FIRST_ELEMENT_B32>(531+ AscendC::MicroAPI::StoreAlign<float, AscendC::MicroAPI::StoreDist::DIST_FIRST_ELEMENT_B32>(
532 maxLocalAddr, vreg10, maskAll);532 maxLocalAddr, vreg10, maskAll);
533- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::StoreDist::DIST_FIRST_ELEMENT_B32>(533+ AscendC::MicroAPI::StoreAlign<float, AscendC::MicroAPI::StoreDist::DIST_FIRST_ELEMENT_B32>(
534 minLocalAddr, vreg13, maskAll);534 minLocalAddr, vreg13, maskAll);
535 } else {535 } else {
536- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::StoreDist::DIST_FIRST_ELEMENT_B32>(536+ AscendC::MicroAPI::StoreAlign<float, AscendC::MicroAPI::StoreDist::DIST_FIRST_ELEMENT_B32>(
537 maxLocalAddr, vregMax, maskAll);537 maxLocalAddr, vregMax, maskAll);
538- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::StoreDist::DIST_FIRST_ELEMENT_B32>(538+ AscendC::MicroAPI::StoreAlign<float, AscendC::MicroAPI::StoreDist::DIST_FIRST_ELEMENT_B32>(
539 minLocalAddr, vregMin, maskAll);539 minLocalAddr, vregMin, maskAll);
540 }540 }
541 }541 }
@@ -544,18 +544,18 @@ __aicore__ inline void DynamicQuantLargeShapeMOE<T, yDtype, hasSmooth, isSymmetr
544 544 
545template <typename T, typename yDtype, int64_t hasSmooth, bool isSymmetrical>545template <typename T, typename yDtype, int64_t hasSmooth, bool isSymmetrical>
546__aicore__ inline void DynamicQuantLargeShapeMOE<T, yDtype, hasSmooth, isSymmetrical>::ComputeY(546__aicore__ inline void DynamicQuantLargeShapeMOE<T, yDtype, hasSmooth, isSymmetrical>::ComputeY(
547- uint32_t elementNum, __local_mem__ float* scaleLocalAddr, __local_mem__ float* offsetLocalAddr)547+ uint32_t elementNum, __ubuf__ float* scaleLocalAddr, __ubuf__ float* offsetLocalAddr)
548{548{
549 LocalTensor<T> inLocal = inQueue.DeQue<T>();549 LocalTensor<T> inLocal = inQueue.DeQue<T>();
550- __local_mem__ T* inLocalAddr = (__local_mem__ T*)inLocal.GetPhyAddr();550+ __ubuf__ T* inLocalAddr = (__ubuf__ T*)inLocal.GetPhyAddr();
551 LocalTensor<yCopyDtype> outLocal = outQueue.AllocTensor<yCopyDtype>();551 LocalTensor<yCopyDtype> outLocal = outQueue.AllocTensor<yCopyDtype>();
552- __local_mem__ yCopyDtype* outAddr = (__local_mem__ yCopyDtype*)outLocal.GetPhyAddr();552+ __ubuf__ yCopyDtype* outAddr = (__ubuf__ yCopyDtype*)outLocal.GetPhyAddr();
553 LocalTensor<T> smoothLocal;553 LocalTensor<T> smoothLocal;
554- __local_mem__ T* smoothLocalAddr;554+ __ubuf__ T* smoothLocalAddr;
555 555 
556 if constexpr (hasSmooth == 1) {556 if constexpr (hasSmooth == 1) {
557 smoothLocal = smoothQueue.DeQue<T>();557 smoothLocal = smoothQueue.DeQue<T>();
558- smoothLocalAddr = (__local_mem__ T*)smoothLocal.GetPhyAddr();558+ smoothLocalAddr = (__ubuf__ T*)smoothLocal.GetPhyAddr();
559 }559 }
560 560 
561 ComputeYVF(inLocalAddr, smoothLocalAddr, outAddr, scaleLocalAddr, offsetLocalAddr, elementNum);561 ComputeYVF(inLocalAddr, smoothLocalAddr, outAddr, scaleLocalAddr, offsetLocalAddr, elementNum);
@@ -570,8 +570,8 @@ __aicore__ inline void DynamicQuantLargeShapeMOE<T, yDtype, hasSmooth, isSymmetr
570 570 
571template <typename T, typename yDtype, int64_t hasSmooth, bool isSymmetrical>571template <typename T, typename yDtype, int64_t hasSmooth, bool isSymmetrical>
572__aicore__ inline void DynamicQuantLargeShapeMOE<T, yDtype, hasSmooth, isSymmetrical>::ComputeYVF(572__aicore__ inline void DynamicQuantLargeShapeMOE<T, yDtype, hasSmooth, isSymmetrical>::ComputeYVF(
573- __local_mem__ T* inLocalAddr, __local_mem__ T* smoothLocalAddr, __local_mem__ yCopyDtype* outAddr,573+ __ubuf__ T* inLocalAddr, __ubuf__ T* smoothLocalAddr, __ubuf__ yCopyDtype* outAddr, __ubuf__ float* scaleLocalAddr,
574- __local_mem__ float* scaleLocalAddr, __local_mem__ float* offsetLocalAddr, uint32_t elementNum)574+ __ubuf__ float* offsetLocalAddr, uint32_t elementNum)
575{575{
576 uint32_t dtypeSize = sizeof(float);576 uint32_t dtypeSize = sizeof(float);
577 uint32_t VL = AscendC::VECTOR_REG_WIDTH / dtypeSize;577 uint32_t VL = AscendC::VECTOR_REG_WIDTH / dtypeSize;
@@ -590,18 +590,18 @@ __aicore__ inline void DynamicQuantLargeShapeMOE<T, yDtype, hasSmooth, isSymmetr
590 AscendC::MicroAPI::RegTensor<yCopyDtype, MicroAPI::RegTraitNumOne> vreg8;590 AscendC::MicroAPI::RegTensor<yCopyDtype, MicroAPI::RegTraitNumOne> vreg8;
591 AscendC::MicroAPI::RegTensor<float, MicroAPI::RegTraitNumOne> vreg9;591 AscendC::MicroAPI::RegTensor<float, MicroAPI::RegTraitNumOne> vreg9;
592 592 
593- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_BRC_B32>(vreg9, scaleLocalAddr);593+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_BRC_B32>(vreg9, scaleLocalAddr);
594 594 
595 AscendC::MicroAPI::MaskReg mask = AscendC::MicroAPI::CreateMask<float, AscendC::MicroAPI::MaskPattern::ALL>();595 AscendC::MicroAPI::MaskReg mask = AscendC::MicroAPI::CreateMask<float, AscendC::MicroAPI::MaskPattern::ALL>();
596 AscendC::MicroAPI::MaskReg mask2 = AscendC::MicroAPI::CreateMask<float, AscendC::MicroAPI::MaskPattern::H>();596 AscendC::MicroAPI::MaskReg mask2 = AscendC::MicroAPI::CreateMask<float, AscendC::MicroAPI::MaskPattern::H>();
597 for (uint16_t i = 0; i < vfLoopNum; i++) {597 for (uint16_t i = 0; i < vfLoopNum; i++) {
598 auto addr = outAddr + i * VL;598 auto addr = outAddr + i * VL;
599- AscendC::MicroAPI::DataCopy<T, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(vreg0, inLocalAddr + i * VL);599+ AscendC::MicroAPI::LoadAlign<T, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(vreg0, inLocalAddr + i * VL);
600 AscendC::MicroAPI::Cast<float, T, castTrait0>(vreg1, vreg0, mask);600 AscendC::MicroAPI::Cast<float, T, castTrait0>(vreg1, vreg0, mask);
601 601 
602 if constexpr (hasSmooth == 1) {602 if constexpr (hasSmooth == 1) {
603- AscendC::MicroAPI::DataCopy<T, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(vreg2,603+ AscendC::MicroAPI::LoadAlign<T, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(vreg2,
604- smoothLocalAddr + i * VL);604+ smoothLocalAddr + i * VL);
605 AscendC::MicroAPI::Cast<float, T, castTrait0>(vreg3, vreg2, mask);605 AscendC::MicroAPI::Cast<float, T, castTrait0>(vreg3, vreg2, mask);
606 AscendC::MicroAPI::Mul(vreg4, vreg1, vreg3, mask);606 AscendC::MicroAPI::Mul(vreg4, vreg1, vreg3, mask);
607 AscendC::MicroAPI::Div(vreg5, vreg4, vreg9, mask);607 AscendC::MicroAPI::Div(vreg5, vreg4, vreg9, mask);
@@ -611,8 +611,8 @@ __aicore__ inline void DynamicQuantLargeShapeMOE<T, yDtype, hasSmooth, isSymmetr
611 611 
612 if constexpr (!isSymmetrical) {612 if constexpr (!isSymmetrical) {
613 AscendC::MicroAPI::RegTensor<float, MicroAPI::RegTraitNumOne> vregOffset;613 AscendC::MicroAPI::RegTensor<float, MicroAPI::RegTraitNumOne> vregOffset;
614- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_BRC_B32>(vregOffset,614+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_BRC_B32>(vregOffset,
615- offsetLocalAddr);615+ offsetLocalAddr);
616 AscendC::MicroAPI::Add(vreg5, vreg5, vregOffset, mask);616 AscendC::MicroAPI::Add(vreg5, vreg5, vregOffset, mask);
617 }617 }
618 618 
@@ -635,11 +635,11 @@ __aicore__ inline void DynamicQuantLargeShapeMOE<T, yDtype, hasSmooth, isSymmetr
635 }635 }
636 636 
637 if constexpr (IsSameType<yDtype, int4b_t>::value) {637 if constexpr (IsSameType<yDtype, int4b_t>::value) {
638- AscendC::MicroAPI::DataCopy<yCopyDtype, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(addr, vreg8,638+ AscendC::MicroAPI::StoreAlign<yCopyDtype, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(addr, vreg8,
639- mask2);639+ mask2);
640 } else {640 } else {
641- AscendC::MicroAPI::DataCopy<yCopyDtype, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(addr, vreg8,641+ AscendC::MicroAPI::StoreAlign<yCopyDtype, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(addr, vreg8,
642- mask);642+ mask);
643 }643 }
644 }644 }
645 }645 }
@@ -690,4 +690,4 @@ __aicore__ inline uint32_t DynamicQuantLargeShapeMOE<T, yDtype, hasSmooth, isSym
690}690}
691 691 
692} // namespace DynamicQuantRegBase692} // namespace DynamicQuantRegBase
693-#endif // DYNAMIC_QUANT_REGBASE_MOE_LARGE_SHAPE_H693+#endif // DYNAMIC_QUANT_REGBASE_MOE_LARGE_SHAPE_H
Mquant/dynamic_quant/op_kernel/arch35/dynamic_quant_regbase_moe_large_shape_pertensor.h+122-128
@@ -61,33 +61,29 @@ private:
61 __aicore__ inline void ProcessScaleRowLoop(uint32_t i, uint32_t j);61 __aicore__ inline void ProcessScaleRowLoop(uint32_t i, uint32_t j);
62 __aicore__ inline void ProcessScaleRow();62 __aicore__ inline void ProcessScaleRow();
63 __aicore__ inline void ProcessScaleCol();63 __aicore__ inline void ProcessScaleCol();
64- __aicore__ inline void ProcessYRow(uint32_t i, uint32_t j, __local_mem__ float* scaleAddr,64+ __aicore__ inline void ProcessYRow(uint32_t i, uint32_t j, __ubuf__ float* scaleAddr, __ubuf__ float* offsetAddr);
65- __local_mem__ float* offsetAddr);
66 __aicore__ inline void CopyInByEle(int64_t offset, uint32_t loopIndex, uint32_t elementNum, uint8_t rightPad);65 __aicore__ inline void CopyInByEle(int64_t offset, uint32_t loopIndex, uint32_t elementNum, uint8_t rightPad);
67 __aicore__ inline void CopyInSmoothByEle(uint32_t smoothBaseOffset, uint32_t loopIndex, uint32_t elementNum,66 __aicore__ inline void CopyInSmoothByEle(uint32_t smoothBaseOffset, uint32_t loopIndex, uint32_t elementNum,
68 uint8_t rightPad);67 uint8_t rightPad);
69 __aicore__ inline void CopyInScaleByEle(int64_t offset, uint32_t elementNum);68 __aicore__ inline void CopyInScaleByEle(int64_t offset, uint32_t elementNum);
70 __aicore__ inline void ComputeMaxRowScale(uint32_t elementNum);69 __aicore__ inline void ComputeMaxRowScale(uint32_t elementNum);
71- __aicore__ inline void ComputeMaxColScale(uint32_t elementNum, __local_mem__ float* maxAddr,70+ __aicore__ inline void ComputeMaxColScale(uint32_t elementNum, __ubuf__ float* maxAddr, __ubuf__ float* minAddr,
72- __local_mem__ float* minAddr, __local_mem__ float* scaleAddr,71+ __ubuf__ float* scaleAddr, __ubuf__ float* offsetAddr);
73- __local_mem__ float* offsetAddr);72+ __aicore__ inline void ComputeMaxRowScaleVF(__ubuf__ T* inLocalAddr, __ubuf__ T* smoothLocalAddr,
74- __aicore__ inline void ComputeMaxRowScaleVF(__local_mem__ T* inLocalAddr, __local_mem__ T* smoothLocalAddr,73+ __ubuf__ float* scaleLocalAddr, __ubuf__ float* maxLocalAddr,
75- __local_mem__ float* scaleLocalAddr, __local_mem__ float* maxLocalAddr,74+ __ubuf__ float* minLocalAddr, uint32_t elementNum);
76- __local_mem__ float* minLocalAddr, uint32_t elementNum);75+ __aicore__ inline void ComputeMaxColScaleVF(__ubuf__ float* scaleLocalAddr, __ubuf__ float* scaleOutLocalAddr,
77- __aicore__ inline void ComputeMaxColScaleVF(__local_mem__ float* scaleLocalAddr,76+ __ubuf__ float* maxLocalAddr, __ubuf__ float* maxOutLocalAddr,
78- __local_mem__ float* scaleOutLocalAddr,77+ __ubuf__ float* minLocalAddr, __ubuf__ float* minOutLocalAddr,
79- __local_mem__ float* maxLocalAddr, __local_mem__ float* maxOutLocalAddr,
80- __local_mem__ float* minLocalAddr, __local_mem__ float* minOutLocalAddr,
81 uint32_t elementNum);78 uint32_t elementNum);
82- __aicore__ inline void ComputeY(uint32_t elementNum, __local_mem__ float* scaleAddr,79+ __aicore__ inline void ComputeY(uint32_t elementNum, __ubuf__ float* scaleAddr, __ubuf__ float* offsetAddr);
83- __local_mem__ float* offsetAddr);80+ __aicore__ inline void ComputeScaleSymVF(__ubuf__ float* maxLocalAddr, __ubuf__ float* minLocalAddr,
84- __aicore__ inline void ComputeScaleSymVF(__local_mem__ float* maxLocalAddr, __local_mem__ float* minLocalAddr,81+ __ubuf__ float* scaleLocalAddr, uint32_t elementNum);
85- __local_mem__ float* scaleLocalAddr, uint32_t elementNum);82+ __aicore__ inline void ComputeOffsetSymVF(__ubuf__ float* maxLocalAddr, __ubuf__ float* scaleLocalAddr,
86- __aicore__ inline void ComputeOffsetSymVF(__local_mem__ float* maxLocalAddr, __local_mem__ float* scaleLocalAddr,83+ __ubuf__ float* offsetLocalAddr, uint32_t elementNum);
87- __local_mem__ float* offsetLocalAddr, uint32_t elementNum);84+ __aicore__ inline void ComputeYVF(__ubuf__ T* inLocalAddr, __ubuf__ T* smoothLocalAddr,
88- __aicore__ inline void ComputeYVF(__local_mem__ T* inLocalAddr, __local_mem__ T* smoothLocalAddr,85+ __ubuf__ yCopyDtype* outAddr, __ubuf__ float* scaleLocalAddr,
89- __local_mem__ yCopyDtype* outAddr, __local_mem__ float* scaleLocalAddr,86+ __ubuf__ float* offsetLocalAddr, uint32_t elementNum);
90- __local_mem__ float* offsetLocalAddr, uint32_t elementNum);
91 __aicore__ inline void ParseTilingData(const DynamicQuantTilingDataArch35& tilingData);87 __aicore__ inline void ParseTilingData(const DynamicQuantTilingDataArch35& tilingData);
92 __aicore__ inline void CopyOutY(int64_t offset, uint32_t element);88 __aicore__ inline void CopyOutY(int64_t offset, uint32_t element);
93 __aicore__ inline void CopyUB2Workspace(int64_t size);89 __aicore__ inline void CopyUB2Workspace(int64_t size);
@@ -388,28 +384,28 @@ __aicore__ inline void DynamicQuantMoeLargeShapePertensor<T, yDtype, hasSmooth,
388 LocalTensor<float> MinOutLocal;384 LocalTensor<float> MinOutLocal;
389 LocalTensor<float> scaleOutLocal;385 LocalTensor<float> scaleOutLocal;
390 LocalTensor<float> offsetLocal;386 LocalTensor<float> offsetLocal;
391- __local_mem__ float* MaxOutLocalAddr;387+ __ubuf__ float* MaxOutLocalAddr;
392- __local_mem__ float* MinOutLocalAddr;388+ __ubuf__ float* MinOutLocalAddr;
393- __local_mem__ float* scaleOutLocalAddr;389+ __ubuf__ float* scaleOutLocalAddr;
394- __local_mem__ float* offsetLocalAddr;390+ __ubuf__ float* offsetLocalAddr;
395 391 
396 if constexpr (isSymmetrical == false) {392 if constexpr (isSymmetrical == false) {
397 MaxOutLocal = MaxOutQueue.AllocTensor<float>();393 MaxOutLocal = MaxOutQueue.AllocTensor<float>();
398 AscendC::Duplicate(MaxOutLocal, MIN_FLOAT_VALUE, 64, 1, 1, 8);394 AscendC::Duplicate(MaxOutLocal, MIN_FLOAT_VALUE, 64, 1, 1, 8);
399- MaxOutLocalAddr = (__local_mem__ float*)MaxOutLocal.GetPhyAddr();395+ MaxOutLocalAddr = (__ubuf__ float*)MaxOutLocal.GetPhyAddr();
400 MinOutLocal = MinOutQueue.AllocTensor<float>();396 MinOutLocal = MinOutQueue.AllocTensor<float>();
401 AscendC::Duplicate(MinOutLocal, MAX_FLOAT_VALUE, 64, 1, 1, 8);397 AscendC::Duplicate(MinOutLocal, MAX_FLOAT_VALUE, 64, 1, 1, 8);
402- MinOutLocalAddr = (__local_mem__ float*)MinOutLocal.GetPhyAddr();398+ MinOutLocalAddr = (__ubuf__ float*)MinOutLocal.GetPhyAddr();
403 scaleOutLocal = scaleOutQueue.AllocTensor<float>();399 scaleOutLocal = scaleOutQueue.AllocTensor<float>();
404 AscendC::Duplicate(scaleOutLocal, (float)0.0, 64, 1, 1, 8);400 AscendC::Duplicate(scaleOutLocal, (float)0.0, 64, 1, 1, 8);
405- scaleOutLocalAddr = (__local_mem__ float*)scaleOutLocal.GetPhyAddr();401+ scaleOutLocalAddr = (__ubuf__ float*)scaleOutLocal.GetPhyAddr();
406 offsetLocal = offsetQueue.AllocTensor<float>();402 offsetLocal = offsetQueue.AllocTensor<float>();
407 AscendC::Duplicate(offsetLocal, (float)0.0, 64, 1, 1, 8);403 AscendC::Duplicate(offsetLocal, (float)0.0, 64, 1, 1, 8);
408- offsetLocalAddr = (__local_mem__ float*)offsetLocal.GetPhyAddr();404+ offsetLocalAddr = (__ubuf__ float*)offsetLocal.GetPhyAddr();
409 } else {405 } else {
410 scaleOutLocal = scaleOutQueue.AllocTensor<float>();406 scaleOutLocal = scaleOutQueue.AllocTensor<float>();
411 AscendC::Duplicate(scaleOutLocal, (float)0.0, 64, 1, 1, 8);407 AscendC::Duplicate(scaleOutLocal, (float)0.0, 64, 1, 1, 8);
412- scaleOutLocalAddr = (__local_mem__ float*)scaleOutLocal.GetPhyAddr();408+ scaleOutLocalAddr = (__ubuf__ float*)scaleOutLocal.GetPhyAddr();
413 }409 }
414 scaleOffset = 0;410 scaleOffset = 0;
415 CopyInScaleByEle(scaleOffset, tilingData_.coreNum);411 CopyInScaleByEle(scaleOffset, tilingData_.coreNum);
@@ -428,12 +424,12 @@ template <typename T, typename yDtype, bool hasSmooth, bool isSymmetrical>
428__aicore__ inline void DynamicQuantMoeLargeShapePertensor<T, yDtype, hasSmooth, isSymmetrical>::ProcessY()424__aicore__ inline void DynamicQuantMoeLargeShapePertensor<T, yDtype, hasSmooth, isSymmetrical>::ProcessY()
429{425{
430 LocalTensor<float> scaleOutLocal = scaleOutQueue.DeQue<float>();426 LocalTensor<float> scaleOutLocal = scaleOutQueue.DeQue<float>();
431- __local_mem__ float* scaleOutLocalAddr = (__local_mem__ float*)scaleOutLocal.GetPhyAddr();427+ __ubuf__ float* scaleOutLocalAddr = (__ubuf__ float*)scaleOutLocal.GetPhyAddr();
432 LocalTensor<float> offsetLocal;428 LocalTensor<float> offsetLocal;
433- __local_mem__ float* offsetLocalAddr;429+ __ubuf__ float* offsetLocalAddr;
434 if constexpr (isSymmetrical == false) {430 if constexpr (isSymmetrical == false) {
435 offsetLocal = offsetQueue.DeQue<float>();431 offsetLocal = offsetQueue.DeQue<float>();
436- offsetLocalAddr = (__local_mem__ float*)offsetLocal.GetPhyAddr();432+ offsetLocalAddr = (__ubuf__ float*)offsetLocal.GetPhyAddr();
437 }433 }
438 434 
439 smoothIndex_ = 0;435 smoothIndex_ = 0;
@@ -462,7 +458,7 @@ __aicore__ inline void DynamicQuantMoeLargeShapePertensor<T, yDtype, hasSmooth,
462 458 
463template <typename T, typename yDtype, bool hasSmooth, bool isSymmetrical>459template <typename T, typename yDtype, bool hasSmooth, bool isSymmetrical>
464__aicore__ inline void DynamicQuantMoeLargeShapePertensor<T, yDtype, hasSmooth, isSymmetrical>::ProcessYRow(460__aicore__ inline void DynamicQuantMoeLargeShapePertensor<T, yDtype, hasSmooth, isSymmetrical>::ProcessYRow(
465- uint32_t i, uint32_t j, __local_mem__ float* scaleAddr, __local_mem__ float* offsetAddr)461+ uint32_t i, uint32_t j, __ubuf__ float* scaleAddr, __ubuf__ float* offsetAddr)
466{462{
467 offsetBase = i * THIRTY_TWO + j;463 offsetBase = i * THIRTY_TWO + j;
468 srcOffset = offsetBase * tilingData_.rowLen;464 srcOffset = offsetBase * tilingData_.rowLen;
@@ -536,24 +532,24 @@ __aicore__ inline void DynamicQuantMoeLargeShapePertensor<T, yDtype, hasSmooth,
536 uint32_t elementNum)532 uint32_t elementNum)
537{533{
538 LocalTensor<T> inLocal = inQueue.DeQue<T>();534 LocalTensor<T> inLocal = inQueue.DeQue<T>();
539- __local_mem__ T* inLocalAddr = (__local_mem__ T*)inLocal.GetPhyAddr();535+ __ubuf__ T* inLocalAddr = (__ubuf__ T*)inLocal.GetPhyAddr();
540 LocalTensor<T> smoothLocal = smoothQueue.DeQue<T>();536 LocalTensor<T> smoothLocal = smoothQueue.DeQue<T>();
541- __local_mem__ T* smoothLocalAddr = (__local_mem__ T*)smoothLocal.GetPhyAddr();537+ __ubuf__ T* smoothLocalAddr = (__ubuf__ T*)smoothLocal.GetPhyAddr();
542 LocalTensor<float> maxToWorkSpaceLocal;538 LocalTensor<float> maxToWorkSpaceLocal;
543- __local_mem__ float* maxToWorkSpaceLocalAddr;539+ __ubuf__ float* maxToWorkSpaceLocalAddr;
544 LocalTensor<float> minToWorkSpaceLocal;540 LocalTensor<float> minToWorkSpaceLocal;
545- __local_mem__ float* minToWorkSpaceLocalAddr;541+ __ubuf__ float* minToWorkSpaceLocalAddr;
546 LocalTensor<float> scaleToWorkSpaceLocal;542 LocalTensor<float> scaleToWorkSpaceLocal;
547- __local_mem__ float* scaleToWorkSpaceLocalAddr;543+ __ubuf__ float* scaleToWorkSpaceLocalAddr;
548 544 
549 if constexpr (isSymmetrical == false) {545 if constexpr (isSymmetrical == false) {
550 maxToWorkSpaceLocal = MaxToWorkSpaceQueue.DeQue<float>();546 maxToWorkSpaceLocal = MaxToWorkSpaceQueue.DeQue<float>();
551- maxToWorkSpaceLocalAddr = (__local_mem__ float*)maxToWorkSpaceLocal.GetPhyAddr();547+ maxToWorkSpaceLocalAddr = (__ubuf__ float*)maxToWorkSpaceLocal.GetPhyAddr();
552 minToWorkSpaceLocal = MinToWorkSpaceQueue.DeQue<float>();548 minToWorkSpaceLocal = MinToWorkSpaceQueue.DeQue<float>();
553- minToWorkSpaceLocalAddr = (__local_mem__ float*)minToWorkSpaceLocal.GetPhyAddr();549+ minToWorkSpaceLocalAddr = (__ubuf__ float*)minToWorkSpaceLocal.GetPhyAddr();
554 } else {550 } else {
555 scaleToWorkSpaceLocal = scaleToWorkSpaceQueue.DeQue<float>();551 scaleToWorkSpaceLocal = scaleToWorkSpaceQueue.DeQue<float>();
556- scaleToWorkSpaceLocalAddr = (__local_mem__ float*)scaleToWorkSpaceLocal.GetPhyAddr();552+ scaleToWorkSpaceLocalAddr = (__ubuf__ float*)scaleToWorkSpaceLocal.GetPhyAddr();
557 }553 }
558 ComputeMaxRowScaleVF(inLocalAddr, smoothLocalAddr, scaleToWorkSpaceLocalAddr, maxToWorkSpaceLocalAddr,554 ComputeMaxRowScaleVF(inLocalAddr, smoothLocalAddr, scaleToWorkSpaceLocalAddr, maxToWorkSpaceLocalAddr,
559 minToWorkSpaceLocalAddr, elementNum);555 minToWorkSpaceLocalAddr, elementNum);
@@ -569,20 +565,20 @@ __aicore__ inline void DynamicQuantMoeLargeShapePertensor<T, yDtype, hasSmooth,
569 565 
570template <typename T, typename yDtype, bool hasSmooth, bool isSymmetrical>566template <typename T, typename yDtype, bool hasSmooth, bool isSymmetrical>
571__aicore__ inline void DynamicQuantMoeLargeShapePertensor<T, yDtype, hasSmooth, isSymmetrical>::ComputeMaxColScale(567__aicore__ inline void DynamicQuantMoeLargeShapePertensor<T, yDtype, hasSmooth, isSymmetrical>::ComputeMaxColScale(
572- uint32_t elementNum, __local_mem__ float* maxAddr, __local_mem__ float* minAddr, __local_mem__ float* scaleAddr,568+ uint32_t elementNum, __ubuf__ float* maxAddr, __ubuf__ float* minAddr, __ubuf__ float* scaleAddr,
573- __local_mem__ float* offsetAddr)569+ __ubuf__ float* offsetAddr)
574{570{
575 LocalTensor<float> scaleFromWorkSpaceLocal;571 LocalTensor<float> scaleFromWorkSpaceLocal;
576- __local_mem__ float* scaleFromWorkSpaceLocalAddr;572+ __ubuf__ float* scaleFromWorkSpaceLocalAddr;
577 LocalTensor<float> maxFromWorkSpaceLocal;573 LocalTensor<float> maxFromWorkSpaceLocal;
578- __local_mem__ float* maxFromWorkSpaceLocalAddr;574+ __ubuf__ float* maxFromWorkSpaceLocalAddr;
579 LocalTensor<float> minFromWorkSpaceLocal;575 LocalTensor<float> minFromWorkSpaceLocal;
580- __local_mem__ float* minFromWorkSpaceLocalAddr;576+ __ubuf__ float* minFromWorkSpaceLocalAddr;
581 if constexpr (isSymmetrical == false) {577 if constexpr (isSymmetrical == false) {
582 maxFromWorkSpaceLocal = MaxFromWorkSpaceQueue.DeQue<float>();578 maxFromWorkSpaceLocal = MaxFromWorkSpaceQueue.DeQue<float>();
583- maxFromWorkSpaceLocalAddr = (__local_mem__ float*)maxFromWorkSpaceLocal.GetPhyAddr();579+ maxFromWorkSpaceLocalAddr = (__ubuf__ float*)maxFromWorkSpaceLocal.GetPhyAddr();
584 minFromWorkSpaceLocal = MinFromWorkSpaceQueue.DeQue<float>();580 minFromWorkSpaceLocal = MinFromWorkSpaceQueue.DeQue<float>();
585- minFromWorkSpaceLocalAddr = (__local_mem__ float*)minFromWorkSpaceLocal.GetPhyAddr();581+ minFromWorkSpaceLocalAddr = (__ubuf__ float*)minFromWorkSpaceLocal.GetPhyAddr();
586 ComputeMaxColScaleVF(scaleFromWorkSpaceLocalAddr, scaleAddr, maxFromWorkSpaceLocalAddr, maxAddr,582 ComputeMaxColScaleVF(scaleFromWorkSpaceLocalAddr, scaleAddr, maxFromWorkSpaceLocalAddr, maxAddr,
587 minFromWorkSpaceLocalAddr, minAddr, elementNum);583 minFromWorkSpaceLocalAddr, minAddr, elementNum);
588 ComputeScaleSymVF(maxAddr, minAddr, scaleAddr, 1);584 ComputeScaleSymVF(maxAddr, minAddr, scaleAddr, 1);
@@ -591,7 +587,7 @@ __aicore__ inline void DynamicQuantMoeLargeShapePertensor<T, yDtype, hasSmooth,
591 MinFromWorkSpaceQueue.FreeTensor(minFromWorkSpaceLocal);587 MinFromWorkSpaceQueue.FreeTensor(minFromWorkSpaceLocal);
592 } else {588 } else {
593 scaleFromWorkSpaceLocal = scaleFromWorkSpaceQueue.DeQue<float>();589 scaleFromWorkSpaceLocal = scaleFromWorkSpaceQueue.DeQue<float>();
594- scaleFromWorkSpaceLocalAddr = (__local_mem__ float*)scaleFromWorkSpaceLocal.GetPhyAddr();590+ scaleFromWorkSpaceLocalAddr = (__ubuf__ float*)scaleFromWorkSpaceLocal.GetPhyAddr();
595 ComputeMaxColScaleVF(scaleFromWorkSpaceLocalAddr, scaleAddr, maxFromWorkSpaceLocalAddr, maxAddr,591 ComputeMaxColScaleVF(scaleFromWorkSpaceLocalAddr, scaleAddr, maxFromWorkSpaceLocalAddr, maxAddr,
596 minFromWorkSpaceLocalAddr, minAddr, elementNum);592 minFromWorkSpaceLocalAddr, minAddr, elementNum);
597 scaleFromWorkSpaceQueue.FreeTensor(scaleFromWorkSpaceLocal);593 scaleFromWorkSpaceQueue.FreeTensor(scaleFromWorkSpaceLocal);
@@ -600,8 +596,8 @@ __aicore__ inline void DynamicQuantMoeLargeShapePertensor<T, yDtype, hasSmooth,
600 596 
601template <typename T, typename yDtype, bool hasSmooth, bool isSymmetrical>597template <typename T, typename yDtype, bool hasSmooth, bool isSymmetrical>
602__aicore__ inline void DynamicQuantMoeLargeShapePertensor<T, yDtype, hasSmooth, isSymmetrical>::ComputeMaxRowScaleVF(598__aicore__ inline void DynamicQuantMoeLargeShapePertensor<T, yDtype, hasSmooth, isSymmetrical>::ComputeMaxRowScaleVF(
603- __local_mem__ T* inLocalAddr, __local_mem__ T* smoothLocalAddr, __local_mem__ float* scaleLocalAddr,599+ __ubuf__ T* inLocalAddr, __ubuf__ T* smoothLocalAddr, __ubuf__ float* scaleLocalAddr, __ubuf__ float* maxLocalAddr,
604- __local_mem__ float* maxLocalAddr, __local_mem__ float* minLocalAddr, uint32_t elementNum)600+ __ubuf__ float* minLocalAddr, uint32_t elementNum)
605{601{
606 uint32_t dtypeSize = sizeof(float);602 uint32_t dtypeSize = sizeof(float);
607 uint32_t VL = AscendC::VECTOR_REG_WIDTH / dtypeSize;603 uint32_t VL = AscendC::VECTOR_REG_WIDTH / dtypeSize;
@@ -621,19 +617,19 @@ __aicore__ inline void DynamicQuantMoeLargeShapePertensor<T, yDtype, hasSmooth,
621 AscendC::MicroAPI::RegTensor<float, MicroAPI::RegTraitNumOne> vreg8_1;617 AscendC::MicroAPI::RegTensor<float, MicroAPI::RegTraitNumOne> vreg8_1;
622 AscendC::MicroAPI::RegTensor<float, MicroAPI::RegTraitNumOne> vreg8_2;618 AscendC::MicroAPI::RegTensor<float, MicroAPI::RegTraitNumOne> vreg8_2;
623 if constexpr (isSymmetrical == false) {619 if constexpr (isSymmetrical == false) {
624- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_BRC_B32>(vreg6_1, maxLocalAddr);620+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_BRC_B32>(vreg6_1, maxLocalAddr);
625- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_BRC_B32>(vreg6_2, minLocalAddr);621+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_BRC_B32>(vreg6_2, minLocalAddr);
626 } else {622 } else {
627- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_BRC_B32>(vreg6_1, scaleLocalAddr);623+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_BRC_B32>(vreg6_1, scaleLocalAddr);
628 }624 }
629 AscendC::MicroAPI::MaskReg mask;625 AscendC::MicroAPI::MaskReg mask;
630 for (uint16_t i = 0; i < static_cast<uint16_t>(vfLoopNum - 1); i++) {626 for (uint16_t i = 0; i < static_cast<uint16_t>(vfLoopNum - 1); i++) {
631 maskNum = elementNum - i * VL;627 maskNum = elementNum - i * VL;
632 mask = AscendC::MicroAPI::UpdateMask<float>(maskNum);628 mask = AscendC::MicroAPI::UpdateMask<float>(maskNum);
633- AscendC::MicroAPI::DataCopy<T, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(vreg0, inLocalAddr + i * VL);629+ AscendC::MicroAPI::LoadAlign<T, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(vreg0, inLocalAddr + i * VL);
634 AscendC::MicroAPI::Cast<float, T, castTrait0>(vreg1, vreg0, mask);630 AscendC::MicroAPI::Cast<float, T, castTrait0>(vreg1, vreg0, mask);
635- AscendC::MicroAPI::DataCopy<T, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(vreg2,631+ AscendC::MicroAPI::LoadAlign<T, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(vreg2,
636- smoothLocalAddr + i * VL);632+ smoothLocalAddr + i * VL);
637 AscendC::MicroAPI::Cast<float, T, castTrait0>(vreg3, vreg2, mask);633 AscendC::MicroAPI::Cast<float, T, castTrait0>(vreg3, vreg2, mask);
638 AscendC::MicroAPI::Mul(vreg1, vreg1, vreg3, mask);634 AscendC::MicroAPI::Mul(vreg1, vreg1, vreg3, mask);
639 if constexpr (isSymmetrical == false) {635 if constexpr (isSymmetrical == false) {
@@ -645,17 +641,17 @@ __aicore__ inline void DynamicQuantMoeLargeShapePertensor<T, yDtype, hasSmooth,
645 AscendC::MicroAPI::Max(vreg6_1, vreg5, vreg6_1, mask);641 AscendC::MicroAPI::Max(vreg6_1, vreg5, vreg6_1, mask);
646 }642 }
647 }643 }
648- AscendC::MicroAPI::ReduceMax<float>(vreg7_1, vreg6_1, mask);644+ AscendC::MicroAPI::Reduce<MicroAPI::ReduceType::MAX, float>(vreg7_1, vreg6_1, mask);
649 if constexpr (isSymmetrical == false) {645 if constexpr (isSymmetrical == false) {
650- AscendC::MicroAPI::ReduceMin<float>(vreg7_2, vreg6_2, mask);646+ AscendC::MicroAPI::Reduce<MicroAPI::ReduceType::MIN, float>(vreg7_2, vreg6_2, mask);
651 }647 }
652 for (uint16_t i = vfLoopNum - 1; i < vfLoopNum; i++) {648 for (uint16_t i = vfLoopNum - 1; i < vfLoopNum; i++) {
653 maskNum = elementNum - i * VL;649 maskNum = elementNum - i * VL;
654 mask = AscendC::MicroAPI::UpdateMask<float>(maskNum);650 mask = AscendC::MicroAPI::UpdateMask<float>(maskNum);
655- AscendC::MicroAPI::DataCopy<T, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(vreg0, inLocalAddr + i * VL);651+ AscendC::MicroAPI::LoadAlign<T, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(vreg0, inLocalAddr + i * VL);
656 AscendC::MicroAPI::Cast<float, T, castTrait0>(vreg1, vreg0, mask);652 AscendC::MicroAPI::Cast<float, T, castTrait0>(vreg1, vreg0, mask);
657- AscendC::MicroAPI::DataCopy<T, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(vreg2,653+ AscendC::MicroAPI::LoadAlign<T, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(vreg2,
658- smoothLocalAddr + i * VL);654+ smoothLocalAddr + i * VL);
659 AscendC::MicroAPI::Cast<float, T, castTrait0>(vreg3, vreg2, mask);655 AscendC::MicroAPI::Cast<float, T, castTrait0>(vreg3, vreg2, mask);
660 AscendC::MicroAPI::Mul(vreg1, vreg1, vreg3, mask);656 AscendC::MicroAPI::Mul(vreg1, vreg1, vreg3, mask);
661 if constexpr (isSymmetrical == false) {657 if constexpr (isSymmetrical == false) {
@@ -666,9 +662,9 @@ __aicore__ inline void DynamicQuantMoeLargeShapePertensor<T, yDtype, hasSmooth,
666 AscendC::MicroAPI::Muls(vreg5, vreg5, float(1.0) / maxValue, mask);662 AscendC::MicroAPI::Muls(vreg5, vreg5, float(1.0) / maxValue, mask);
667 AscendC::MicroAPI::Max(vreg6_1, vreg5, vreg6_1, mask);663 AscendC::MicroAPI::Max(vreg6_1, vreg5, vreg6_1, mask);
668 }664 }
669- AscendC::MicroAPI::ReduceMax<float>(vreg8_1, vreg6_1, mask);665+ AscendC::MicroAPI::Reduce<MicroAPI::ReduceType::MAX, float>(vreg8_1, vreg6_1, mask);
670 if constexpr (isSymmetrical == false) {666 if constexpr (isSymmetrical == false) {
671- AscendC::MicroAPI::ReduceMin<float>(vreg8_2, vreg6_2, mask);667+ AscendC::MicroAPI::Reduce<MicroAPI::ReduceType::MIN, float>(vreg8_2, vreg6_2, mask);
672 }668 }
673 AscendC::MicroAPI::Max(vreg8_1, vreg7_1, vreg8_1, mask);669 AscendC::MicroAPI::Max(vreg8_1, vreg7_1, vreg8_1, mask);
674 if constexpr (isSymmetrical == false) {670 if constexpr (isSymmetrical == false) {
@@ -676,22 +672,21 @@ __aicore__ inline void DynamicQuantMoeLargeShapePertensor<T, yDtype, hasSmooth,
676 }672 }
677 }673 }
678 if constexpr (isSymmetrical == false) {674 if constexpr (isSymmetrical == false) {
679- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::StoreDist::DIST_FIRST_ELEMENT_B32>(maxLocalAddr,675+ AscendC::MicroAPI::StoreAlign<float, AscendC::MicroAPI::StoreDist::DIST_FIRST_ELEMENT_B32>(maxLocalAddr,
680- vreg8_1, mask);676+ vreg8_1, mask);
681- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::StoreDist::DIST_FIRST_ELEMENT_B32>(minLocalAddr,677+ AscendC::MicroAPI::StoreAlign<float, AscendC::MicroAPI::StoreDist::DIST_FIRST_ELEMENT_B32>(minLocalAddr,
682- vreg8_2, mask);678+ vreg8_2, mask);
683 } else {679 } else {
684- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::StoreDist::DIST_FIRST_ELEMENT_B32>(scaleLocalAddr,680+ AscendC::MicroAPI::StoreAlign<float, AscendC::MicroAPI::StoreDist::DIST_FIRST_ELEMENT_B32>(scaleLocalAddr,
685- vreg8_1, mask);681+ vreg8_1, mask);
686 }682 }
687 }683 }
688}684}
689 685 
690template <typename T, typename yDtype, bool hasSmooth, bool isSymmetrical>686template <typename T, typename yDtype, bool hasSmooth, bool isSymmetrical>
691__aicore__ inline void DynamicQuantMoeLargeShapePertensor<T, yDtype, hasSmooth, isSymmetrical>::ComputeMaxColScaleVF(687__aicore__ inline void DynamicQuantMoeLargeShapePertensor<T, yDtype, hasSmooth, isSymmetrical>::ComputeMaxColScaleVF(
692- __local_mem__ float* scaleLocalAddr, __local_mem__ float* scaleOutLocalAddr, __local_mem__ float* maxLocalAddr,688+ __ubuf__ float* scaleLocalAddr, __ubuf__ float* scaleOutLocalAddr, __ubuf__ float* maxLocalAddr,
693- __local_mem__ float* maxOutLocalAddr, __local_mem__ float* minLocalAddr, __local_mem__ float* minOutLocalAddr,689+ __ubuf__ float* maxOutLocalAddr, __ubuf__ float* minLocalAddr, __ubuf__ float* minOutLocalAddr, uint32_t elementNum)
694- uint32_t elementNum)
695{690{
696 uint32_t dtypeSize = sizeof(float);691 uint32_t dtypeSize = sizeof(float);
697 uint32_t VL = AscendC::VECTOR_REG_WIDTH / dtypeSize;692 uint32_t VL = AscendC::VECTOR_REG_WIDTH / dtypeSize;
@@ -702,72 +697,71 @@ __aicore__ inline void DynamicQuantMoeLargeShapePertensor<T, yDtype, hasSmooth,
702 AscendC::MicroAPI::RegTensor<float, MicroAPI::RegTraitNumOne> vreg0_1, vreg0_2, vreg1_1, vreg1_2, vreg2_1,697 AscendC::MicroAPI::RegTensor<float, MicroAPI::RegTraitNumOne> vreg0_1, vreg0_2, vreg1_1, vreg1_2, vreg2_1,
703 vreg2_2, vreg3_1, vreg3_2;698 vreg2_2, vreg3_1, vreg3_2;
704 if constexpr (isSymmetrical == false) {699 if constexpr (isSymmetrical == false) {
705- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_BRC_B32>(vreg1_1, maxOutLocalAddr);700+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_BRC_B32>(vreg1_1, maxOutLocalAddr);
706- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_BRC_B32>(vreg1_2, minOutLocalAddr);701+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_BRC_B32>(vreg1_2, minOutLocalAddr);
707 } else {702 } else {
708- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_BRC_B32>(vreg1_1, scaleOutLocalAddr);703+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_BRC_B32>(vreg1_1, scaleOutLocalAddr);
709 }704 }
710 AscendC::MicroAPI::MaskReg mask = AscendC::MicroAPI::CreateMask<float, AscendC::MicroAPI::MaskPattern::ALL>();705 AscendC::MicroAPI::MaskReg mask = AscendC::MicroAPI::CreateMask<float, AscendC::MicroAPI::MaskPattern::ALL>();
711 for (uint16_t i = 0; i < static_cast<uint16_t>(vfLoopNum - 1); i++) {706 for (uint16_t i = 0; i < static_cast<uint16_t>(vfLoopNum - 1); i++) {
712 maskNum = elementNum - i * VL;707 maskNum = elementNum - i * VL;
713 mask = AscendC::MicroAPI::UpdateMask<float>(maskNum);708 mask = AscendC::MicroAPI::UpdateMask<float>(maskNum);
714 if constexpr (isSymmetrical == false) {709 if constexpr (isSymmetrical == false) {
715- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(vreg0_1,710+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(vreg0_1,
716- maxLocalAddr + i * VL);711+ maxLocalAddr + i * VL);
717 AscendC::MicroAPI::Max(vreg1_1, vreg0_1, vreg1_1, mask);712 AscendC::MicroAPI::Max(vreg1_1, vreg0_1, vreg1_1, mask);
718- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(vreg0_2,713+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(vreg0_2,
719- minLocalAddr + i * VL);714+ minLocalAddr + i * VL);
720 AscendC::MicroAPI::Min(vreg1_2, vreg0_2, vreg1_2, mask);715 AscendC::MicroAPI::Min(vreg1_2, vreg0_2, vreg1_2, mask);
721 } else {716 } else {
722- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(vreg0_1,717+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(vreg0_1,
723- scaleLocalAddr + i * VL);718+ scaleLocalAddr + i * VL);
724 AscendC::MicroAPI::Max(vreg1_1, vreg0_1, vreg1_1, mask);719 AscendC::MicroAPI::Max(vreg1_1, vreg0_1, vreg1_1, mask);
725 }720 }
726 }721 }
727 if constexpr (isSymmetrical == false) {722 if constexpr (isSymmetrical == false) {
728- AscendC::MicroAPI::ReduceMax<float>(vreg2_1, vreg1_1, mask);723+ AscendC::MicroAPI::Reduce<MicroAPI::ReduceType::MAX, float>(vreg2_1, vreg1_1, mask);
729- AscendC::MicroAPI::ReduceMin<float>(vreg2_2, vreg1_2, mask);724+ AscendC::MicroAPI::Reduce<MicroAPI::ReduceType::MIN, float>(vreg2_2, vreg1_2, mask);
730 } else {725 } else {
731- AscendC::MicroAPI::ReduceMax<float>(vreg2_1, vreg1_1, mask);726+ AscendC::MicroAPI::Reduce<MicroAPI::ReduceType::MAX, float>(vreg2_1, vreg1_1, mask);
732 }727 }
733 for (uint16_t i = vfLoopNum - 1; i < vfLoopNum; i++) {728 for (uint16_t i = vfLoopNum - 1; i < vfLoopNum; i++) {
734 maskNum = elementNum - i * VL;729 maskNum = elementNum - i * VL;
735 mask = AscendC::MicroAPI::UpdateMask<float>(maskNum);730 mask = AscendC::MicroAPI::UpdateMask<float>(maskNum);
736 if constexpr (isSymmetrical == false) {731 if constexpr (isSymmetrical == false) {
737- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(vreg0_1,732+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(vreg0_1,
738- maxLocalAddr + i * VL);733+ maxLocalAddr + i * VL);
739 AscendC::MicroAPI::Max(vreg1_1, vreg0_1, vreg1_1, mask);734 AscendC::MicroAPI::Max(vreg1_1, vreg0_1, vreg1_1, mask);
740- AscendC::MicroAPI::ReduceMax<float>(vreg3_1, vreg1_1, mask);735+ AscendC::MicroAPI::Reduce<MicroAPI::ReduceType::MAX, float>(vreg3_1, vreg1_1, mask);
741 AscendC::MicroAPI::Max(vreg3_1, vreg2_1, vreg3_1, mask);736 AscendC::MicroAPI::Max(vreg3_1, vreg2_1, vreg3_1, mask);
742- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(vreg0_2,737+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(vreg0_2,
743- minLocalAddr + i * VL);738+ minLocalAddr + i * VL);
744 AscendC::MicroAPI::Min(vreg1_2, vreg0_2, vreg1_2, mask);739 AscendC::MicroAPI::Min(vreg1_2, vreg0_2, vreg1_2, mask);
745- AscendC::MicroAPI::ReduceMin<float>(vreg3_2, vreg1_2, mask);740+ AscendC::MicroAPI::Reduce<MicroAPI::ReduceType::MIN, float>(vreg3_2, vreg1_2, mask);
746 AscendC::MicroAPI::Min(vreg3_2, vreg2_2, vreg3_2, mask);741 AscendC::MicroAPI::Min(vreg3_2, vreg2_2, vreg3_2, mask);
747 } else {742 } else {
748- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(vreg0_1,743+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(vreg0_1,
749- scaleLocalAddr + i * VL);744+ scaleLocalAddr + i * VL);
750 AscendC::MicroAPI::Max(vreg1_1, vreg0_1, vreg1_1, mask);745 AscendC::MicroAPI::Max(vreg1_1, vreg0_1, vreg1_1, mask);
751- AscendC::MicroAPI::ReduceMax<float>(vreg3_1, vreg1_1, mask);746+ AscendC::MicroAPI::Reduce<MicroAPI::ReduceType::MAX, float>(vreg3_1, vreg1_1, mask);
752 AscendC::MicroAPI::Max(vreg3_1, vreg2_1, vreg3_1, mask);747 AscendC::MicroAPI::Max(vreg3_1, vreg2_1, vreg3_1, mask);
753 }748 }
754 }749 }
755 if constexpr (isSymmetrical == false) {750 if constexpr (isSymmetrical == false) {
756- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::StoreDist::DIST_FIRST_ELEMENT_B32>(maxOutLocalAddr,751+ AscendC::MicroAPI::StoreAlign<float, AscendC::MicroAPI::StoreDist::DIST_FIRST_ELEMENT_B32>(maxOutLocalAddr,
757- vreg3_1, mask);752+ vreg3_1, mask);
758- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::StoreDist::DIST_FIRST_ELEMENT_B32>(minOutLocalAddr,753+ AscendC::MicroAPI::StoreAlign<float, AscendC::MicroAPI::StoreDist::DIST_FIRST_ELEMENT_B32>(minOutLocalAddr,
759- vreg3_2, mask);754+ vreg3_2, mask);
760 } else {755 } else {
761- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::StoreDist::DIST_FIRST_ELEMENT_B32>(scaleOutLocalAddr,756+ AscendC::MicroAPI::StoreAlign<float, AscendC::MicroAPI::StoreDist::DIST_FIRST_ELEMENT_B32>(
762- vreg3_1, mask);757+ scaleOutLocalAddr, vreg3_1, mask);
763 }758 }
764 }759 }
765}760}
766 761 
767template <typename T, typename yDtype, bool hasSmooth, bool isSymmetrical>762template <typename T, typename yDtype, bool hasSmooth, bool isSymmetrical>
768__aicore__ inline void DynamicQuantMoeLargeShapePertensor<T, yDtype, hasSmooth, isSymmetrical>::ComputeScaleSymVF(763__aicore__ inline void DynamicQuantMoeLargeShapePertensor<T, yDtype, hasSmooth, isSymmetrical>::ComputeScaleSymVF(
769- __local_mem__ float* maxLocalAddr, __local_mem__ float* minLocalAddr, __local_mem__ float* scaleLocalAddr,764+ __ubuf__ float* maxLocalAddr, __ubuf__ float* minLocalAddr, __ubuf__ float* scaleLocalAddr, uint32_t elementNum)
770- uint32_t elementNum)
771{765{
772 uint32_t VL = AscendC::VECTOR_REG_WIDTH / sizeof(float);766 uint32_t VL = AscendC::VECTOR_REG_WIDTH / sizeof(float);
773 uint16_t vfLoopNum = (elementNum + VL - 1) / VL;767 uint16_t vfLoopNum = (elementNum + VL - 1) / VL;
@@ -779,20 +773,19 @@ __aicore__ inline void DynamicQuantMoeLargeShapePertensor<T, yDtype, hasSmooth,
779 for (uint16_t i = 0; i < vfLoopNum; i++) {773 for (uint16_t i = 0; i < vfLoopNum; i++) {
780 maskNum = elementNum - i * VL;774 maskNum = elementNum - i * VL;
781 mask = AscendC::MicroAPI::UpdateMask<float>(maskNum);775 mask = AscendC::MicroAPI::UpdateMask<float>(maskNum);
782- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(vreg0, maxLocalAddr + i * VL);776+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(vreg0, maxLocalAddr + i * VL);
783- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(vreg1, minLocalAddr + i * VL);777+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(vreg1, minLocalAddr + i * VL);
784 AscendC::MicroAPI::Sub(vreg1, vreg0, vreg1, mask);778 AscendC::MicroAPI::Sub(vreg1, vreg0, vreg1, mask);
785 AscendC::MicroAPI::Muls(vreg1, vreg1, float(1.0) / maxValueNoSym, mask);779 AscendC::MicroAPI::Muls(vreg1, vreg1, float(1.0) / maxValueNoSym, mask);
786- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::StoreDist::DIST_FIRST_ELEMENT_B32>(scaleLocalAddr,780+ AscendC::MicroAPI::StoreAlign<float, AscendC::MicroAPI::StoreDist::DIST_FIRST_ELEMENT_B32>(scaleLocalAddr,
787- vreg1, mask);781+ vreg1, mask);
788 }782 }
789 }783 }
790}784}
791 785 
792template <typename T, typename yDtype, bool hasSmooth, bool isSymmetrical>786template <typename T, typename yDtype, bool hasSmooth, bool isSymmetrical>
793__aicore__ inline void DynamicQuantMoeLargeShapePertensor<T, yDtype, hasSmooth, isSymmetrical>::ComputeOffsetSymVF(787__aicore__ inline void DynamicQuantMoeLargeShapePertensor<T, yDtype, hasSmooth, isSymmetrical>::ComputeOffsetSymVF(
794- __local_mem__ float* maxLocalAddr, __local_mem__ float* scaleLocalAddr, __local_mem__ float* offsetLocalAddr,788+ __ubuf__ float* maxLocalAddr, __ubuf__ float* scaleLocalAddr, __ubuf__ float* offsetLocalAddr, uint32_t elementNum)
795- uint32_t elementNum)
796{789{
797 uint32_t VL = AscendC::VECTOR_REG_WIDTH / sizeof(float);790 uint32_t VL = AscendC::VECTOR_REG_WIDTH / sizeof(float);
798 uint16_t vfLoopNum = (elementNum + VL - 1) / VL;791 uint16_t vfLoopNum = (elementNum + VL - 1) / VL;
@@ -805,27 +798,27 @@ __aicore__ inline void DynamicQuantMoeLargeShapePertensor<T, yDtype, hasSmooth,
805 for (uint16_t i = 0; i < vfLoopNum; i++) {798 for (uint16_t i = 0; i < vfLoopNum; i++) {
806 maskNum = elementNum - i * VL;799 maskNum = elementNum - i * VL;
807 mask = AscendC::MicroAPI::UpdateMask<float>(maskNum);800 mask = AscendC::MicroAPI::UpdateMask<float>(maskNum);
808- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(vreg0, maxLocalAddr + i * VL);801+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(vreg0, maxLocalAddr + i * VL);
809- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(vreg1, scaleLocalAddr + i * VL);802+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(vreg1, scaleLocalAddr + i * VL);
810 AscendC::MicroAPI::Div<float, &mode>(vreg1, vreg0, vreg1, mask);803 AscendC::MicroAPI::Div<float, &mode>(vreg1, vreg0, vreg1, mask);
811 AscendC::MicroAPI::Muls(vreg1, vreg1, float(-1.0), mask);804 AscendC::MicroAPI::Muls(vreg1, vreg1, float(-1.0), mask);
812 AscendC::MicroAPI::Adds(vreg1, vreg1, maxValue, mask);805 AscendC::MicroAPI::Adds(vreg1, vreg1, maxValue, mask);
813- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::StoreDist::DIST_FIRST_ELEMENT_B32>(offsetLocalAddr,806+ AscendC::MicroAPI::StoreAlign<float, AscendC::MicroAPI::StoreDist::DIST_FIRST_ELEMENT_B32>(offsetLocalAddr,
814- vreg1, mask);807+ vreg1, mask);
815 }808 }
816 }809 }
817}810}
818 811 
819template <typename T, typename yDtype, bool hasSmooth, bool isSymmetrical>812template <typename T, typename yDtype, bool hasSmooth, bool isSymmetrical>
820__aicore__ inline void DynamicQuantMoeLargeShapePertensor<T, yDtype, hasSmooth, isSymmetrical>::ComputeY(813__aicore__ inline void DynamicQuantMoeLargeShapePertensor<T, yDtype, hasSmooth, isSymmetrical>::ComputeY(
821- uint32_t elementNum, __local_mem__ float* scaleAddr, __local_mem__ float* offsetAddr)814+ uint32_t elementNum, __ubuf__ float* scaleAddr, __ubuf__ float* offsetAddr)
822{815{
823 LocalTensor<T> inLocal = inQueue.DeQue<T>();816 LocalTensor<T> inLocal = inQueue.DeQue<T>();
824- __local_mem__ T* inLocalAddr = (__local_mem__ T*)inLocal.GetPhyAddr();817+ __ubuf__ T* inLocalAddr = (__ubuf__ T*)inLocal.GetPhyAddr();
825 LocalTensor<yCopyDtype> outLocal = outQueue.AllocTensor<yCopyDtype>();818 LocalTensor<yCopyDtype> outLocal = outQueue.AllocTensor<yCopyDtype>();
826- __local_mem__ yCopyDtype* outAddr = (__local_mem__ yCopyDtype*)outLocal.GetPhyAddr();819+ __ubuf__ yCopyDtype* outAddr = (__ubuf__ yCopyDtype*)outLocal.GetPhyAddr();
827 LocalTensor<T> smoothLocal = smoothQueue.DeQue<T>();820 LocalTensor<T> smoothLocal = smoothQueue.DeQue<T>();
828- __local_mem__ T* smoothLocalAddr = (__local_mem__ T*)smoothLocal.GetPhyAddr();821+ __ubuf__ T* smoothLocalAddr = (__ubuf__ T*)smoothLocal.GetPhyAddr();
829 ComputeYVF(inLocalAddr, smoothLocalAddr, outAddr, scaleAddr, offsetAddr, elementNum);822 ComputeYVF(inLocalAddr, smoothLocalAddr, outAddr, scaleAddr, offsetAddr, elementNum);
830 smoothQueue.FreeTensor(smoothLocal);823 smoothQueue.FreeTensor(smoothLocal);
831 inQueue.FreeTensor(inLocal);824 inQueue.FreeTensor(inLocal);
@@ -834,8 +827,8 @@ __aicore__ inline void DynamicQuantMoeLargeShapePertensor<T, yDtype, hasSmooth,
834 827 
835template <typename T, typename yDtype, bool hasSmooth, bool isSymmetrical>828template <typename T, typename yDtype, bool hasSmooth, bool isSymmetrical>
836__aicore__ inline void DynamicQuantMoeLargeShapePertensor<T, yDtype, hasSmooth, isSymmetrical>::ComputeYVF(829__aicore__ inline void DynamicQuantMoeLargeShapePertensor<T, yDtype, hasSmooth, isSymmetrical>::ComputeYVF(
837- __local_mem__ T* inLocalAddr, __local_mem__ T* smoothLocalAddr, __local_mem__ yCopyDtype* outAddr,830+ __ubuf__ T* inLocalAddr, __ubuf__ T* smoothLocalAddr, __ubuf__ yCopyDtype* outAddr, __ubuf__ float* scaleLocalAddr,
838- __local_mem__ float* scaleLocalAddr, __local_mem__ float* offsetLocalAddr, uint32_t elementNum)831+ __ubuf__ float* offsetLocalAddr, uint32_t elementNum)
839{832{
840 uint32_t VL = AscendC::VECTOR_REG_WIDTH / sizeof(float);833 uint32_t VL = AscendC::VECTOR_REG_WIDTH / sizeof(float);
841 uint16_t vfLoopNum = (elementNum + VL - 1) / VL;834 uint16_t vfLoopNum = (elementNum + VL - 1) / VL;
@@ -853,18 +846,19 @@ __aicore__ inline void DynamicQuantMoeLargeShapePertensor<T, yDtype, hasSmooth,
853 AscendC::MicroAPI::RegTensor<float, MicroAPI::RegTraitNumOne> vreg9;846 AscendC::MicroAPI::RegTensor<float, MicroAPI::RegTraitNumOne> vreg9;
854 AscendC::MicroAPI::RegTensor<float, MicroAPI::RegTraitNumOne> vreg_offset;847 AscendC::MicroAPI::RegTensor<float, MicroAPI::RegTraitNumOne> vreg_offset;
855 848 
856- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_BRC_B32>(vreg9, scaleLocalAddr);849+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_BRC_B32>(vreg9, scaleLocalAddr);
857 if constexpr (isSymmetrical == false) {850 if constexpr (isSymmetrical == false) {
858- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_BRC_B32>(vreg_offset, offsetLocalAddr);851+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_BRC_B32>(vreg_offset,
852+ offsetLocalAddr);
859 }853 }
860 AscendC::MicroAPI::MaskReg mask = AscendC::MicroAPI::CreateMask<float, AscendC::MicroAPI::MaskPattern::ALL>();854 AscendC::MicroAPI::MaskReg mask = AscendC::MicroAPI::CreateMask<float, AscendC::MicroAPI::MaskPattern::ALL>();
861 AscendC::MicroAPI::MaskReg mask2 = AscendC::MicroAPI::CreateMask<float, AscendC::MicroAPI::MaskPattern::H>();855 AscendC::MicroAPI::MaskReg mask2 = AscendC::MicroAPI::CreateMask<float, AscendC::MicroAPI::MaskPattern::H>();
862 for (uint16_t i = 0; i < vfLoopNum; i++) {856 for (uint16_t i = 0; i < vfLoopNum; i++) {
863 auto addr = outAddr + i * VL;857 auto addr = outAddr + i * VL;
864- AscendC::MicroAPI::DataCopy<T, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(vreg0, inLocalAddr + i * VL);858+ AscendC::MicroAPI::LoadAlign<T, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(vreg0, inLocalAddr + i * VL);
865 AscendC::MicroAPI::Cast<float, T, castTrait0>(vreg1, vreg0, mask);859 AscendC::MicroAPI::Cast<float, T, castTrait0>(vreg1, vreg0, mask);
866- AscendC::MicroAPI::DataCopy<T, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(vreg2,860+ AscendC::MicroAPI::LoadAlign<T, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(vreg2,
867- smoothLocalAddr + i * VL);861+ smoothLocalAddr + i * VL);
868 AscendC::MicroAPI::Cast<float, T, castTrait0>(vreg3, vreg2, mask);862 AscendC::MicroAPI::Cast<float, T, castTrait0>(vreg3, vreg2, mask);
869 AscendC::MicroAPI::Mul(vreg4, vreg1, vreg3, mask);863 AscendC::MicroAPI::Mul(vreg4, vreg1, vreg3, mask);
870 AscendC::MicroAPI::Div(vreg5, vreg4, vreg9, mask);864 AscendC::MicroAPI::Div(vreg5, vreg4, vreg9, mask);
@@ -889,11 +883,11 @@ __aicore__ inline void DynamicQuantMoeLargeShapePertensor<T, yDtype, hasSmooth,
889 AscendC::MicroAPI::Cast<yDtype, float, castTrait32tofp8>(vreg8, vreg5, mask);883 AscendC::MicroAPI::Cast<yDtype, float, castTrait32tofp8>(vreg8, vreg5, mask);
890 }884 }
891 if constexpr (IsSameType<yDtype, int4b_t>::value) {885 if constexpr (IsSameType<yDtype, int4b_t>::value) {
892- AscendC::MicroAPI::DataCopy<yCopyDtype, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(addr, vreg8,886+ AscendC::MicroAPI::StoreAlign<yCopyDtype, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(addr, vreg8,
893- mask2);887+ mask2);
894 } else {888 } else {
895- AscendC::MicroAPI::DataCopy<yCopyDtype, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(addr, vreg8,889+ AscendC::MicroAPI::StoreAlign<yCopyDtype, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(addr, vreg8,
896- mask);890+ mask);
897 }891 }
898 }892 }
899 }893 }
Mquant/dynamic_quant/op_kernel/arch35/dynamic_quant_regbase_perchannel_full_load.h+27-27
@@ -42,12 +42,12 @@ private:
42 uint32_t nBlockSize);42 uint32_t nBlockSize);
43 __aicore__ inline void CopyIn(uint32_t bBlockSize, uint32_t nBlockSize, uint64_t xOffset, uint64_t smoothOffset);43 __aicore__ inline void CopyIn(uint32_t bBlockSize, uint32_t nBlockSize, uint64_t xOffset, uint64_t smoothOffset);
44 __aicore__ inline void Compute(uint32_t bBlockSize, uint32_t nBlockSize);44 __aicore__ inline void Compute(uint32_t bBlockSize, uint32_t nBlockSize);
45- __aicore__ inline void ComputeVFforSymmetric(__local_mem__ xDtype* inAddr, __local_mem__ xDtype* smoothAddr,45+ __aicore__ inline void ComputeVFforSymmetric(__ubuf__ xDtype* inAddr, __ubuf__ xDtype* smoothAddr,
46- __local_mem__ yCopyDtype* yAddr, __local_mem__ float* scaleAddr,46+ __ubuf__ yCopyDtype* yAddr, __ubuf__ float* scaleAddr,
47 uint32_t bBlockSize, uint32_t nBlockSize);47 uint32_t bBlockSize, uint32_t nBlockSize);
48- __aicore__ inline void ComputeVFforNoSymmetric(__local_mem__ xDtype* inAddr, __local_mem__ xDtype* smoothAddr,48+ __aicore__ inline void ComputeVFforNoSymmetric(__ubuf__ xDtype* inAddr, __ubuf__ xDtype* smoothAddr,
49- __local_mem__ yCopyDtype* yAddr, __local_mem__ float* scaleAddr,49+ __ubuf__ yCopyDtype* yAddr, __ubuf__ float* scaleAddr,
50- __local_mem__ float* offsetAddr, uint32_t bBlockSize,50+ __ubuf__ float* offsetAddr, uint32_t bBlockSize,
51 uint32_t nBlockSize);51 uint32_t nBlockSize);
52 __aicore__ inline void CopyOut(uint32_t bBlockSize, uint32_t nBlockSize, uint64_t xOffset, uint64_t scaleOffset);52 __aicore__ inline void CopyOut(uint32_t bBlockSize, uint32_t nBlockSize, uint64_t xOffset, uint64_t scaleOffset);
53 53 
@@ -251,8 +251,8 @@ __aicore__ inline void DynamicQuantRegbasePerChannnelFullLoad<xDtype, yDtype, ha
251template <typename xDtype, typename yDtype, bool hasSmooth, bool isSymmetrical>251template <typename xDtype, typename yDtype, bool hasSmooth, bool isSymmetrical>
252__aicore__ inline void252__aicore__ inline void
253DynamicQuantRegbasePerChannnelFullLoad<xDtype, yDtype, hasSmooth, isSymmetrical>::ComputeVFforSymmetric(253DynamicQuantRegbasePerChannnelFullLoad<xDtype, yDtype, hasSmooth, isSymmetrical>::ComputeVFforSymmetric(
254- __local_mem__ xDtype* inAddr, __local_mem__ xDtype* smoothAddr, __local_mem__ yCopyDtype* yAddr,254+ __ubuf__ xDtype* inAddr, __ubuf__ xDtype* smoothAddr, __ubuf__ yCopyDtype* yAddr, __ubuf__ float* scaleAddr,
255- __local_mem__ float* scaleAddr, uint32_t bBlockSize, uint32_t nBlockSize)255+ uint32_t bBlockSize, uint32_t nBlockSize)
256{256{
257 uint32_t nSizeScale = ops::CeilAlign(nBlockSize, ALIGN_NUMBER_FP32);257 uint32_t nSizeScale = ops::CeilAlign(nBlockSize, ALIGN_NUMBER_FP32);
258 uint32_t nSize = ops::CeilAlign(nBlockSize, ALIGN_NUMBER_FP16);258 uint32_t nSize = ops::CeilAlign(nBlockSize, ALIGN_NUMBER_FP16);
@@ -287,11 +287,11 @@ DynamicQuantRegbasePerChannnelFullLoad<xDtype, yDtype, hasSmooth, isSymmetrical>
287 preg0 = MicroAPI::UpdateMask<float>(sregN);287 preg0 = MicroAPI::UpdateMask<float>(sregN);
288 MicroAPI::Duplicate<float>(vregColMax, NEG_INFINITY, preg0);288 MicroAPI::Duplicate<float>(vregColMax, NEG_INFINITY, preg0);
289 for (uint16_t j = 0; j < mLoopNum; j++) {289 for (uint16_t j = 0; j < mLoopNum; j++) {
290- MicroAPI::DataCopy<xDtype, MicroAPI::LoadDist::DIST_UNPACK_B16>(290+ MicroAPI::LoadAlign<xDtype, MicroAPI::LoadDist::DIST_UNPACK_B16>(
291 vregIn, (__ubuf__ xDtype*)(inAddr + i * REG_LEN + j * nSize + bIdx * mLen_ * nSize));291 vregIn, (__ubuf__ xDtype*)(inAddr + i * REG_LEN + j * nSize + bIdx * mLen_ * nSize));
292 MicroAPI::Cast<float, xDtype, castTraitB16ToB32>(vregInFp32, vregIn, preg0);292 MicroAPI::Cast<float, xDtype, castTraitB16ToB32>(vregInFp32, vregIn, preg0);
293 if constexpr (hasSmooth) {293 if constexpr (hasSmooth) {
294- MicroAPI::DataCopy<xDtype, MicroAPI::LoadDist::DIST_BRC_B16>(294+ MicroAPI::LoadAlign<xDtype, MicroAPI::LoadDist::DIST_BRC_B16>(
295 vregSmooth, (__ubuf__ xDtype*)(smoothAddr + j));295 vregSmooth, (__ubuf__ xDtype*)(smoothAddr + j));
296 MicroAPI::Cast<float, xDtype, castTraitB16ToB32>(vregSmoothFp32, vregSmooth, preg0);296 MicroAPI::Cast<float, xDtype, castTraitB16ToB32>(vregSmoothFp32, vregSmooth, preg0);
297 MicroAPI::Mul<float>(vregInFp32, vregInFp32, vregSmoothFp32, preg0);297 MicroAPI::Mul<float>(vregInFp32, vregInFp32, vregSmoothFp32, preg0);
@@ -302,11 +302,11 @@ DynamicQuantRegbasePerChannnelFullLoad<xDtype, yDtype, hasSmooth, isSymmetrical>
302 MicroAPI::Mul(vregOutScale, vregColMax, vregMaxFactor, preg0);302 MicroAPI::Mul(vregOutScale, vregColMax, vregMaxFactor, preg0);
303 for (uint16_t k = 0; k < mLoopNum; k++) {303 for (uint16_t k = 0; k < mLoopNum; k++) {
304 auto addr = yAddr + i * REG_LEN + (bIdx * mLoopNum + k) * nSizeOut;304 auto addr = yAddr + i * REG_LEN + (bIdx * mLoopNum + k) * nSizeOut;
305- MicroAPI::DataCopy<xDtype, MicroAPI::LoadDist::DIST_UNPACK_B16>(305+ MicroAPI::LoadAlign<xDtype, MicroAPI::LoadDist::DIST_UNPACK_B16>(
306 vregIn, (__ubuf__ xDtype*)(inAddr + i * REG_LEN + k * nSize + bIdx * mLen_ * nSize));306 vregIn, (__ubuf__ xDtype*)(inAddr + i * REG_LEN + k * nSize + bIdx * mLen_ * nSize));
307 MicroAPI::Cast<float, xDtype, castTraitB16ToB32>(vregInFp32, vregIn, preg0);307 MicroAPI::Cast<float, xDtype, castTraitB16ToB32>(vregInFp32, vregIn, preg0);
308 if constexpr (hasSmooth) {308 if constexpr (hasSmooth) {
309- MicroAPI::DataCopy<xDtype, MicroAPI::LoadDist::DIST_BRC_B16>(309+ MicroAPI::LoadAlign<xDtype, MicroAPI::LoadDist::DIST_BRC_B16>(
310 vregSmooth, (__ubuf__ xDtype*)(smoothAddr + k));310 vregSmooth, (__ubuf__ xDtype*)(smoothAddr + k));
311 MicroAPI::Cast<float, xDtype, castTraitB16ToB32>(vregSmoothFp32, vregSmooth, preg0);311 MicroAPI::Cast<float, xDtype, castTraitB16ToB32>(vregSmoothFp32, vregSmooth, preg0);
312 MicroAPI::Mul<float>(vregInFp32, vregInFp32, vregSmoothFp32, preg0);312 MicroAPI::Mul<float>(vregInFp32, vregInFp32, vregSmoothFp32, preg0);
@@ -315,13 +315,13 @@ DynamicQuantRegbasePerChannnelFullLoad<xDtype, yDtype, hasSmooth, isSymmetrical>
315 CastToDstType<yDtype, yCopyDtype>(vregOutFp32, vregOut, preg0);315 CastToDstType<yDtype, yCopyDtype>(vregOutFp32, vregOut, preg0);
316 if constexpr (IsSameType<yDtype, int4b_t>::value) {316 if constexpr (IsSameType<yDtype, int4b_t>::value) {
317 addr = yAddr + (i * REG_LEN + (bIdx * mLen_ + k) * nSizeOut) / 2;317 addr = yAddr + (i * REG_LEN + (bIdx * mLen_ + k) * nSizeOut) / 2;
318- MicroAPI::DataCopy<yCopyDtype, MicroAPI::StoreDist::DIST_PACK4_B32>(addr, vregOut, pregH);318+ MicroAPI::StoreAlign<yCopyDtype, MicroAPI::StoreDist::DIST_PACK4_B32>(addr, vregOut, pregH);
319 } else {319 } else {
320- MicroAPI::DataCopy<yCopyDtype, MicroAPI::StoreDist::DIST_PACK4_B32>(addr, vregOut, preg0);320+ MicroAPI::StoreAlign<yCopyDtype, MicroAPI::StoreDist::DIST_PACK4_B32>(addr, vregOut, preg0);
321 }321 }
322 }322 }
323- MicroAPI::DataCopy<float>((__ubuf__ float*)(scaleAddr + i * REG_LEN + bIdx * nSizeScale), vregOutScale,323+ MicroAPI::StoreAlign<float>((__ubuf__ float*)(scaleAddr + i * REG_LEN + bIdx * nSizeScale),
324- preg0);324+ vregOutScale, preg0);
325 }325 }
326 }326 }
327 }327 }
@@ -330,8 +330,8 @@ DynamicQuantRegbasePerChannnelFullLoad<xDtype, yDtype, hasSmooth, isSymmetrical>
330template <typename xDtype, typename yDtype, bool hasSmooth, bool isSymmetrical>330template <typename xDtype, typename yDtype, bool hasSmooth, bool isSymmetrical>
331__aicore__ inline void331__aicore__ inline void
332DynamicQuantRegbasePerChannnelFullLoad<xDtype, yDtype, hasSmooth, isSymmetrical>::ComputeVFforNoSymmetric(332DynamicQuantRegbasePerChannnelFullLoad<xDtype, yDtype, hasSmooth, isSymmetrical>::ComputeVFforNoSymmetric(
333- __local_mem__ xDtype* inAddr, __local_mem__ xDtype* smoothAddr, __local_mem__ yCopyDtype* yAddr,333+ __ubuf__ xDtype* inAddr, __ubuf__ xDtype* smoothAddr, __ubuf__ yCopyDtype* yAddr, __ubuf__ float* scaleAddr,
334- __local_mem__ float* scaleAddr, __local_mem__ float* offsetAddr, uint32_t bBlockSize, uint32_t nBlockSize)334+ __ubuf__ float* offsetAddr, uint32_t bBlockSize, uint32_t nBlockSize)
335{335{
336 uint32_t nSizeScale = ops::CeilAlign(nBlockSize, ALIGN_NUMBER_FP32);336 uint32_t nSizeScale = ops::CeilAlign(nBlockSize, ALIGN_NUMBER_FP32);
337 uint32_t nSize = ops::CeilAlign(nBlockSize, ALIGN_NUMBER_FP16);337 uint32_t nSize = ops::CeilAlign(nBlockSize, ALIGN_NUMBER_FP16);
@@ -376,11 +376,11 @@ DynamicQuantRegbasePerChannnelFullLoad<xDtype, yDtype, hasSmooth, isSymmetrical>
376 MicroAPI::Duplicate<float>(vregColMax, NEG_INFINITY, preg0);376 MicroAPI::Duplicate<float>(vregColMax, NEG_INFINITY, preg0);
377 MicroAPI::Duplicate<float>(vregColMin, POS_INFINITY, preg0);377 MicroAPI::Duplicate<float>(vregColMin, POS_INFINITY, preg0);
378 for (uint16_t j = 0; j < mLoopNum; j++) {378 for (uint16_t j = 0; j < mLoopNum; j++) {
379- MicroAPI::DataCopy<xDtype, MicroAPI::LoadDist::DIST_UNPACK_B16>(379+ MicroAPI::LoadAlign<xDtype, MicroAPI::LoadDist::DIST_UNPACK_B16>(
380 vregIn, (__ubuf__ xDtype*)(inAddr + i * REG_LEN + (j + bIdx * mLen_) * nSize));380 vregIn, (__ubuf__ xDtype*)(inAddr + i * REG_LEN + (j + bIdx * mLen_) * nSize));
381 MicroAPI::Cast<float, xDtype, castTraitB16ToB32>(vregInFp32, vregIn, preg0);381 MicroAPI::Cast<float, xDtype, castTraitB16ToB32>(vregInFp32, vregIn, preg0);
382 if constexpr (hasSmooth) {382 if constexpr (hasSmooth) {
383- MicroAPI::DataCopy<xDtype, MicroAPI::LoadDist::DIST_BRC_B16>(383+ MicroAPI::LoadAlign<xDtype, MicroAPI::LoadDist::DIST_BRC_B16>(
384 vregSmooth, (__ubuf__ xDtype*)(smoothAddr + j));384 vregSmooth, (__ubuf__ xDtype*)(smoothAddr + j));
385 MicroAPI::Cast<float, xDtype, castTraitB16ToB32>(vregSmoothFp32, vregSmooth, preg0);385 MicroAPI::Cast<float, xDtype, castTraitB16ToB32>(vregSmoothFp32, vregSmooth, preg0);
386 MicroAPI::Mul<float>(vregInFp32, vregInFp32, vregSmoothFp32, preg0);386 MicroAPI::Mul<float>(vregInFp32, vregInFp32, vregSmoothFp32, preg0);
@@ -395,11 +395,11 @@ DynamicQuantRegbasePerChannnelFullLoad<xDtype, yDtype, hasSmooth, isSymmetrical>
395 395 
396 for (uint16_t k = 0; k < mLoopNum; k++) {396 for (uint16_t k = 0; k < mLoopNum; k++) {
397 auto addr = yAddr + i * REG_LEN + (k + bIdx * mLen_) * nSizeOut;397 auto addr = yAddr + i * REG_LEN + (k + bIdx * mLen_) * nSizeOut;
398- MicroAPI::DataCopy<xDtype, MicroAPI::LoadDist::DIST_UNPACK_B16>(398+ MicroAPI::LoadAlign<xDtype, MicroAPI::LoadDist::DIST_UNPACK_B16>(
399 vregIn, (__ubuf__ xDtype*)(inAddr + i * REG_LEN + (k + bIdx * mLen_) * nSize));399 vregIn, (__ubuf__ xDtype*)(inAddr + i * REG_LEN + (k + bIdx * mLen_) * nSize));
400 MicroAPI::Cast<float, xDtype, castTraitB16ToB32>(vregInFp32, vregIn, preg0);400 MicroAPI::Cast<float, xDtype, castTraitB16ToB32>(vregInFp32, vregIn, preg0);
401 if constexpr (hasSmooth) {401 if constexpr (hasSmooth) {
402- MicroAPI::DataCopy<xDtype, MicroAPI::LoadDist::DIST_BRC_B16>(402+ MicroAPI::LoadAlign<xDtype, MicroAPI::LoadDist::DIST_BRC_B16>(
403 vregSmooth, (__ubuf__ xDtype*)(smoothAddr + k));403 vregSmooth, (__ubuf__ xDtype*)(smoothAddr + k));
404 MicroAPI::Cast<float, xDtype, castTraitB16ToB32>(vregSmoothFp32, vregSmooth, preg0);404 MicroAPI::Cast<float, xDtype, castTraitB16ToB32>(vregSmoothFp32, vregSmooth, preg0);
405 MicroAPI::Mul<float>(vregInFp32, vregInFp32, vregSmoothFp32, preg0);405 MicroAPI::Mul<float>(vregInFp32, vregInFp32, vregSmoothFp32, preg0);
@@ -410,15 +410,15 @@ DynamicQuantRegbasePerChannnelFullLoad<xDtype, yDtype, hasSmooth, isSymmetrical>
410 CastToDstType<yDtype, yCopyDtype>(vregOutFp32, vregOut, preg0);410 CastToDstType<yDtype, yCopyDtype>(vregOutFp32, vregOut, preg0);
411 if constexpr (IsSameType<yDtype, int4b_t>::value) {411 if constexpr (IsSameType<yDtype, int4b_t>::value) {
412 addr = yAddr + (i * REG_LEN + (k + bIdx * mLen_) * nSizeOut) / 2;412 addr = yAddr + (i * REG_LEN + (k + bIdx * mLen_) * nSizeOut) / 2;
413- MicroAPI::DataCopy<yCopyDtype, MicroAPI::StoreDist::DIST_PACK4_B32>(addr, vregOut, pregH);413+ MicroAPI::StoreAlign<yCopyDtype, MicroAPI::StoreDist::DIST_PACK4_B32>(addr, vregOut, pregH);
414 } else {414 } else {
415- MicroAPI::DataCopy<yCopyDtype, MicroAPI::StoreDist::DIST_PACK4_B32>(addr, vregOut, preg0);415+ MicroAPI::StoreAlign<yCopyDtype, MicroAPI::StoreDist::DIST_PACK4_B32>(addr, vregOut, preg0);
416 }416 }
417 }417 }
418- MicroAPI::DataCopy<float>((__ubuf__ float*)scaleAddr + i * REG_LEN + bIdx * nSizeScale, vregOutScale,418+ MicroAPI::StoreAlign<float>((__ubuf__ float*)scaleAddr + i * REG_LEN + bIdx * nSizeScale, vregOutScale,
419- preg0);419+ preg0);
420- MicroAPI::DataCopy<float>((__ubuf__ float*)offsetAddr + i * REG_LEN + bIdx * nSizeScale, vregOffset,420+ MicroAPI::StoreAlign<float>((__ubuf__ float*)offsetAddr + i * REG_LEN + bIdx * nSizeScale, vregOffset,
421- preg0);421+ preg0);
422 }422 }
423 }423 }
424 }424 }
Mquant/dynamic_quant/op_kernel/arch35/dynamic_quant_regbase_perchannel_recompute.h+48-49
@@ -152,27 +152,25 @@ private:
152 uint32_t blockColNumForY);152 uint32_t blockColNumForY);
153 153 
154 template <bool isLastBlock>154 template <bool isLastBlock>
155- __aicore__ inline void ComputeMaxVFforSymmetric(__local_mem__ xDtype* xAddr, __local_mem__ xDtype* smoothAddr,155+ __aicore__ inline void ComputeMaxVFforSymmetric(__ubuf__ xDtype* xAddr, __ubuf__ xDtype* smoothAddr,
156- __local_mem__ float* scaleAddr,156+ __ubuf__ float* scaleAddr, __ubuf__ float* colMaxLocalAddr,
157- __local_mem__ float* colMaxLocalAddr, uint32_t blockRowNum,157+ uint32_t blockRowNum, uint32_t blockColNum);
158- uint32_t blockColNum);
159 158 
160 __aicore__ inline void ComputeVFMinMaxforNoSymmetric(__ubuf__ xDtype* inAddr, __ubuf__ xDtype* smoothAddr,159 __aicore__ inline void ComputeVFMinMaxforNoSymmetric(__ubuf__ xDtype* inAddr, __ubuf__ xDtype* smoothAddr,
161 __ubuf__ float* scaleAddr, __ubuf__ float* colMaxLocalAddr,160 __ubuf__ float* scaleAddr, __ubuf__ float* colMaxLocalAddr,
162 __ubuf__ float* colMinLocalAddr, __ubuf__ float* offsetAddr,161 __ubuf__ float* colMinLocalAddr, __ubuf__ float* offsetAddr,
163 uint32_t curBaseM, uint32_t curBlockSize);162 uint32_t curBaseM, uint32_t curBlockSize);
164 163 
165- __aicore__ inline void ComputeVFforSymmetric(__local_mem__ xDtype* xAddr, __local_mem__ xDtype* smoothAddr,164+ __aicore__ inline void ComputeVFforSymmetric(__ubuf__ xDtype* xAddr, __ubuf__ xDtype* smoothAddr,
166- __local_mem__ float* scaleAddr, __local_mem__ float* colMaxLocalAddr,165+ __ubuf__ float* scaleAddr, __ubuf__ float* colMaxLocalAddr,
167- __local_mem__ yCopyDtype* yAddr, uint32_t blockRowNum,166+ __ubuf__ yCopyDtype* yAddr, uint32_t blockRowNum,
168 uint32_t blockColNumForX, uint32_t blockColNumForY);167 uint32_t blockColNumForX, uint32_t blockColNumForY);
169 168 
170- __aicore__ inline void ComputeVFforNoSymmetric(__local_mem__ xDtype* xAddr, __local_mem__ xDtype* smoothAddr,169+ __aicore__ inline void ComputeVFforNoSymmetric(__ubuf__ xDtype* xAddr, __ubuf__ xDtype* smoothAddr,
171- __local_mem__ float* scaleAddr, __local_mem__ float* colMaxLocalAddr,170+ __ubuf__ float* scaleAddr, __ubuf__ float* colMaxLocalAddr,
172- __local_mem__ float* colMinLocalAddr,171+ __ubuf__ float* colMinLocalAddr, __ubuf__ yCopyDtype* yAddr,
173- __local_mem__ yCopyDtype* yAddr, __local_mem__ float* offsetAddr,172+ __ubuf__ float* offsetAddr, uint32_t blockRowNum,
174- uint32_t blockRowNum, uint32_t blockColNumForX,173+ uint32_t blockColNumForX, uint32_t blockColNumForY);
175- uint32_t blockColNumForY);
176 174 
177 __aicore__ inline void CopyOutScaleAndOffset(uint32_t blockColNum, uint64_t scaleOffset);175 __aicore__ inline void CopyOutScaleAndOffset(uint32_t blockColNum, uint64_t scaleOffset);
178 176 
@@ -402,9 +400,8 @@ __aicore__ inline void DynamicQuantRegbasePerChannnelRecompute<xDtype, yDtype, h
402template <typename xDtype, typename yDtype, bool hasSmooth, bool isSymmetrical>400template <typename xDtype, typename yDtype, bool hasSmooth, bool isSymmetrical>
403__aicore__ inline void401__aicore__ inline void
404DynamicQuantRegbasePerChannnelRecompute<xDtype, yDtype, hasSmooth, isSymmetrical>::ComputeVFforSymmetric(402DynamicQuantRegbasePerChannnelRecompute<xDtype, yDtype, hasSmooth, isSymmetrical>::ComputeVFforSymmetric(
405- __local_mem__ xDtype* xAddr, __local_mem__ xDtype* smoothAddr, __local_mem__ float* scaleAddr,403+ __ubuf__ xDtype* xAddr, __ubuf__ xDtype* smoothAddr, __ubuf__ float* scaleAddr, __ubuf__ float* colMaxLocalAddr,
406- __local_mem__ float* colMaxLocalAddr, __local_mem__ yCopyDtype* yAddr, uint32_t blockRowNum,404+ __ubuf__ yCopyDtype* yAddr, uint32_t blockRowNum, uint32_t blockColNumForX, uint32_t blockColNumForY)
407- uint32_t blockColNumForX, uint32_t blockColNumForY)
408{405{
409 constexpr uint16_t elementNumPerLoop = FP32_VF_ALIGNED_NUM; // 从UB一次读入VF的元素个数406 constexpr uint16_t elementNumPerLoop = FP32_VF_ALIGNED_NUM; // 从UB一次读入VF的元素个数
410 uint16_t nLoopNum = ops::CeilDiv(blockColNumForX, static_cast<uint32_t>(elementNumPerLoop));407 uint16_t nLoopNum = ops::CeilDiv(blockColNumForX, static_cast<uint32_t>(elementNumPerLoop));
@@ -432,15 +429,15 @@ DynamicQuantRegbasePerChannnelRecompute<xDtype, yDtype, hasSmooth, isSymmetrical
432 mask = MicroAPI::UpdateMask<float>(currColNum);429 mask = MicroAPI::UpdateMask<float>(currColNum);
433 // 从ub中读入colMax430 // 从ub中读入colMax
434 // 插入ub同步，直接读取ub中存储的max和scale431 // 插入ub同步，直接读取ub中存储的max和scale
435- MicroAPI::DataCopy<float>(vregInScale, (__ubuf__ float*)(scaleAddr + nIdx * elementNumPerLoop));432+ MicroAPI::LoadAlign<float>(vregInScale, (__ubuf__ float*)(scaleAddr + nIdx * elementNumPerLoop));
436 for (uint16_t mIdx = 0; mIdx < mLoopNum; mIdx++) {433 for (uint16_t mIdx = 0; mIdx < mLoopNum; mIdx++) {
437 auto addr = yAddr + mIdx * blockColNumForY + nIdx * elementNumPerLoop;434 auto addr = yAddr + mIdx * blockColNumForY + nIdx * elementNumPerLoop;
438- MicroAPI::DataCopy<xDtype, MicroAPI::LoadDist::DIST_UNPACK_B16>(435+ MicroAPI::LoadAlign<xDtype, MicroAPI::LoadDist::DIST_UNPACK_B16>(
439 vregIn, (__ubuf__ xDtype*)(xAddr + mIdx * blockColNumForX + nIdx * elementNumPerLoop));436 vregIn, (__ubuf__ xDtype*)(xAddr + mIdx * blockColNumForX + nIdx * elementNumPerLoop));
440 MicroAPI::Cast<float, xDtype, castTraitB16ToB32>(vregInFp32, vregIn, mask);437 MicroAPI::Cast<float, xDtype, castTraitB16ToB32>(vregInFp32, vregIn, mask);
441 if constexpr (hasSmooth) {438 if constexpr (hasSmooth) {
442- MicroAPI::DataCopy<xDtype, MicroAPI::LoadDist::DIST_BRC_B16>(vregSmooth,439+ MicroAPI::LoadAlign<xDtype, MicroAPI::LoadDist::DIST_BRC_B16>(
443- (__ubuf__ xDtype*)(smoothAddr + mIdx));440+ vregSmooth, (__ubuf__ xDtype*)(smoothAddr + mIdx));
444 MicroAPI::Cast<float, xDtype, castTraitB16ToB32>(vregSmoothFp32, vregSmooth, mask);441 MicroAPI::Cast<float, xDtype, castTraitB16ToB32>(vregSmoothFp32, vregSmooth, mask);
445 MicroAPI::Mul<float>(vregInFp32, vregInFp32, vregSmoothFp32, mask);442 MicroAPI::Mul<float>(vregInFp32, vregInFp32, vregSmoothFp32, mask);
446 }443 }
@@ -449,9 +446,9 @@ DynamicQuantRegbasePerChannnelRecompute<xDtype, yDtype, hasSmooth, isSymmetrical
449 CastToDstType<yDtype, yCopyDtype>(vregOutFp32, vregOut, mask);446 CastToDstType<yDtype, yCopyDtype>(vregOutFp32, vregOut, mask);
450 if constexpr (IsSameType<yDtype, int4b_t>::value) {447 if constexpr (IsSameType<yDtype, int4b_t>::value) {
451 addr = yAddr + (mIdx * blockColNumForY + nIdx * elementNumPerLoop) / 2;448 addr = yAddr + (mIdx * blockColNumForY + nIdx * elementNumPerLoop) / 2;
452- MicroAPI::DataCopy<yCopyDtype, MicroAPI::StoreDist::DIST_PACK4_B32>(addr, vregOut, pregH);449+ MicroAPI::StoreAlign<yCopyDtype, MicroAPI::StoreDist::DIST_PACK4_B32>(addr, vregOut, pregH);
453 } else {450 } else {
454- MicroAPI::DataCopy<yCopyDtype, MicroAPI::StoreDist::DIST_PACK4_B32>(addr, vregOut, mask);451+ MicroAPI::StoreAlign<yCopyDtype, MicroAPI::StoreDist::DIST_PACK4_B32>(addr, vregOut, mask);
455 }452 }
456 }453 }
457 }454 }
@@ -462,8 +459,8 @@ template <typename xDtype, typename yDtype, bool hasSmooth, bool isSymmetrical>
462template <bool isLastBlock>459template <bool isLastBlock>
463__aicore__ inline void460__aicore__ inline void
464DynamicQuantRegbasePerChannnelRecompute<xDtype, yDtype, hasSmooth, isSymmetrical>::ComputeMaxVFforSymmetric(461DynamicQuantRegbasePerChannnelRecompute<xDtype, yDtype, hasSmooth, isSymmetrical>::ComputeMaxVFforSymmetric(
465- __local_mem__ xDtype* xAddr, __local_mem__ xDtype* smoothAddr, __local_mem__ float* scaleAddr,462+ __ubuf__ xDtype* xAddr, __ubuf__ xDtype* smoothAddr, __ubuf__ float* scaleAddr, __ubuf__ float* colMaxLocalAddr,
466- __local_mem__ float* colMaxLocalAddr, uint32_t blockRowNum, uint32_t blockColNum)463+ uint32_t blockRowNum, uint32_t blockColNum)
467{464{
468 constexpr uint16_t elementNumPerLoop = FP32_VF_ALIGNED_NUM; // 从UB一次读入VF的元素个数465 constexpr uint16_t elementNumPerLoop = FP32_VF_ALIGNED_NUM; // 从UB一次读入VF的元素个数
469 uint16_t nLoopNum = ops::CeilDiv(blockColNum, static_cast<uint32_t>(elementNumPerLoop));466 uint16_t nLoopNum = ops::CeilDiv(blockColNum, static_cast<uint32_t>(elementNumPerLoop));
@@ -486,14 +483,14 @@ DynamicQuantRegbasePerChannnelRecompute<xDtype, yDtype, hasSmooth, isSymmetrical
486 // 从ub中读入colMax483 // 从ub中读入colMax
487 // 插入ub同步484 // 插入ub同步
488 MicroAPI::LocalMemBar<MicroAPI::MemType::VEC_STORE, MicroAPI::MemType::VEC_LOAD>();485 MicroAPI::LocalMemBar<MicroAPI::MemType::VEC_STORE, MicroAPI::MemType::VEC_LOAD>();
489- MicroAPI::DataCopy<float>(vregColMax, (__ubuf__ float*)(colMaxLocalAddr + nIdx * elementNumPerLoop));486+ MicroAPI::LoadAlign<float>(vregColMax, (__ubuf__ float*)(colMaxLocalAddr + nIdx * elementNumPerLoop));
490 for (uint16_t mIdx = 0; mIdx < mLoopNum; mIdx++) {487 for (uint16_t mIdx = 0; mIdx < mLoopNum; mIdx++) {
491- MicroAPI::DataCopy<xDtype, MicroAPI::LoadDist::DIST_UNPACK_B16>(488+ MicroAPI::LoadAlign<xDtype, MicroAPI::LoadDist::DIST_UNPACK_B16>(
492 vregIn, (__ubuf__ xDtype*)(xAddr + mIdx * blockColNum + nIdx * elementNumPerLoop));489 vregIn, (__ubuf__ xDtype*)(xAddr + mIdx * blockColNum + nIdx * elementNumPerLoop));
493 MicroAPI::Cast<float, xDtype, castTraitB16ToB32>(vregInFp32, vregIn, mask);490 MicroAPI::Cast<float, xDtype, castTraitB16ToB32>(vregInFp32, vregIn, mask);
494 if constexpr (hasSmooth) {491 if constexpr (hasSmooth) {
495- MicroAPI::DataCopy<xDtype, MicroAPI::LoadDist::DIST_BRC_B16>(vregSmooth,492+ MicroAPI::LoadAlign<xDtype, MicroAPI::LoadDist::DIST_BRC_B16>(
496- (__ubuf__ xDtype*)(smoothAddr + mIdx));493+ vregSmooth, (__ubuf__ xDtype*)(smoothAddr + mIdx));
497 MicroAPI::Cast<float, xDtype, castTraitB16ToB32>(vregSmoothFp32, vregSmooth, mask);494 MicroAPI::Cast<float, xDtype, castTraitB16ToB32>(vregSmoothFp32, vregSmooth, mask);
498 MicroAPI::Mul<float>(vregInFp32, vregInFp32, vregSmoothFp32, mask);495 MicroAPI::Mul<float>(vregInFp32, vregInFp32, vregSmoothFp32, mask);
499 }496 }
@@ -501,10 +498,10 @@ DynamicQuantRegbasePerChannnelRecompute<xDtype, yDtype, hasSmooth, isSymmetrical
501 MicroAPI::Max<float>(vregColMax, vregAbs, vregColMax, mask);498 MicroAPI::Max<float>(vregColMax, vregAbs, vregColMax, mask);
502 }499 }
503 // 局部colMax写入ub，为后续计算做准备500 // 局部colMax写入ub，为后续计算做准备
504- MicroAPI::DataCopy<float>((__ubuf__ float*)colMaxLocalAddr + nIdx * elementNumPerLoop, vregColMax, mask);501+ MicroAPI::StoreAlign<float>((__ubuf__ float*)colMaxLocalAddr + nIdx * elementNumPerLoop, vregColMax, mask);
505 if constexpr (isLastBlock) {502 if constexpr (isLastBlock) {
506 MicroAPI::Muls(vregColMax, vregColMax, scalar, mask);503 MicroAPI::Muls(vregColMax, vregColMax, scalar, mask);
507- MicroAPI::DataCopy<float>((__ubuf__ float*)scaleAddr + nIdx * elementNumPerLoop, vregColMax, mask);504+ MicroAPI::StoreAlign<float>((__ubuf__ float*)scaleAddr + nIdx * elementNumPerLoop, vregColMax, mask);
508 }505 }
509 }506 }
510 }507 }
@@ -536,23 +533,25 @@ DynamicQuantRegbasePerChannnelRecompute<xDtype, yDtype, hasSmooth, isSymmetrical
536 for (uint16_t nIdxvf = 0; nIdxvf < nLoopNum; nIdxvf++) {533 for (uint16_t nIdxvf = 0; nIdxvf < nLoopNum; nIdxvf++) {
537 preg0 = MicroAPI::UpdateMask<float>(count);534 preg0 = MicroAPI::UpdateMask<float>(count);
538 MicroAPI::LocalMemBar<MicroAPI::MemType::VEC_STORE, MicroAPI::MemType::VEC_LOAD>();535 MicroAPI::LocalMemBar<MicroAPI::MemType::VEC_STORE, MicroAPI::MemType::VEC_LOAD>();
539- MicroAPI::DataCopy<float>(vregColMax, (__ubuf__ float*)(colMaxLocalAddr + nIdxvf * elementNumPerLoop));536+ MicroAPI::LoadAlign<float>(vregColMax, (__ubuf__ float*)(colMaxLocalAddr + nIdxvf * elementNumPerLoop));
540- MicroAPI::DataCopy<float>(vregColMin, (__ubuf__ float*)(colMinLocalAddr + nIdxvf * elementNumPerLoop));537+ MicroAPI::LoadAlign<float>(vregColMin, (__ubuf__ float*)(colMinLocalAddr + nIdxvf * elementNumPerLoop));
541 for (uint16_t mIdx = 0; mIdx < mLoopNum; mIdx++) {538 for (uint16_t mIdx = 0; mIdx < mLoopNum; mIdx++) {
542- MicroAPI::DataCopy<xDtype, MicroAPI::LoadDist::DIST_UNPACK_B16>(539+ MicroAPI::LoadAlign<xDtype, MicroAPI::LoadDist::DIST_UNPACK_B16>(
543 vregIn, (__ubuf__ xDtype*)(inAddr + mIdx * curBaseNAligned + nIdxvf * elementNumPerLoop));540 vregIn, (__ubuf__ xDtype*)(inAddr + mIdx * curBaseNAligned + nIdxvf * elementNumPerLoop));
544 MicroAPI::Cast<float, xDtype, castTraitB16ToB32>(vregInFp32, vregIn, preg0);541 MicroAPI::Cast<float, xDtype, castTraitB16ToB32>(vregInFp32, vregIn, preg0);
545 if constexpr (hasSmooth) {542 if constexpr (hasSmooth) {
546- MicroAPI::DataCopy<xDtype, MicroAPI::LoadDist::DIST_BRC_B16>(vregSmooth,543+ MicroAPI::LoadAlign<xDtype, MicroAPI::LoadDist::DIST_BRC_B16>(
547- (__ubuf__ xDtype*)(smoothAddr + mIdx));544+ vregSmooth, (__ubuf__ xDtype*)(smoothAddr + mIdx));
548 MicroAPI::Cast<float, xDtype, castTraitB16ToB32>(vregSmoothFp32, vregSmooth, preg0);545 MicroAPI::Cast<float, xDtype, castTraitB16ToB32>(vregSmoothFp32, vregSmooth, preg0);
549 MicroAPI::Mul<float>(vregInFp32, vregInFp32, vregSmoothFp32, preg0);546 MicroAPI::Mul<float>(vregInFp32, vregInFp32, vregSmoothFp32, preg0);
550 }547 }
551 MicroAPI::Max<float>(vregColMax, vregInFp32, vregColMax, preg0); // max(x)548 MicroAPI::Max<float>(vregColMax, vregInFp32, vregColMax, preg0); // max(x)
552 MicroAPI::Min<float>(vregColMin, vregInFp32, vregColMin, preg0); // min(x)549 MicroAPI::Min<float>(vregColMin, vregInFp32, vregColMin, preg0); // min(x)
553 }550 }
554- MicroAPI::DataCopy<float>((__ubuf__ float*)colMaxLocalAddr + nIdxvf * elementNumPerLoop, vregColMax, preg0);551+ MicroAPI::StoreAlign<float>((__ubuf__ float*)colMaxLocalAddr + nIdxvf * elementNumPerLoop, vregColMax,
555- MicroAPI::DataCopy<float>((__ubuf__ float*)colMinLocalAddr + nIdxvf * elementNumPerLoop, vregColMin, preg0);552+ preg0);
553+ MicroAPI::StoreAlign<float>((__ubuf__ float*)colMinLocalAddr + nIdxvf * elementNumPerLoop, vregColMin,
554+ preg0);
556 }555 }
557 }556 }
558}557}
@@ -560,9 +559,9 @@ DynamicQuantRegbasePerChannnelRecompute<xDtype, yDtype, hasSmooth, isSymmetrical
560template <typename xDtype, typename yDtype, bool hasSmooth, bool isSymmetrical>559template <typename xDtype, typename yDtype, bool hasSmooth, bool isSymmetrical>
561__aicore__ inline void560__aicore__ inline void
562DynamicQuantRegbasePerChannnelRecompute<xDtype, yDtype, hasSmooth, isSymmetrical>::ComputeVFforNoSymmetric(561DynamicQuantRegbasePerChannnelRecompute<xDtype, yDtype, hasSmooth, isSymmetrical>::ComputeVFforNoSymmetric(
563- __local_mem__ xDtype* xAddr, __local_mem__ xDtype* smoothAddr, __local_mem__ float* scaleAddr,562+ __ubuf__ xDtype* xAddr, __ubuf__ xDtype* smoothAddr, __ubuf__ float* scaleAddr, __ubuf__ float* colMaxLocalAddr,
564- __local_mem__ float* colMaxLocalAddr, __local_mem__ float* colMinLocalAddr, __local_mem__ yCopyDtype* yAddr,563+ __ubuf__ float* colMinLocalAddr, __ubuf__ yCopyDtype* yAddr, __ubuf__ float* offsetAddr, uint32_t blockRowNum,
565- __local_mem__ float* offsetAddr, uint32_t blockRowNum, uint32_t blockColNumForX, uint32_t blockColNumForY)564+ uint32_t blockColNumForX, uint32_t blockColNumForY)
566{565{
567 // 一个vf里面最多只能容纳256个字节566 // 一个vf里面最多只能容纳256个字节
568 uint16_t elementNumPerLoop = FP32_VF_ALIGNED_NUM;567 uint16_t elementNumPerLoop = FP32_VF_ALIGNED_NUM;
@@ -609,8 +608,8 @@ DynamicQuantRegbasePerChannnelRecompute<xDtype, yDtype, hasSmooth, isSymmetrical
609 for (uint16_t nIdxvf = 0; nIdxvf < nLoopNum; nIdxvf++) {608 for (uint16_t nIdxvf = 0; nIdxvf < nLoopNum; nIdxvf++) {
610 preg0 = MicroAPI::UpdateMask<float>(count);609 preg0 = MicroAPI::UpdateMask<float>(count);
611 // 将min max scale offset拷贝进来610 // 将min max scale offset拷贝进来
612- MicroAPI::DataCopy<float>(vregColMax, (__ubuf__ float*)(colMaxLocalAddr + nIdxvf * elementNumPerLoop));611+ MicroAPI::LoadAlign<float>(vregColMax, (__ubuf__ float*)(colMaxLocalAddr + nIdxvf * elementNumPerLoop));
613- MicroAPI::DataCopy<float>(vregColMin, (__ubuf__ float*)(colMinLocalAddr + nIdxvf * elementNumPerLoop));612+ MicroAPI::LoadAlign<float>(vregColMin, (__ubuf__ float*)(colMinLocalAddr + nIdxvf * elementNumPerLoop));
614 613 
615 // scaleout614 // scaleout
616 MicroAPI::Sub<float>(vregSub, vregColMax, vregColMin, preg0); // max(x)-min(x)615 MicroAPI::Sub<float>(vregSub, vregColMax, vregColMin, preg0); // max(x)-min(x)
@@ -623,12 +622,12 @@ DynamicQuantRegbasePerChannnelRecompute<xDtype, yDtype, hasSmooth, isSymmetrical
623 // y622 // y
624 for (uint16_t mIdx = 0; mIdx < mLoopNum; mIdx++) {623 for (uint16_t mIdx = 0; mIdx < mLoopNum; mIdx++) {
625 auto addr = yAddr + mIdx * curBaseNAlignedB8 + nIdxvf * elementNumPerLoop;624 auto addr = yAddr + mIdx * curBaseNAlignedB8 + nIdxvf * elementNumPerLoop;
626- MicroAPI::DataCopy<xDtype, MicroAPI::LoadDist::DIST_UNPACK_B16>(625+ MicroAPI::LoadAlign<xDtype, MicroAPI::LoadDist::DIST_UNPACK_B16>(
627 vregIn, (__ubuf__ xDtype*)(xAddr + mIdx * curBaseNAligned + nIdxvf * elementNumPerLoop));626 vregIn, (__ubuf__ xDtype*)(xAddr + mIdx * curBaseNAligned + nIdxvf * elementNumPerLoop));
628 MicroAPI::Cast<float, xDtype, castTraitB16ToB32>(vregInFp32, vregIn, preg0);627 MicroAPI::Cast<float, xDtype, castTraitB16ToB32>(vregInFp32, vregIn, preg0);
629 if constexpr (hasSmooth) {628 if constexpr (hasSmooth) {
630- MicroAPI::DataCopy<xDtype, MicroAPI::LoadDist::DIST_BRC_B16>(vregSmooth,629+ MicroAPI::LoadAlign<xDtype, MicroAPI::LoadDist::DIST_BRC_B16>(
631- (__ubuf__ xDtype*)(smoothAddr + mIdx));630+ vregSmooth, (__ubuf__ xDtype*)(smoothAddr + mIdx));
632 MicroAPI::Cast<float, xDtype, castTraitB16ToB32>(vregSmoothFp32, vregSmooth, preg0);631 MicroAPI::Cast<float, xDtype, castTraitB16ToB32>(vregSmoothFp32, vregSmooth, preg0);
633 MicroAPI::Mul<float>(vregInFp32, vregInFp32, vregSmoothFp32, preg0);632 MicroAPI::Mul<float>(vregInFp32, vregInFp32, vregSmoothFp32, preg0);
634 }633 }
@@ -637,13 +636,13 @@ DynamicQuantRegbasePerChannnelRecompute<xDtype, yDtype, hasSmooth, isSymmetrical
637 CastToDstType<yDtype, yCopyDtype>(vregOutFp32, vregOut, preg0);636 CastToDstType<yDtype, yCopyDtype>(vregOutFp32, vregOut, preg0);
638 if constexpr (IsSameType<yDtype, int4b_t>::value) {637 if constexpr (IsSameType<yDtype, int4b_t>::value) {
639 addr = yAddr + (mIdx * curBaseNAlignedB8 + nIdxvf * elementNumPerLoop) / 2;638 addr = yAddr + (mIdx * curBaseNAlignedB8 + nIdxvf * elementNumPerLoop) / 2;
640- MicroAPI::DataCopy<yCopyDtype, MicroAPI::StoreDist::DIST_PACK4_B32>(addr, vregOut, pregH);639+ MicroAPI::StoreAlign<yCopyDtype, MicroAPI::StoreDist::DIST_PACK4_B32>(addr, vregOut, pregH);
641 } else {640 } else {
642- MicroAPI::DataCopy<yCopyDtype, MicroAPI::StoreDist::DIST_PACK4_B32>(addr, vregOut, preg0);641+ MicroAPI::StoreAlign<yCopyDtype, MicroAPI::StoreDist::DIST_PACK4_B32>(addr, vregOut, preg0);
643 }642 }
644 }643 }
645- MicroAPI::DataCopy<float>((__ubuf__ float*)scaleAddr + nIdxvf * elementNumPerLoop, vregScale, preg0);644+ MicroAPI::StoreAlign<float>((__ubuf__ float*)scaleAddr + nIdxvf * elementNumPerLoop, vregScale, preg0);
646- MicroAPI::DataCopy<float>((__ubuf__ float*)offsetAddr + nIdxvf * elementNumPerLoop, vregOffset, preg0);645+ MicroAPI::StoreAlign<float>((__ubuf__ float*)offsetAddr + nIdxvf * elementNumPerLoop, vregOffset, preg0);
647 }646 }
648 }647 }
649}648}
Mquant/dynamic_quant/op_kernel/arch35/dynamic_quant_regbase_perchannel_split_m.h+23-23
@@ -363,11 +363,11 @@ __aicore__ inline void DynamicQuantRegbasePerChannnelSplitM<xDtype, yDtype, hasS
363 preg0 = MicroAPI::UpdateMask<float>(sregN);363 preg0 = MicroAPI::UpdateMask<float>(sregN);
364 MicroAPI::Duplicate<float>(vregColMax, 0.0f, preg0);364 MicroAPI::Duplicate<float>(vregColMax, 0.0f, preg0);
365 for (uint16_t j = 0; j < mLoopNum; j++) {365 for (uint16_t j = 0; j < mLoopNum; j++) {
366- MicroAPI::DataCopy<xDtype, MicroAPI::LoadDist::DIST_UNPACK_B16>(366+ MicroAPI::LoadAlign<xDtype, MicroAPI::LoadDist::DIST_UNPACK_B16>(
367 vregIn, (__ubuf__ xDtype*)(inAddr + i * vl + j * nSize));367 vregIn, (__ubuf__ xDtype*)(inAddr + i * vl + j * nSize));
368 MicroAPI::Cast<float, xDtype, castTraitB16ToB32>(vregInFp32, vregIn, preg0);368 MicroAPI::Cast<float, xDtype, castTraitB16ToB32>(vregInFp32, vregIn, preg0);
369 if constexpr (hasSmooth) {369 if constexpr (hasSmooth) {
370- MicroAPI::DataCopy<xDtype, MicroAPI::LoadDist::DIST_BRC_B16>(370+ MicroAPI::LoadAlign<xDtype, MicroAPI::LoadDist::DIST_BRC_B16>(
371 vregSmooth, (__ubuf__ xDtype*)(smoothAddr + j));371 vregSmooth, (__ubuf__ xDtype*)(smoothAddr + j));
372 MicroAPI::Cast<float, xDtype, castTraitB16ToB32>(vregSmoothFp32, vregSmooth, preg0);372 MicroAPI::Cast<float, xDtype, castTraitB16ToB32>(vregSmoothFp32, vregSmooth, preg0);
373 MicroAPI::Mul<float>(vregInFp32, vregInFp32, vregSmoothFp32, preg0);373 MicroAPI::Mul<float>(vregInFp32, vregInFp32, vregSmoothFp32, preg0);
@@ -375,7 +375,7 @@ __aicore__ inline void DynamicQuantRegbasePerChannnelSplitM<xDtype, yDtype, hasS
375 MicroAPI::Abs<float>(vregAbs, vregInFp32, preg0);375 MicroAPI::Abs<float>(vregAbs, vregInFp32, preg0);
376 MicroAPI::Max<float>(vregColMax, vregAbs, vregColMax, preg0);376 MicroAPI::Max<float>(vregColMax, vregAbs, vregColMax, preg0);
377 }377 }
378- MicroAPI::DataCopy<float>((__ubuf__ float*)colMaxLocalAddr + i * vl, vregColMax, preg0);378+ MicroAPI::StoreAlign<float>((__ubuf__ float*)colMaxLocalAddr + i * vl, vregColMax, preg0);
379 }379 }
380 } else {380 } else {
381 for (uint16_t i = 0; i < nLoopNum; i++) {381 for (uint16_t i = 0; i < nLoopNum; i++) {
@@ -383,20 +383,20 @@ __aicore__ inline void DynamicQuantRegbasePerChannnelSplitM<xDtype, yDtype, hasS
383 MicroAPI::Duplicate<float>(vregColMax, NEG_INFINITY, preg0);383 MicroAPI::Duplicate<float>(vregColMax, NEG_INFINITY, preg0);
384 MicroAPI::Duplicate<float>(vregColMin, POS_INFINITY, preg0);384 MicroAPI::Duplicate<float>(vregColMin, POS_INFINITY, preg0);
385 for (uint16_t j = 0; j < mLoopNum; j++) {385 for (uint16_t j = 0; j < mLoopNum; j++) {
386- MicroAPI::DataCopy<xDtype, MicroAPI::LoadDist::DIST_UNPACK_B16>(386+ MicroAPI::LoadAlign<xDtype, MicroAPI::LoadDist::DIST_UNPACK_B16>(
387 vregIn, (__ubuf__ xDtype*)(inAddr + i * vl + j * nSize));387 vregIn, (__ubuf__ xDtype*)(inAddr + i * vl + j * nSize));
388 MicroAPI::Cast<float, xDtype, castTraitB16ToB32>(vregInFp32, vregIn, preg0);388 MicroAPI::Cast<float, xDtype, castTraitB16ToB32>(vregInFp32, vregIn, preg0);
389 if constexpr (hasSmooth) {389 if constexpr (hasSmooth) {
390- MicroAPI::DataCopy<xDtype, MicroAPI::LoadDist::DIST_BRC_B16>(vregSmooth,390+ MicroAPI::LoadAlign<xDtype, MicroAPI::LoadDist::DIST_BRC_B16>(vregSmooth,
391- (__ubuf__ xDtype*)smoothAddr + j);391+ (__ubuf__ xDtype*)smoothAddr + j);
392 MicroAPI::Cast<float, xDtype, castTraitB16ToB32>(vregSmoothFp32, vregSmooth, preg0);392 MicroAPI::Cast<float, xDtype, castTraitB16ToB32>(vregSmoothFp32, vregSmooth, preg0);
393 MicroAPI::Mul<float>(vregInFp32, vregInFp32, vregSmoothFp32, preg0);393 MicroAPI::Mul<float>(vregInFp32, vregInFp32, vregSmoothFp32, preg0);
394 }394 }
395 MicroAPI::Max<float>(vregColMax, vregInFp32, vregColMax, preg0);395 MicroAPI::Max<float>(vregColMax, vregInFp32, vregColMax, preg0);
396 MicroAPI::Min<float>(vregColMin, vregInFp32, vregColMin, preg0);396 MicroAPI::Min<float>(vregColMin, vregInFp32, vregColMin, preg0);
397 }397 }
398- MicroAPI::DataCopy<float>((__ubuf__ float*)colMaxLocalAddr + i * vl, vregColMax, preg0);398+ MicroAPI::StoreAlign<float>((__ubuf__ float*)colMaxLocalAddr + i * vl, vregColMax, preg0);
399- MicroAPI::DataCopy<float>((__ubuf__ float*)colMinLocalAddr + i * vl, vregColMin, preg0);399+ MicroAPI::StoreAlign<float>((__ubuf__ float*)colMinLocalAddr + i * vl, vregColMin, preg0);
400 }400 }
401 }401 }
402 }402 }
@@ -499,15 +499,15 @@ __aicore__ inline void DynamicQuantRegbasePerChannnelSplitM<xDtype, yDtype, hasS
499 if constexpr (isSymmetrical) {499 if constexpr (isSymmetrical) {
500 for (uint16_t i = 0; i < nLoopNum; i++) {500 for (uint16_t i = 0; i < nLoopNum; i++) {
501 preg0 = MicroAPI::UpdateMask<float>(sregN);501 preg0 = MicroAPI::UpdateMask<float>(sregN);
502- MicroAPI::DataCopy<float>(vregColMax, (__ubuf__ float*)(colMaxLocalAddr + i * vl));502+ MicroAPI::LoadAlign<float>(vregColMax, (__ubuf__ float*)(colMaxLocalAddr + i * vl));
503 MicroAPI::Mul<float>(vregScale, vregColMax, vregMaxFactor, preg0);503 MicroAPI::Mul<float>(vregScale, vregColMax, vregMaxFactor, preg0);
504 for (uint16_t j = 0; j < mLoopNum; j++) {504 for (uint16_t j = 0; j < mLoopNum; j++) {
505 auto outAddr = yAddr + i * vl + j * nSizeOut;505 auto outAddr = yAddr + i * vl + j * nSizeOut;
506- MicroAPI::DataCopy<xDtype, MicroAPI::LoadDist::DIST_UNPACK_B16>(506+ MicroAPI::LoadAlign<xDtype, MicroAPI::LoadDist::DIST_UNPACK_B16>(
507 vregIn, (__ubuf__ xDtype*)(inAddr + i * vl + j * nSize));507 vregIn, (__ubuf__ xDtype*)(inAddr + i * vl + j * nSize));
508 MicroAPI::Cast<float, xDtype, castTraitB16ToB32>(vregInFp32, vregIn, preg0);508 MicroAPI::Cast<float, xDtype, castTraitB16ToB32>(vregInFp32, vregIn, preg0);
509 if constexpr (hasSmooth) {509 if constexpr (hasSmooth) {
510- MicroAPI::DataCopy<xDtype, MicroAPI::LoadDist::DIST_BRC_B16>(510+ MicroAPI::LoadAlign<xDtype, MicroAPI::LoadDist::DIST_BRC_B16>(
511 vregSmooth, (__ubuf__ xDtype*)(smoothAddr + j));511 vregSmooth, (__ubuf__ xDtype*)(smoothAddr + j));
512 MicroAPI::Cast<float, xDtype, castTraitB16ToB32>(vregSmoothFp32, vregSmooth, preg0);512 MicroAPI::Cast<float, xDtype, castTraitB16ToB32>(vregSmoothFp32, vregSmooth, preg0);
513 MicroAPI::Mul<float>(vregInFp32, vregInFp32, vregSmoothFp32, preg0);513 MicroAPI::Mul<float>(vregInFp32, vregInFp32, vregSmoothFp32, preg0);
@@ -517,31 +517,31 @@ __aicore__ inline void DynamicQuantRegbasePerChannnelSplitM<xDtype, yDtype, hasS
517 CastToDstType<yDtype, yCopyDtype>(vregOutFp32, vregOut, preg0);517 CastToDstType<yDtype, yCopyDtype>(vregOutFp32, vregOut, preg0);
518 if constexpr (IsSameType<yDtype, int4b_t>::value) {518 if constexpr (IsSameType<yDtype, int4b_t>::value) {
519 outAddr = yAddr + (i * vl + j * nSizeOut) / 2;519 outAddr = yAddr + (i * vl + j * nSizeOut) / 2;
520- MicroAPI::DataCopy<yCopyDtype, MicroAPI::StoreDist::DIST_PACK4_B32>(outAddr, vregOut, pregH);520+ MicroAPI::StoreAlign<yCopyDtype, MicroAPI::StoreDist::DIST_PACK4_B32>(outAddr, vregOut, pregH);
521 } else {521 } else {
522- MicroAPI::DataCopy<yCopyDtype, MicroAPI::StoreDist::DIST_PACK4_B32>(outAddr, vregOut, preg0);522+ MicroAPI::StoreAlign<yCopyDtype, MicroAPI::StoreDist::DIST_PACK4_B32>(outAddr, vregOut, preg0);
523 }523 }
524 }524 }
525- MicroAPI::DataCopy<float>((__ubuf__ float*)scaleLocalAddr + i * vl, vregScale, preg0);525+ MicroAPI::StoreAlign<float>((__ubuf__ float*)scaleLocalAddr + i * vl, vregScale, preg0);
526 }526 }
527 } else {527 } else {
528 MicroAPI::Duplicate<float>(vregOffsetVal, offsetValue_, pregAll);528 MicroAPI::Duplicate<float>(vregOffsetVal, offsetValue_, pregAll);
529 MicroAPI::Duplicate<float>(vregOffsetDivVal, offsetDivValue_, pregAll);529 MicroAPI::Duplicate<float>(vregOffsetDivVal, offsetDivValue_, pregAll);
530 for (uint16_t i = 0; i < nLoopNum; i++) {530 for (uint16_t i = 0; i < nLoopNum; i++) {
531 preg0 = MicroAPI::UpdateMask<float>(sregN);531 preg0 = MicroAPI::UpdateMask<float>(sregN);
532- MicroAPI::DataCopy<float>(vregColMax, (__ubuf__ float*)(colMaxLocalAddr + i * vl));532+ MicroAPI::LoadAlign<float>(vregColMax, (__ubuf__ float*)(colMaxLocalAddr + i * vl));
533- MicroAPI::DataCopy<float>(vregColMin, (__ubuf__ float*)(colMinLocalAddr + i * vl));533+ MicroAPI::LoadAlign<float>(vregColMin, (__ubuf__ float*)(colMinLocalAddr + i * vl));
534 MicroAPI::Sub<float>(vregSub, vregColMax, vregColMin, preg0);534 MicroAPI::Sub<float>(vregSub, vregColMax, vregColMin, preg0);
535 MicroAPI::Mul<float>(vregScale, vregSub, vregOffsetDivVal, preg0);535 MicroAPI::Mul<float>(vregScale, vregSub, vregOffsetDivVal, preg0);
536 MicroAPI::Div<float, &divHighPrecisionMode>(vregMaxDivScale, vregColMax, vregScale, preg0);536 MicroAPI::Div<float, &divHighPrecisionMode>(vregMaxDivScale, vregColMax, vregScale, preg0);
537 MicroAPI::Sub<float>(vregOffset, vregOffsetVal, vregMaxDivScale, preg0);537 MicroAPI::Sub<float>(vregOffset, vregOffsetVal, vregMaxDivScale, preg0);
538 for (uint16_t j = 0; j < mLoopNum; j++) {538 for (uint16_t j = 0; j < mLoopNum; j++) {
539 auto outAddr = yAddr + i * vl + j * nSizeOut;539 auto outAddr = yAddr + i * vl + j * nSizeOut;
540- MicroAPI::DataCopy<xDtype, MicroAPI::LoadDist::DIST_UNPACK_B16>(540+ MicroAPI::LoadAlign<xDtype, MicroAPI::LoadDist::DIST_UNPACK_B16>(
541 vregIn, (__ubuf__ xDtype*)inAddr + i * vl + j * nSize);541 vregIn, (__ubuf__ xDtype*)inAddr + i * vl + j * nSize);
542 MicroAPI::Cast<float, xDtype, castTraitB16ToB32>(vregInFp32, vregIn, preg0);542 MicroAPI::Cast<float, xDtype, castTraitB16ToB32>(vregInFp32, vregIn, preg0);
543 if constexpr (hasSmooth) {543 if constexpr (hasSmooth) {
544- MicroAPI::DataCopy<xDtype, MicroAPI::LoadDist::DIST_BRC_B16>(544+ MicroAPI::LoadAlign<xDtype, MicroAPI::LoadDist::DIST_BRC_B16>(
545 vregSmooth, (__ubuf__ xDtype*)(smoothAddr + j));545 vregSmooth, (__ubuf__ xDtype*)(smoothAddr + j));
546 MicroAPI::Cast<float, xDtype, castTraitB16ToB32>(vregSmoothFp32, vregSmooth, preg0);546 MicroAPI::Cast<float, xDtype, castTraitB16ToB32>(vregSmoothFp32, vregSmooth, preg0);
547 MicroAPI::Mul<float>(vregInFp32, vregInFp32, vregSmoothFp32, preg0);547 MicroAPI::Mul<float>(vregInFp32, vregInFp32, vregSmoothFp32, preg0);
@@ -552,13 +552,13 @@ __aicore__ inline void DynamicQuantRegbasePerChannnelSplitM<xDtype, yDtype, hasS
552 CastToDstType<yDtype, yCopyDtype>(vregOutFp32, vregOut, preg0);552 CastToDstType<yDtype, yCopyDtype>(vregOutFp32, vregOut, preg0);
553 if constexpr (IsSameType<yDtype, int4b_t>::value) {553 if constexpr (IsSameType<yDtype, int4b_t>::value) {
554 outAddr = yAddr + (i * vl + j * nSizeOut) / 2;554 outAddr = yAddr + (i * vl + j * nSizeOut) / 2;
555- MicroAPI::DataCopy<yCopyDtype, MicroAPI::StoreDist::DIST_PACK4_B32>(outAddr, vregOut, pregH);555+ MicroAPI::StoreAlign<yCopyDtype, MicroAPI::StoreDist::DIST_PACK4_B32>(outAddr, vregOut, pregH);
556 } else {556 } else {
557- MicroAPI::DataCopy<yCopyDtype, MicroAPI::StoreDist::DIST_PACK4_B32>(outAddr, vregOut, preg0);557+ MicroAPI::StoreAlign<yCopyDtype, MicroAPI::StoreDist::DIST_PACK4_B32>(outAddr, vregOut, preg0);
558 }558 }
559 }559 }
560- MicroAPI::DataCopy<float>((__ubuf__ float*)scaleLocalAddr + i * vl, vregScale, preg0);560+ MicroAPI::StoreAlign<float>((__ubuf__ float*)scaleLocalAddr + i * vl, vregScale, preg0);
561- MicroAPI::DataCopy<float>((__ubuf__ float*)offsetLocalAddr + i * vl, vregOffset, preg0);561+ MicroAPI::StoreAlign<float>((__ubuf__ float*)offsetLocalAddr + i * vl, vregOffset, preg0);
562 }562 }
563 }563 }
564 }564 }
@@ -612,4 +612,4 @@ __aicore__ inline void DynamicQuantRegbasePerChannnelSplitM<xDtype, yDtype, hasS
612}612}
613 613 
614} // namespace DynamicQuantPerChannel614} // namespace DynamicQuantPerChannel
615-#endif // DYNAMIC_QUANT_REGBASE_PERCHANNEL_SPLIT_M_H615+#endif // DYNAMIC_QUANT_REGBASE_PERCHANNEL_SPLIT_M_H
Mquant/dynamic_quant/op_kernel/arch35/dynamic_quant_regbase_pertoken_large_multicore.h+109-115
@@ -62,31 +62,27 @@ private:
62 __aicore__ inline void ProcessScaleRow();62 __aicore__ inline void ProcessScaleRow();
63 __aicore__ inline void ProcessScaleCol();63 __aicore__ inline void ProcessScaleCol();
64 __aicore__ inline void ProcessScaleRowLoop(uint32_t i, uint32_t j);64 __aicore__ inline void ProcessScaleRowLoop(uint32_t i, uint32_t j);
65- __aicore__ inline void ProcessYRow(uint32_t i, uint32_t j, __local_mem__ float* scaleAddr,65+ __aicore__ inline void ProcessYRow(uint32_t i, uint32_t j, __ubuf__ float* scaleAddr, __ubuf__ float* offsetAddr);
66- __local_mem__ float* offsetAddr);
67 __aicore__ inline void CopyInByEle(int64_t offset, uint32_t loopIndex, uint32_t elementNum, uint8_t rightPadding);66 __aicore__ inline void CopyInByEle(int64_t offset, uint32_t loopIndex, uint32_t elementNum, uint8_t rightPadding);
68 __aicore__ inline void CopyInScaleByEle(int64_t offset, uint32_t elementNum);67 __aicore__ inline void CopyInScaleByEle(int64_t offset, uint32_t elementNum);
69 __aicore__ inline void ComputeMaxRowScale(uint32_t elementNum);68 __aicore__ inline void ComputeMaxRowScale(uint32_t elementNum);
70- __aicore__ inline void ComputeMaxColScale(uint32_t elementNum, __local_mem__ float* maxAddr,69+ __aicore__ inline void ComputeMaxColScale(uint32_t elementNum, __ubuf__ float* maxAddr, __ubuf__ float* minAddr,
71- __local_mem__ float* minAddr, __local_mem__ float* scaleAddr,70+ __ubuf__ float* scaleAddr, __ubuf__ float* offsetAddr);
72- __local_mem__ float* offsetAddr);71+ __aicore__ inline void ComputeMaxRowScaleVF(__ubuf__ T* inLocalAddr, __ubuf__ T* smoothLocalAddr,
73- __aicore__ inline void ComputeMaxRowScaleVF(__local_mem__ T* inLocalAddr, __local_mem__ T* smoothLocalAddr,72+ __ubuf__ float* scaleLocalAddr, __ubuf__ float* maxLocalAddr,
74- __local_mem__ float* scaleLocalAddr, __local_mem__ float* maxLocalAddr,73+ __ubuf__ float* minLocalAddr, uint32_t elementNum);
75- __local_mem__ float* minLocalAddr, uint32_t elementNum);74+ __aicore__ inline void ComputeMaxColScaleVF(__ubuf__ float* scaleLocalAddr, __ubuf__ float* scaleOutLocalAddr,
76- __aicore__ inline void ComputeMaxColScaleVF(__local_mem__ float* scaleLocalAddr,75+ __ubuf__ float* maxLocalAddr, __ubuf__ float* maxOutLocalAddr,
77- __local_mem__ float* scaleOutLocalAddr,76+ __ubuf__ float* minLocalAddr, __ubuf__ float* minOutLocalAddr,
78- __local_mem__ float* maxLocalAddr, __local_mem__ float* maxOutLocalAddr,
79- __local_mem__ float* minLocalAddr, __local_mem__ float* minOutLocalAddr,
80 uint32_t elementNum);77 uint32_t elementNum);
81- __aicore__ inline void ComputeY(uint32_t elementNum, __local_mem__ float* scaleAddr,78+ __aicore__ inline void ComputeY(uint32_t elementNum, __ubuf__ float* scaleAddr, __ubuf__ float* offsetAddr);
82- __local_mem__ float* offsetAddr);79+ __aicore__ inline void ComputeScaleSymVF(__ubuf__ float* maxLocalAddr, __ubuf__ float* minLocalAddr,
83- __aicore__ inline void ComputeScaleSymVF(__local_mem__ float* maxLocalAddr, __local_mem__ float* minLocalAddr,80+ __ubuf__ float* scaleLocalAddr, uint32_t elementNum);
84- __local_mem__ float* scaleLocalAddr, uint32_t elementNum);81+ __aicore__ inline void ComputeOffsetSymVF(__ubuf__ float* maxLocalAddr, __ubuf__ float* scaleLocalAddr,
85- __aicore__ inline void ComputeOffsetSymVF(__local_mem__ float* maxLocalAddr, __local_mem__ float* scaleLocalAddr,82+ __ubuf__ float* offsetLocalAddr, uint32_t elementNum);
86- __local_mem__ float* offsetLocalAddr, uint32_t elementNum);83+ __aicore__ inline void ComputeYVF(__ubuf__ T* inLocalAddr, __ubuf__ T* smoothLocalAddr,
87- __aicore__ inline void ComputeYVF(__local_mem__ T* inLocalAddr, __local_mem__ T* smoothLocalAddr,84+ __ubuf__ yCopyDtype* outAddr, __ubuf__ float* scaleLocalAddr,
88- __local_mem__ yCopyDtype* outAddr, __local_mem__ float* scaleLocalAddr,85+ __ubuf__ float* offsetLocalAddr, uint32_t elementNum);
89- __local_mem__ float* offsetLocalAddr, uint32_t elementNum);
90 __aicore__ inline void ParseTilingData(const DynamicQuantTilingDataArch35& tilingData);86 __aicore__ inline void ParseTilingData(const DynamicQuantTilingDataArch35& tilingData);
91 __aicore__ inline void CopyOutY(int64_t offset, uint32_t element);87 __aicore__ inline void CopyOutY(int64_t offset, uint32_t element);
92 __aicore__ inline void CopyUB2Workspace(int64_t size);88 __aicore__ inline void CopyUB2Workspace(int64_t size);
@@ -317,14 +313,14 @@ template <typename T, typename yDtype, int64_t hasSmooth, bool isSymmetrical>
317__aicore__ inline void DynamicQuantPertenLargeMulticore<T, yDtype, hasSmooth, isSymmetrical>::ProcessY()313__aicore__ inline void DynamicQuantPertenLargeMulticore<T, yDtype, hasSmooth, isSymmetrical>::ProcessY()
318{314{
319 LocalTensor<float> scaleOutLocal = scaleOutQueue.DeQue<float>();315 LocalTensor<float> scaleOutLocal = scaleOutQueue.DeQue<float>();
320- __local_mem__ float* scaleOutLocalAddr = (__local_mem__ float*)scaleOutLocal.GetPhyAddr();316+ __ubuf__ float* scaleOutLocalAddr = (__ubuf__ float*)scaleOutLocal.GetPhyAddr();
321 317 
322 LocalTensor<float> offsetLocal;318 LocalTensor<float> offsetLocal;
323- __local_mem__ float* offsetLocalAddr;319+ __ubuf__ float* offsetLocalAddr;
324 320 
325 if constexpr (isSymmetrical == false) {321 if constexpr (isSymmetrical == false) {
326 offsetLocal = offsetQueue.DeQue<float>();322 offsetLocal = offsetQueue.DeQue<float>();
327- offsetLocalAddr = (__local_mem__ float*)offsetLocal.GetPhyAddr();323+ offsetLocalAddr = (__ubuf__ float*)offsetLocal.GetPhyAddr();
328 }324 }
329 325 
330 ProcessYRow(0, 0, scaleOutLocalAddr, offsetLocalAddr);326 ProcessYRow(0, 0, scaleOutLocalAddr, offsetLocalAddr);
@@ -392,31 +388,31 @@ __aicore__ inline void DynamicQuantPertenLargeMulticore<T, yDtype, hasSmooth, is
392 LocalTensor<float> MinOutLocal;388 LocalTensor<float> MinOutLocal;
393 LocalTensor<float> scaleOutLocal;389 LocalTensor<float> scaleOutLocal;
394 LocalTensor<float> offsetLocal;390 LocalTensor<float> offsetLocal;
395- __local_mem__ float* MaxOutLocalAddr;391+ __ubuf__ float* MaxOutLocalAddr;
396- __local_mem__ float* MinOutLocalAddr;392+ __ubuf__ float* MinOutLocalAddr;
397- __local_mem__ float* scaleOutLocalAddr;393+ __ubuf__ float* scaleOutLocalAddr;
398- __local_mem__ float* offsetLocalAddr;394+ __ubuf__ float* offsetLocalAddr;
399 395 
400 if constexpr (isSymmetrical == false) {396 if constexpr (isSymmetrical == false) {
401 MaxOutLocal = MaxOutQueue.AllocTensor<float>();397 MaxOutLocal = MaxOutQueue.AllocTensor<float>();
402 AscendC::Duplicate(MaxOutLocal, MIN_FLOAT_VALUE, 64, 1, 1, 8);398 AscendC::Duplicate(MaxOutLocal, MIN_FLOAT_VALUE, 64, 1, 1, 8);
403- MaxOutLocalAddr = (__local_mem__ float*)MaxOutLocal.GetPhyAddr();399+ MaxOutLocalAddr = (__ubuf__ float*)MaxOutLocal.GetPhyAddr();
404 400 
405 MinOutLocal = MinOutQueue.AllocTensor<float>();401 MinOutLocal = MinOutQueue.AllocTensor<float>();
406 AscendC::Duplicate(MinOutLocal, MAX_FLOAT_VALUE, 64, 1, 1, 8);402 AscendC::Duplicate(MinOutLocal, MAX_FLOAT_VALUE, 64, 1, 1, 8);
407- MinOutLocalAddr = (__local_mem__ float*)MinOutLocal.GetPhyAddr();403+ MinOutLocalAddr = (__ubuf__ float*)MinOutLocal.GetPhyAddr();
408 404 
409 scaleOutLocal = scaleOutQueue.AllocTensor<float>();405 scaleOutLocal = scaleOutQueue.AllocTensor<float>();
410 AscendC::Duplicate(scaleOutLocal, (float)0.0, 64, 1, 1, 8);406 AscendC::Duplicate(scaleOutLocal, (float)0.0, 64, 1, 1, 8);
411- scaleOutLocalAddr = (__local_mem__ float*)scaleOutLocal.GetPhyAddr();407+ scaleOutLocalAddr = (__ubuf__ float*)scaleOutLocal.GetPhyAddr();
412 408 
413 offsetLocal = offsetQueue.AllocTensor<float>();409 offsetLocal = offsetQueue.AllocTensor<float>();
414 AscendC::Duplicate(offsetLocal, (float)0.0, 64, 1, 1, 8);410 AscendC::Duplicate(offsetLocal, (float)0.0, 64, 1, 1, 8);
415- offsetLocalAddr = (__local_mem__ float*)offsetLocal.GetPhyAddr();411+ offsetLocalAddr = (__ubuf__ float*)offsetLocal.GetPhyAddr();
416 } else {412 } else {
417 scaleOutLocal = scaleOutQueue.AllocTensor<float>();413 scaleOutLocal = scaleOutQueue.AllocTensor<float>();
418 AscendC::Duplicate(scaleOutLocal, (float)0.0, 64, 1, 1, 8);414 AscendC::Duplicate(scaleOutLocal, (float)0.0, 64, 1, 1, 8);
419- scaleOutLocalAddr = (__local_mem__ float*)scaleOutLocal.GetPhyAddr();415+ scaleOutLocalAddr = (__ubuf__ float*)scaleOutLocal.GetPhyAddr();
420 }416 }
421 scaleOffset = 0;417 scaleOffset = 0;
422 CopyInScaleByEle(static_cast<int64_t>(tokenIdx) * static_cast<int64_t>(coreNumPerToken), coreNumPerToken);418 CopyInScaleByEle(static_cast<int64_t>(tokenIdx) * static_cast<int64_t>(coreNumPerToken), coreNumPerToken);
@@ -503,30 +499,30 @@ __aicore__ inline void DynamicQuantPertenLargeMulticore<T, yDtype, hasSmooth, is
503 uint32_t elementNum)499 uint32_t elementNum)
504{500{
505 LocalTensor<T> inLocal = inQueue.DeQue<T>();501 LocalTensor<T> inLocal = inQueue.DeQue<T>();
506- __local_mem__ T* inLocalAddr = (__local_mem__ T*)inLocal.GetPhyAddr();502+ __ubuf__ T* inLocalAddr = (__ubuf__ T*)inLocal.GetPhyAddr();
507 LocalTensor<T> smoothLocal;503 LocalTensor<T> smoothLocal;
508- __local_mem__ T* smoothLocalAddr;504+ __ubuf__ T* smoothLocalAddr;
509 505 
510 LocalTensor<float> maxToWorkSpaceLocal;506 LocalTensor<float> maxToWorkSpaceLocal;
511- __local_mem__ float* maxToWorkSpaceLocalAddr;507+ __ubuf__ float* maxToWorkSpaceLocalAddr;
512 LocalTensor<float> minToWorkSpaceLocal;508 LocalTensor<float> minToWorkSpaceLocal;
513- __local_mem__ float* minToWorkSpaceLocalAddr;509+ __ubuf__ float* minToWorkSpaceLocalAddr;
514 LocalTensor<float> scaleToWorkSpaceLocal;510 LocalTensor<float> scaleToWorkSpaceLocal;
515- __local_mem__ float* scaleToWorkSpaceLocalAddr;511+ __ubuf__ float* scaleToWorkSpaceLocalAddr;
516 512 
517 if constexpr (hasSmooth == 1) {513 if constexpr (hasSmooth == 1) {
518 smoothLocal = smoothQueue.DeQue<T>();514 smoothLocal = smoothQueue.DeQue<T>();
519- smoothLocalAddr = (__local_mem__ T*)smoothLocal.GetPhyAddr();515+ smoothLocalAddr = (__ubuf__ T*)smoothLocal.GetPhyAddr();
520 }516 }
521 if constexpr (isSymmetrical == false) {517 if constexpr (isSymmetrical == false) {
522 maxToWorkSpaceLocal = MaxToWorkSpaceQueue.DeQue<float>();518 maxToWorkSpaceLocal = MaxToWorkSpaceQueue.DeQue<float>();
523- maxToWorkSpaceLocalAddr = (__local_mem__ float*)maxToWorkSpaceLocal.GetPhyAddr();519+ maxToWorkSpaceLocalAddr = (__ubuf__ float*)maxToWorkSpaceLocal.GetPhyAddr();
524 520 
525 minToWorkSpaceLocal = MinToWorkSpaceQueue.DeQue<float>();521 minToWorkSpaceLocal = MinToWorkSpaceQueue.DeQue<float>();
526- minToWorkSpaceLocalAddr = (__local_mem__ float*)minToWorkSpaceLocal.GetPhyAddr();522+ minToWorkSpaceLocalAddr = (__ubuf__ float*)minToWorkSpaceLocal.GetPhyAddr();
527 } else {523 } else {
528 scaleToWorkSpaceLocal = scaleToWorkSpaceQueue.DeQue<float>();524 scaleToWorkSpaceLocal = scaleToWorkSpaceQueue.DeQue<float>();
529- scaleToWorkSpaceLocalAddr = (__local_mem__ float*)scaleToWorkSpaceLocal.GetPhyAddr();525+ scaleToWorkSpaceLocalAddr = (__ubuf__ float*)scaleToWorkSpaceLocal.GetPhyAddr();
530 }526 }
531 527 
532 ComputeMaxRowScaleVF(inLocalAddr, smoothLocalAddr, scaleToWorkSpaceLocalAddr, maxToWorkSpaceLocalAddr,528 ComputeMaxRowScaleVF(inLocalAddr, smoothLocalAddr, scaleToWorkSpaceLocalAddr, maxToWorkSpaceLocalAddr,
@@ -545,24 +541,24 @@ __aicore__ inline void DynamicQuantPertenLargeMulticore<T, yDtype, hasSmooth, is
545 541 
546template <typename T, typename yDtype, int64_t hasSmooth, bool isSymmetrical>542template <typename T, typename yDtype, int64_t hasSmooth, bool isSymmetrical>
547__aicore__ inline void DynamicQuantPertenLargeMulticore<T, yDtype, hasSmooth, isSymmetrical>::ComputeMaxColScale(543__aicore__ inline void DynamicQuantPertenLargeMulticore<T, yDtype, hasSmooth, isSymmetrical>::ComputeMaxColScale(
548- uint32_t elementNum, __local_mem__ float* maxAddr, __local_mem__ float* minAddr, __local_mem__ float* scaleAddr,544+ uint32_t elementNum, __ubuf__ float* maxAddr, __ubuf__ float* minAddr, __ubuf__ float* scaleAddr,
549- __local_mem__ float* offsetAddr)545+ __ubuf__ float* offsetAddr)
550{546{
551 LocalTensor<float> scaleFromWorkSpaceLocal;547 LocalTensor<float> scaleFromWorkSpaceLocal;
552- __local_mem__ float* scaleFromWorkSpaceLocalAddr;548+ __ubuf__ float* scaleFromWorkSpaceLocalAddr;
553 549 
554 LocalTensor<float> maxFromWorkSpaceLocal;550 LocalTensor<float> maxFromWorkSpaceLocal;
555- __local_mem__ float* maxFromWorkSpaceLocalAddr;551+ __ubuf__ float* maxFromWorkSpaceLocalAddr;
556 552 
557 LocalTensor<float> minFromWorkSpaceLocal;553 LocalTensor<float> minFromWorkSpaceLocal;
558- __local_mem__ float* minFromWorkSpaceLocalAddr;554+ __ubuf__ float* minFromWorkSpaceLocalAddr;
559 555 
560 if constexpr (isSymmetrical == false) {556 if constexpr (isSymmetrical == false) {
561 maxFromWorkSpaceLocal = MaxFromWorkSpaceQueue.DeQue<float>();557 maxFromWorkSpaceLocal = MaxFromWorkSpaceQueue.DeQue<float>();
562- maxFromWorkSpaceLocalAddr = (__local_mem__ float*)maxFromWorkSpaceLocal.GetPhyAddr();558+ maxFromWorkSpaceLocalAddr = (__ubuf__ float*)maxFromWorkSpaceLocal.GetPhyAddr();
563 559 
564 minFromWorkSpaceLocal = MinFromWorkSpaceQueue.DeQue<float>();560 minFromWorkSpaceLocal = MinFromWorkSpaceQueue.DeQue<float>();
565- minFromWorkSpaceLocalAddr = (__local_mem__ float*)minFromWorkSpaceLocal.GetPhyAddr();561+ minFromWorkSpaceLocalAddr = (__ubuf__ float*)minFromWorkSpaceLocal.GetPhyAddr();
566 562 
567 ComputeMaxColScaleVF(scaleFromWorkSpaceLocalAddr, scaleAddr, maxFromWorkSpaceLocalAddr, maxAddr,563 ComputeMaxColScaleVF(scaleFromWorkSpaceLocalAddr, scaleAddr, maxFromWorkSpaceLocalAddr, maxAddr,
568 minFromWorkSpaceLocalAddr, minAddr, elementNum);564 minFromWorkSpaceLocalAddr, minAddr, elementNum);
@@ -573,7 +569,7 @@ __aicore__ inline void DynamicQuantPertenLargeMulticore<T, yDtype, hasSmooth, is
573 MinFromWorkSpaceQueue.FreeTensor(minFromWorkSpaceLocal);569 MinFromWorkSpaceQueue.FreeTensor(minFromWorkSpaceLocal);
574 } else {570 } else {
575 scaleFromWorkSpaceLocal = scaleFromWorkSpaceQueue.DeQue<float>();571 scaleFromWorkSpaceLocal = scaleFromWorkSpaceQueue.DeQue<float>();
576- scaleFromWorkSpaceLocalAddr = (__local_mem__ float*)scaleFromWorkSpaceLocal.GetPhyAddr();572+ scaleFromWorkSpaceLocalAddr = (__ubuf__ float*)scaleFromWorkSpaceLocal.GetPhyAddr();
577 573 
578 ComputeMaxColScaleVF(scaleFromWorkSpaceLocalAddr, scaleAddr, maxFromWorkSpaceLocalAddr, maxAddr,574 ComputeMaxColScaleVF(scaleFromWorkSpaceLocalAddr, scaleAddr, maxFromWorkSpaceLocalAddr, maxAddr,
579 minFromWorkSpaceLocalAddr, minAddr, elementNum);575 minFromWorkSpaceLocalAddr, minAddr, elementNum);
@@ -583,8 +579,8 @@ __aicore__ inline void DynamicQuantPertenLargeMulticore<T, yDtype, hasSmooth, is
583 579 
584template <typename T, typename yDtype, int64_t hasSmooth, bool isSymmetrical>580template <typename T, typename yDtype, int64_t hasSmooth, bool isSymmetrical>
585__aicore__ inline void DynamicQuantPertenLargeMulticore<T, yDtype, hasSmooth, isSymmetrical>::ComputeMaxRowScaleVF(581__aicore__ inline void DynamicQuantPertenLargeMulticore<T, yDtype, hasSmooth, isSymmetrical>::ComputeMaxRowScaleVF(
586- __local_mem__ T* inLocalAddr, __local_mem__ T* smoothLocalAddr, __local_mem__ float* scaleLocalAddr,582+ __ubuf__ T* inLocalAddr, __ubuf__ T* smoothLocalAddr, __ubuf__ float* scaleLocalAddr, __ubuf__ float* maxLocalAddr,
587- __local_mem__ float* maxLocalAddr, __local_mem__ float* minLocalAddr, uint32_t elementNum)583+ __ubuf__ float* minLocalAddr, uint32_t elementNum)
588{584{
589 uint32_t dtypeSize = sizeof(float);585 uint32_t dtypeSize = sizeof(float);
590 uint32_t VL = AscendC::VECTOR_REG_WIDTH / dtypeSize;586 uint32_t VL = AscendC::VECTOR_REG_WIDTH / dtypeSize;
@@ -610,19 +606,19 @@ __aicore__ inline void DynamicQuantPertenLargeMulticore<T, yDtype, hasSmooth, is
610 AscendC::Reg::MaskReg preg1 = AscendC::Reg::CreateMask<float, AscendC::Reg::MaskPattern::ALL>();606 AscendC::Reg::MaskReg preg1 = AscendC::Reg::CreateMask<float, AscendC::Reg::MaskPattern::ALL>();
611 607 
612 if constexpr (isSymmetrical == false) {608 if constexpr (isSymmetrical == false) {
613- AscendC::Reg::DataCopy<float, AscendC::Reg::LoadDist::DIST_BRC_B32>(vreg9_1, maxLocalAddr);609+ AscendC::Reg::LoadAlign<float, AscendC::Reg::LoadDist::DIST_BRC_B32>(vreg9_1, maxLocalAddr);
614- AscendC::Reg::DataCopy<float, AscendC::Reg::LoadDist::DIST_BRC_B32>(vreg9_2, minLocalAddr);610+ AscendC::Reg::LoadAlign<float, AscendC::Reg::LoadDist::DIST_BRC_B32>(vreg9_2, minLocalAddr);
615 } else {611 } else {
616- AscendC::Reg::DataCopy<float, AscendC::Reg::LoadDist::DIST_BRC_B32>(vreg9_1, scaleLocalAddr);612+ AscendC::Reg::LoadAlign<float, AscendC::Reg::LoadDist::DIST_BRC_B32>(vreg9_1, scaleLocalAddr);
617 }613 }
618 614 
619 for (uint16_t i = 0; i < vfLoopNum; i++) {615 for (uint16_t i = 0; i < vfLoopNum; i++) {
620 uint32_t sreg0 = elementNum - i * VL;616 uint32_t sreg0 = elementNum - i * VL;
621 preg0 = AscendC::Reg::UpdateMask<float>(sreg0);617 preg0 = AscendC::Reg::UpdateMask<float>(sreg0);
622- AscendC::Reg::DataCopy<T, AscendC::Reg::LoadDist::DIST_UNPACK_B16>(vreg1, inLocalAddr + i * VL);618+ AscendC::Reg::LoadAlign<T, AscendC::Reg::LoadDist::DIST_UNPACK_B16>(vreg1, inLocalAddr + i * VL);
623 AscendC::Reg::Cast<float, T, castTrait0>(vreg3, vreg1, preg0);619 AscendC::Reg::Cast<float, T, castTrait0>(vreg3, vreg1, preg0);
624 if constexpr (hasSmooth == 1) {620 if constexpr (hasSmooth == 1) {
625- AscendC::Reg::DataCopy<T, AscendC::Reg::LoadDist::DIST_UNPACK_B16>(vreg2, smoothLocalAddr + i * VL);621+ AscendC::Reg::LoadAlign<T, AscendC::Reg::LoadDist::DIST_UNPACK_B16>(vreg2, smoothLocalAddr + i * VL);
626 AscendC::Reg::Cast<float, T, castTrait0>(vreg4, vreg2, preg0);622 AscendC::Reg::Cast<float, T, castTrait0>(vreg4, vreg2, preg0);
627 AscendC::Reg::Mul(vreg3, vreg3, vreg4, preg0);623 AscendC::Reg::Mul(vreg3, vreg3, vreg4, preg0);
628 }624 }
@@ -637,25 +633,24 @@ __aicore__ inline void DynamicQuantPertenLargeMulticore<T, yDtype, hasSmooth, is
637 }633 }
638 634 
639 if constexpr (isSymmetrical == false) {635 if constexpr (isSymmetrical == false) {
640- AscendC::Reg::ReduceMax(vreg8_1, vreg9_1, preg1);636+ AscendC::Reg::Reduce<Reg::ReduceType::MAX>(vreg8_1, vreg9_1, preg1);
641- AscendC::Reg::ReduceMin(vreg8_2, vreg9_2, preg1);637+ AscendC::Reg::Reduce<Reg::ReduceType::MIN>(vreg8_2, vreg9_2, preg1);
642- AscendC::Reg::DataCopy<float, AscendC::Reg::StoreDist::DIST_FIRST_ELEMENT_B32>(maxLocalAddr, vreg8_1,638+ AscendC::Reg::StoreAlign<float, AscendC::Reg::StoreDist::DIST_FIRST_ELEMENT_B32>(maxLocalAddr, vreg8_1,
643- preg0);639+ preg0);
644- AscendC::Reg::DataCopy<float, AscendC::Reg::StoreDist::DIST_FIRST_ELEMENT_B32>(minLocalAddr, vreg8_2,640+ AscendC::Reg::StoreAlign<float, AscendC::Reg::StoreDist::DIST_FIRST_ELEMENT_B32>(minLocalAddr, vreg8_2,
645- preg0);641+ preg0);
646 } else {642 } else {
647- AscendC::Reg::ReduceMax(vreg8_1, vreg9_1, preg1);643+ AscendC::Reg::Reduce<Reg::ReduceType::MAX>(vreg8_1, vreg9_1, preg1);
648- AscendC::Reg::DataCopy<float, AscendC::Reg::StoreDist::DIST_FIRST_ELEMENT_B32>(scaleLocalAddr, vreg8_1,644+ AscendC::Reg::StoreAlign<float, AscendC::Reg::StoreDist::DIST_FIRST_ELEMENT_B32>(scaleLocalAddr, vreg8_1,
649- preg0);645+ preg0);
650 }646 }
651 }647 }
652}648}
653 649 
654template <typename T, typename yDtype, int64_t hasSmooth, bool isSymmetrical>650template <typename T, typename yDtype, int64_t hasSmooth, bool isSymmetrical>
655__aicore__ inline void DynamicQuantPertenLargeMulticore<T, yDtype, hasSmooth, isSymmetrical>::ComputeMaxColScaleVF(651__aicore__ inline void DynamicQuantPertenLargeMulticore<T, yDtype, hasSmooth, isSymmetrical>::ComputeMaxColScaleVF(
656- __local_mem__ float* scaleLocalAddr, __local_mem__ float* scaleOutLocalAddr, __local_mem__ float* maxLocalAddr,652+ __ubuf__ float* scaleLocalAddr, __ubuf__ float* scaleOutLocalAddr, __ubuf__ float* maxLocalAddr,
657- __local_mem__ float* maxOutLocalAddr, __local_mem__ float* minLocalAddr, __local_mem__ float* minOutLocalAddr,653+ __ubuf__ float* maxOutLocalAddr, __ubuf__ float* minLocalAddr, __ubuf__ float* minOutLocalAddr, uint32_t elementNum)
658- uint32_t elementNum)
659{654{
660 uint32_t dtypeSize = sizeof(float);655 uint32_t dtypeSize = sizeof(float);
661 uint32_t VL = AscendC::VECTOR_REG_WIDTH / dtypeSize;656 uint32_t VL = AscendC::VECTOR_REG_WIDTH / dtypeSize;
@@ -676,69 +671,68 @@ __aicore__ inline void DynamicQuantPertenLargeMulticore<T, yDtype, hasSmooth, is
676 AscendC::Reg::MaskReg mask = AscendC::Reg::CreateMask<float, AscendC::Reg::MaskPattern::ALL>();671 AscendC::Reg::MaskReg mask = AscendC::Reg::CreateMask<float, AscendC::Reg::MaskPattern::ALL>();
677 672 
678 if constexpr (isSymmetrical == false) {673 if constexpr (isSymmetrical == false) {
679- AscendC::Reg::DataCopy<float, AscendC::Reg::LoadDist::DIST_BRC_B32>(vreg1_1, maxOutLocalAddr);674+ AscendC::Reg::LoadAlign<float, AscendC::Reg::LoadDist::DIST_BRC_B32>(vreg1_1, maxOutLocalAddr);
680- AscendC::Reg::DataCopy<float, AscendC::Reg::LoadDist::DIST_BRC_B32>(vreg1_2, minOutLocalAddr);675+ AscendC::Reg::LoadAlign<float, AscendC::Reg::LoadDist::DIST_BRC_B32>(vreg1_2, minOutLocalAddr);
681 } else {676 } else {
682- AscendC::Reg::DataCopy<float, AscendC::Reg::LoadDist::DIST_BRC_B32>(vreg1_1, scaleOutLocalAddr);677+ AscendC::Reg::LoadAlign<float, AscendC::Reg::LoadDist::DIST_BRC_B32>(vreg1_1, scaleOutLocalAddr);
683 }678 }
684 679 
685 for (uint16_t i = 0; i < static_cast<uint16_t>(vfLoopNum - 1); i++) {680 for (uint16_t i = 0; i < static_cast<uint16_t>(vfLoopNum - 1); i++) {
686 maskNum = elementNum - i * VL;681 maskNum = elementNum - i * VL;
687 mask = AscendC::Reg::UpdateMask<float>(maskNum);682 mask = AscendC::Reg::UpdateMask<float>(maskNum);
688 if constexpr (isSymmetrical == false) {683 if constexpr (isSymmetrical == false) {
689- AscendC::Reg::DataCopy<float, AscendC::Reg::LoadDist::DIST_NORM>(vreg0_1, maxLocalAddr + i * VL);684+ AscendC::Reg::LoadAlign<float, AscendC::Reg::LoadDist::DIST_NORM>(vreg0_1, maxLocalAddr + i * VL);
690 AscendC::Reg::Max(vreg1_1, vreg0_1, vreg1_1, mask);685 AscendC::Reg::Max(vreg1_1, vreg0_1, vreg1_1, mask);
691- AscendC::Reg::DataCopy<float, AscendC::Reg::LoadDist::DIST_NORM>(vreg0_2, minLocalAddr + i * VL);686+ AscendC::Reg::LoadAlign<float, AscendC::Reg::LoadDist::DIST_NORM>(vreg0_2, minLocalAddr + i * VL);
692 AscendC::Reg::Min(vreg1_2, vreg0_2, vreg1_2, mask);687 AscendC::Reg::Min(vreg1_2, vreg0_2, vreg1_2, mask);
693 } else {688 } else {
694- AscendC::Reg::DataCopy<float, AscendC::Reg::LoadDist::DIST_NORM>(vreg0_1, scaleLocalAddr + i * VL);689+ AscendC::Reg::LoadAlign<float, AscendC::Reg::LoadDist::DIST_NORM>(vreg0_1, scaleLocalAddr + i * VL);
695 AscendC::Reg::Max(vreg1_1, vreg0_1, vreg1_1, mask);690 AscendC::Reg::Max(vreg1_1, vreg0_1, vreg1_1, mask);
696 }691 }
697 }692 }
698 {693 {
699 if constexpr (isSymmetrical == false) {694 if constexpr (isSymmetrical == false) {
700- AscendC::Reg::ReduceMax<float>(vreg2_1, vreg1_1, mask);695+ AscendC::Reg::Reduce<Reg::ReduceType::MAX, float>(vreg2_1, vreg1_1, mask);
701- AscendC::Reg::ReduceMin<float>(vreg2_2, vreg1_2, mask);696+ AscendC::Reg::Reduce<Reg::ReduceType::MIN, float>(vreg2_2, vreg1_2, mask);
702 } else {697 } else {
703- AscendC::Reg::ReduceMax<float>(vreg2_1, vreg1_1, mask);698+ AscendC::Reg::Reduce<Reg::ReduceType::MAX, float>(vreg2_1, vreg1_1, mask);
704 }699 }
705 }700 }
706 for (uint16_t i = vfLoopNum - 1; i < vfLoopNum; i++) {701 for (uint16_t i = vfLoopNum - 1; i < vfLoopNum; i++) {
707 maskNum = elementNum - i * VL;702 maskNum = elementNum - i * VL;
708 mask = AscendC::Reg::UpdateMask<float>(maskNum);703 mask = AscendC::Reg::UpdateMask<float>(maskNum);
709 if constexpr (isSymmetrical == false) {704 if constexpr (isSymmetrical == false) {
710- AscendC::Reg::DataCopy<float, AscendC::Reg::LoadDist::DIST_NORM>(vreg0_1, maxLocalAddr + i * VL);705+ AscendC::Reg::LoadAlign<float, AscendC::Reg::LoadDist::DIST_NORM>(vreg0_1, maxLocalAddr + i * VL);
711 AscendC::Reg::Max(vreg1_1, vreg0_1, vreg1_1, mask);706 AscendC::Reg::Max(vreg1_1, vreg0_1, vreg1_1, mask);
712- AscendC::Reg::ReduceMax<float>(vreg3_1, vreg1_1, mask);707+ AscendC::Reg::Reduce<Reg::ReduceType::MAX, float>(vreg3_1, vreg1_1, mask);
713 AscendC::Reg::Max(vreg3_1, vreg2_1, vreg3_1, mask);708 AscendC::Reg::Max(vreg3_1, vreg2_1, vreg3_1, mask);
714 709 
715- AscendC::Reg::DataCopy<float, AscendC::Reg::LoadDist::DIST_NORM>(vreg0_2, minLocalAddr + i * VL);710+ AscendC::Reg::LoadAlign<float, AscendC::Reg::LoadDist::DIST_NORM>(vreg0_2, minLocalAddr + i * VL);
716 AscendC::Reg::Min(vreg1_2, vreg0_2, vreg1_2, mask);711 AscendC::Reg::Min(vreg1_2, vreg0_2, vreg1_2, mask);
717- AscendC::Reg::ReduceMin<float>(vreg3_2, vreg1_2, mask);712+ AscendC::Reg::Reduce<Reg::ReduceType::MIN, float>(vreg3_2, vreg1_2, mask);
718 AscendC::Reg::Min(vreg3_2, vreg2_2, vreg3_2, mask);713 AscendC::Reg::Min(vreg3_2, vreg2_2, vreg3_2, mask);
719 } else {714 } else {
720- AscendC::Reg::DataCopy<float, AscendC::Reg::LoadDist::DIST_NORM>(vreg0_1, scaleLocalAddr + i * VL);715+ AscendC::Reg::LoadAlign<float, AscendC::Reg::LoadDist::DIST_NORM>(vreg0_1, scaleLocalAddr + i * VL);
721 AscendC::Reg::Max(vreg1_1, vreg0_1, vreg1_1, mask);716 AscendC::Reg::Max(vreg1_1, vreg0_1, vreg1_1, mask);
722- AscendC::Reg::ReduceMax<float>(vreg3_1, vreg1_1, mask);717+ AscendC::Reg::Reduce<Reg::ReduceType::MAX, float>(vreg3_1, vreg1_1, mask);
723 AscendC::Reg::Max(vreg3_1, vreg2_1, vreg3_1, mask);718 AscendC::Reg::Max(vreg3_1, vreg2_1, vreg3_1, mask);
724 }719 }
725 }720 }
726 if constexpr (isSymmetrical == false) {721 if constexpr (isSymmetrical == false) {
727- AscendC::Reg::DataCopy<float, AscendC::Reg::StoreDist::DIST_FIRST_ELEMENT_B32>(maxOutLocalAddr, vreg3_1,722+ AscendC::Reg::StoreAlign<float, AscendC::Reg::StoreDist::DIST_FIRST_ELEMENT_B32>(maxOutLocalAddr, vreg3_1,
728- mask);723+ mask);
729- AscendC::Reg::DataCopy<float, AscendC::Reg::StoreDist::DIST_FIRST_ELEMENT_B32>(minOutLocalAddr, vreg3_2,724+ AscendC::Reg::StoreAlign<float, AscendC::Reg::StoreDist::DIST_FIRST_ELEMENT_B32>(minOutLocalAddr, vreg3_2,
730- mask);725+ mask);
731 } else {726 } else {
732- AscendC::Reg::DataCopy<float, AscendC::Reg::StoreDist::DIST_FIRST_ELEMENT_B32>(scaleOutLocalAddr, vreg3_1,727+ AscendC::Reg::StoreAlign<float, AscendC::Reg::StoreDist::DIST_FIRST_ELEMENT_B32>(scaleOutLocalAddr, vreg3_1,
733- mask);728+ mask);
734 }729 }
735 }730 }
736}731}
737 732 
738template <typename T, typename yDtype, int64_t hasSmooth, bool isSymmetrical>733template <typename T, typename yDtype, int64_t hasSmooth, bool isSymmetrical>
739__aicore__ inline void DynamicQuantPertenLargeMulticore<T, yDtype, hasSmooth, isSymmetrical>::ComputeScaleSymVF(734__aicore__ inline void DynamicQuantPertenLargeMulticore<T, yDtype, hasSmooth, isSymmetrical>::ComputeScaleSymVF(
740- __local_mem__ float* maxLocalAddr, __local_mem__ float* minLocalAddr, __local_mem__ float* scaleLocalAddr,735+ __ubuf__ float* maxLocalAddr, __ubuf__ float* minLocalAddr, __ubuf__ float* scaleLocalAddr, uint32_t elementNum)
741- uint32_t elementNum)
742{736{
743 uint32_t dtypeSize = sizeof(float);737 uint32_t dtypeSize = sizeof(float);
744 uint32_t VL = AscendC::VECTOR_REG_WIDTH / dtypeSize;738 uint32_t VL = AscendC::VECTOR_REG_WIDTH / dtypeSize;
@@ -755,19 +749,19 @@ __aicore__ inline void DynamicQuantPertenLargeMulticore<T, yDtype, hasSmooth, is
755 maskNum = elementNum - i * VL;749 maskNum = elementNum - i * VL;
756 mask = AscendC::Reg::UpdateMask<float>(maskNum);750 mask = AscendC::Reg::UpdateMask<float>(maskNum);
757 751 
758- AscendC::Reg::DataCopy<float, AscendC::Reg::LoadDist::DIST_NORM>(vreg0, maxLocalAddr + i * VL);752+ AscendC::Reg::LoadAlign<float, AscendC::Reg::LoadDist::DIST_NORM>(vreg0, maxLocalAddr + i * VL);
759- AscendC::Reg::DataCopy<float, AscendC::Reg::LoadDist::DIST_NORM>(vreg1, minLocalAddr + i * VL);753+ AscendC::Reg::LoadAlign<float, AscendC::Reg::LoadDist::DIST_NORM>(vreg1, minLocalAddr + i * VL);
760 AscendC::Reg::Sub(vreg1, vreg0, vreg1, mask);754 AscendC::Reg::Sub(vreg1, vreg0, vreg1, mask);
761 AscendC::Reg::Muls(vreg1, vreg1, float(1.0) / maxValueNoSym, mask);755 AscendC::Reg::Muls(vreg1, vreg1, float(1.0) / maxValueNoSym, mask);
762- AscendC::Reg::DataCopy<float, AscendC::Reg::StoreDist::DIST_FIRST_ELEMENT_B32>(scaleLocalAddr, vreg1, mask);756+ AscendC::Reg::StoreAlign<float, AscendC::Reg::StoreDist::DIST_FIRST_ELEMENT_B32>(scaleLocalAddr, vreg1,
757+ mask);
763 }758 }
764 }759 }
765}760}
766 761 
767template <typename T, typename yDtype, int64_t hasSmooth, bool isSymmetrical>762template <typename T, typename yDtype, int64_t hasSmooth, bool isSymmetrical>
768__aicore__ inline void DynamicQuantPertenLargeMulticore<T, yDtype, hasSmooth, isSymmetrical>::ComputeOffsetSymVF(763__aicore__ inline void DynamicQuantPertenLargeMulticore<T, yDtype, hasSmooth, isSymmetrical>::ComputeOffsetSymVF(
769- __local_mem__ float* maxLocalAddr, __local_mem__ float* scaleLocalAddr, __local_mem__ float* offsetLocalAddr,764+ __ubuf__ float* maxLocalAddr, __ubuf__ float* scaleLocalAddr, __ubuf__ float* offsetLocalAddr, uint32_t elementNum)
770- uint32_t elementNum)
771{765{
772 uint32_t dtypeSize = sizeof(float);766 uint32_t dtypeSize = sizeof(float);
773 uint32_t VL = AscendC::VECTOR_REG_WIDTH / dtypeSize;767 uint32_t VL = AscendC::VECTOR_REG_WIDTH / dtypeSize;
@@ -785,32 +779,32 @@ __aicore__ inline void DynamicQuantPertenLargeMulticore<T, yDtype, hasSmooth, is
785 maskNum = elementNum - i * VL;779 maskNum = elementNum - i * VL;
786 mask = AscendC::Reg::UpdateMask<float>(maskNum);780 mask = AscendC::Reg::UpdateMask<float>(maskNum);
787 781 
788- AscendC::Reg::DataCopy<float, AscendC::Reg::LoadDist::DIST_NORM>(vreg0, maxLocalAddr + i * VL);782+ AscendC::Reg::LoadAlign<float, AscendC::Reg::LoadDist::DIST_NORM>(vreg0, maxLocalAddr + i * VL);
789- AscendC::Reg::DataCopy<float, AscendC::Reg::LoadDist::DIST_NORM>(vreg1, scaleLocalAddr + i * VL);783+ AscendC::Reg::LoadAlign<float, AscendC::Reg::LoadDist::DIST_NORM>(vreg1, scaleLocalAddr + i * VL);
790 AscendC::Reg::Div<float, &mode>(vreg1, vreg0, vreg1, mask);784 AscendC::Reg::Div<float, &mode>(vreg1, vreg0, vreg1, mask);
791 AscendC::Reg::Muls(vreg1, vreg1, float(-1.0), mask);785 AscendC::Reg::Muls(vreg1, vreg1, float(-1.0), mask);
792 AscendC::Reg::Adds(vreg1, vreg1, maxValue, mask);786 AscendC::Reg::Adds(vreg1, vreg1, maxValue, mask);
793 787 
794- AscendC::Reg::DataCopy<float, AscendC::Reg::StoreDist::DIST_FIRST_ELEMENT_B32>(offsetLocalAddr, vreg1,788+ AscendC::Reg::StoreAlign<float, AscendC::Reg::StoreDist::DIST_FIRST_ELEMENT_B32>(offsetLocalAddr, vreg1,
795- mask);789+ mask);
796 }790 }
797 }791 }
798}792}
799 793 
800template <typename T, typename yDtype, int64_t hasSmooth, bool isSymmetrical>794template <typename T, typename yDtype, int64_t hasSmooth, bool isSymmetrical>
801__aicore__ inline void DynamicQuantPertenLargeMulticore<T, yDtype, hasSmooth, isSymmetrical>::ComputeY(795__aicore__ inline void DynamicQuantPertenLargeMulticore<T, yDtype, hasSmooth, isSymmetrical>::ComputeY(
802- uint32_t elementNum, __local_mem__ float* scaleAddr, __local_mem__ float* offsetAddr)796+ uint32_t elementNum, __ubuf__ float* scaleAddr, __ubuf__ float* offsetAddr)
803{797{
804 LocalTensor<T> inLocal = inQueue.DeQue<T>();798 LocalTensor<T> inLocal = inQueue.DeQue<T>();
805- __local_mem__ T* inLocalAddr = (__local_mem__ T*)inLocal.GetPhyAddr();799+ __ubuf__ T* inLocalAddr = (__ubuf__ T*)inLocal.GetPhyAddr();
806 LocalTensor<yCopyDtype> outLocal = outQueue.AllocTensor<yCopyDtype>();800 LocalTensor<yCopyDtype> outLocal = outQueue.AllocTensor<yCopyDtype>();
807- __local_mem__ yCopyDtype* outAddr = (__local_mem__ yCopyDtype*)outLocal.GetPhyAddr();801+ __ubuf__ yCopyDtype* outAddr = (__ubuf__ yCopyDtype*)outLocal.GetPhyAddr();
808 LocalTensor<T> smoothLocal;802 LocalTensor<T> smoothLocal;
809- __local_mem__ T* smoothLocalAddr;803+ __ubuf__ T* smoothLocalAddr;
810 804 
811 if constexpr (hasSmooth == 1) {805 if constexpr (hasSmooth == 1) {
812 smoothLocal = smoothQueue.DeQue<T>();806 smoothLocal = smoothQueue.DeQue<T>();
813- smoothLocalAddr = (__local_mem__ T*)smoothLocal.GetPhyAddr();807+ smoothLocalAddr = (__ubuf__ T*)smoothLocal.GetPhyAddr();
814 }808 }
815 809 
816 ComputeYVF(inLocalAddr, smoothLocalAddr, outAddr, scaleAddr, offsetAddr, elementNum);810 ComputeYVF(inLocalAddr, smoothLocalAddr, outAddr, scaleAddr, offsetAddr, elementNum);
@@ -824,8 +818,8 @@ __aicore__ inline void DynamicQuantPertenLargeMulticore<T, yDtype, hasSmooth, is
824 818 
825template <typename T, typename yDtype, int64_t hasSmooth, bool isSymmetrical>819template <typename T, typename yDtype, int64_t hasSmooth, bool isSymmetrical>
826__aicore__ inline void DynamicQuantPertenLargeMulticore<T, yDtype, hasSmooth, isSymmetrical>::ComputeYVF(820__aicore__ inline void DynamicQuantPertenLargeMulticore<T, yDtype, hasSmooth, isSymmetrical>::ComputeYVF(
827- __local_mem__ T* inLocalAddr, __local_mem__ T* smoothLocalAddr, __local_mem__ yCopyDtype* outAddr,821+ __ubuf__ T* inLocalAddr, __ubuf__ T* smoothLocalAddr, __ubuf__ yCopyDtype* outAddr, __ubuf__ float* scaleLocalAddr,
828- __local_mem__ float* scaleLocalAddr, __local_mem__ float* offsetLocalAddr, uint32_t elementNum)822+ __ubuf__ float* offsetLocalAddr, uint32_t elementNum)
829{823{
830 uint32_t dtypeSize = sizeof(float);824 uint32_t dtypeSize = sizeof(float);
831 uint32_t VL = AscendC::VECTOR_REG_WIDTH / dtypeSize;825 uint32_t VL = AscendC::VECTOR_REG_WIDTH / dtypeSize;
@@ -849,18 +843,18 @@ __aicore__ inline void DynamicQuantPertenLargeMulticore<T, yDtype, hasSmooth, is
849 AscendC::Reg::MaskReg mask;843 AscendC::Reg::MaskReg mask;
850 AscendC::Reg::MaskReg mask2 = AscendC::Reg::CreateMask<float, AscendC::Reg::MaskPattern::H>();844 AscendC::Reg::MaskReg mask2 = AscendC::Reg::CreateMask<float, AscendC::Reg::MaskPattern::H>();
851 845 
852- AscendC::Reg::DataCopy<float, AscendC::Reg::LoadDist::DIST_BRC_B32>(vreg_scale, scaleLocalAddr);846+ AscendC::Reg::LoadAlign<float, AscendC::Reg::LoadDist::DIST_BRC_B32>(vreg_scale, scaleLocalAddr);
853 if constexpr (isSymmetrical == false) {847 if constexpr (isSymmetrical == false) {
854- AscendC::Reg::DataCopy<float, AscendC::Reg::LoadDist::DIST_BRC_B32>(vreg_offset, offsetLocalAddr);848+ AscendC::Reg::LoadAlign<float, AscendC::Reg::LoadDist::DIST_BRC_B32>(vreg_offset, offsetLocalAddr);
855 }849 }
856 850 
857 for (uint16_t i = 0; i < vfLoopNum; i++) {851 for (uint16_t i = 0; i < vfLoopNum; i++) {
858 auto addr = outAddr + i * VL;852 auto addr = outAddr + i * VL;
859 mask = AscendC::Reg::CreateMask<float, AscendC::Reg::MaskPattern::ALL>();853 mask = AscendC::Reg::CreateMask<float, AscendC::Reg::MaskPattern::ALL>();
860- AscendC::Reg::DataCopy<T, AscendC::Reg::LoadDist::DIST_UNPACK_B16>(vreg0, inLocalAddr + i * VL);854+ AscendC::Reg::LoadAlign<T, AscendC::Reg::LoadDist::DIST_UNPACK_B16>(vreg0, inLocalAddr + i * VL);
861 AscendC::Reg::Cast<float, T, castTrait0>(vreg1, vreg0, mask);855 AscendC::Reg::Cast<float, T, castTrait0>(vreg1, vreg0, mask);
862 if constexpr (hasSmooth == 1) {856 if constexpr (hasSmooth == 1) {
863- AscendC::Reg::DataCopy<T, AscendC::Reg::LoadDist::DIST_UNPACK_B16>(vreg2, smoothLocalAddr + i * VL);857+ AscendC::Reg::LoadAlign<T, AscendC::Reg::LoadDist::DIST_UNPACK_B16>(vreg2, smoothLocalAddr + i * VL);
864 AscendC::Reg::Cast<float, T, castTrait0>(vreg3, vreg2, mask);858 AscendC::Reg::Cast<float, T, castTrait0>(vreg3, vreg2, mask);
865 AscendC::Reg::Mul(vreg4, vreg1, vreg3, mask);859 AscendC::Reg::Mul(vreg4, vreg1, vreg3, mask);
866 AscendC::Reg::Div(vreg5, vreg4, vreg_scale, mask);860 AscendC::Reg::Div(vreg5, vreg4, vreg_scale, mask);
@@ -890,9 +884,9 @@ __aicore__ inline void DynamicQuantPertenLargeMulticore<T, yDtype, hasSmooth, is
890 }884 }
891 885 
892 if constexpr (IsSameType<yDtype, int4b_t>::value) {886 if constexpr (IsSameType<yDtype, int4b_t>::value) {
893- AscendC::Reg::DataCopy<yCopyDtype, AscendC::Reg::StoreDist::DIST_PACK4_B32>(addr, vreg8, mask2);887+ AscendC::Reg::StoreAlign<yCopyDtype, AscendC::Reg::StoreDist::DIST_PACK4_B32>(addr, vreg8, mask2);
894 } else {888 } else {
895- AscendC::Reg::DataCopy<yCopyDtype, AscendC::Reg::StoreDist::DIST_PACK4_B32>(addr, vreg8, mask);889+ AscendC::Reg::StoreAlign<yCopyDtype, AscendC::Reg::StoreDist::DIST_PACK4_B32>(addr, vreg8, mask);
896 }890 }
897 }891 }
898 }892 }
@@ -916,7 +910,7 @@ __aicore__ inline void DynamicQuantPertenLargeMulticore<T, yDtype, hasSmooth, is
916 910 
917template <typename T, typename yDtype, int64_t hasSmooth, bool isSymmetrical>911template <typename T, typename yDtype, int64_t hasSmooth, bool isSymmetrical>
918__aicore__ inline void DynamicQuantPertenLargeMulticore<T, yDtype, hasSmooth, isSymmetrical>::ProcessYRow(912__aicore__ inline void DynamicQuantPertenLargeMulticore<T, yDtype, hasSmooth, isSymmetrical>::ProcessYRow(
919- uint32_t i, uint32_t j, __local_mem__ float* scaleAddr, __local_mem__ float* offsetAddr)913+ uint32_t i, uint32_t j, __ubuf__ float* scaleAddr, __ubuf__ float* offsetAddr)
920{914{
921 offsetBase = i * THIRTY_TWO + j;915 offsetBase = i * THIRTY_TWO + j;
922 srcOffset = tokenIdx * tilingData_.rowLen + coreStartOffset + offsetBase * tilingData_.innerLoopEle;916 srcOffset = tokenIdx * tilingData_.rowLen + coreStartOffset + offsetBase * tilingData_.innerLoopEle;
Mquant/grouped_dynamic_block_quant/op_kernel/arch35/grouped_dynamic_block_quant_large_block.h+30-30
@@ -37,11 +37,11 @@ public:
37private:37private:
38 __aicore__ inline void ParseTilingData(const GroupedDynamicBlockQuantTilingData& tilingData);38 __aicore__ inline void ParseTilingData(const GroupedDynamicBlockQuantTilingData& tilingData);
39 __aicore__ inline void CopyIn(int64_t offset, int64_t rowNum, int64_t dataLen);39 __aicore__ inline void CopyIn(int64_t offset, int64_t rowNum, int64_t dataLen);
40- __aicore__ inline void ComputeXTmpMax(int64_t rowNum, int64_t colNum, __local_mem__ T* xLocalAddr,40+ __aicore__ inline void ComputeXTmpMax(int64_t rowNum, int64_t colNum, __ubuf__ T* xLocalAddr,
41- __local_mem__ T* xLocalMaxTmp);41+ __ubuf__ T* xLocalMaxTmp);
42- __aicore__ inline void ComputeScaleVF(__local_mem__ float* scaleLocalTmp, __local_mem__ T* xLocalMaxTmp);42+ __aicore__ inline void ComputeScaleVF(__ubuf__ float* scaleLocalTmp, __ubuf__ T* xLocalMaxTmp);
43- __aicore__ inline void ComputeOutVF(int64_t xTotalNum, __local_mem__ T* xLocalAddr, __local_mem__ float* scaleLocal,43+ __aicore__ inline void ComputeOutVF(int64_t xTotalNum, __ubuf__ T* xLocalAddr, __ubuf__ float* scaleLocal,
44- __local_mem__ U* outLocal);44+ __ubuf__ U* outLocal);
45 __aicore__ inline void CopyOutScale(int64_t baseScaleOffset);45 __aicore__ inline void CopyOutScale(int64_t baseScaleOffset);
46 __aicore__ inline void CopyOutY(int64_t rowNum, int64_t colNum, int64_t baseYGmOffset);46 __aicore__ inline void CopyOutY(int64_t rowNum, int64_t colNum, int64_t baseYGmOffset);
47 47 
@@ -255,10 +255,10 @@ __aicore__ inline void GroupedDynamicBlockQuantLargeBlock<T, U, RMode>::ProcessO
255 255 
256 LocalTensor<T> xLocalMaxTmp = xLocalMaxBuffer_.Get<T>();256 LocalTensor<T> xLocalMaxTmp = xLocalMaxBuffer_.Get<T>();
257 AscendC::Duplicate(xLocalMaxTmp, static_cast<T>(0), xLocalMaxTmp.GetSize());257 AscendC::Duplicate(xLocalMaxTmp, static_cast<T>(0), xLocalMaxTmp.GetSize());
258- __local_mem__ T* xLocalMaxTmpAddr = (__local_mem__ T*)xLocalMaxTmp.GetPhyAddr();258+ __ubuf__ T* xLocalMaxTmpAddr = (__ubuf__ T*)xLocalMaxTmp.GetPhyAddr();
259 259 
260 LocalTensor<T> inLocal;260 LocalTensor<T> inLocal;
261- __local_mem__ T* xLocalAddr;261+ __ubuf__ T* xLocalAddr;
262 262 
263 // compute Max(abs(x))263 // compute Max(abs(x))
264 for (int64_t ubLoopNumIdx = ubLoopNum - 1; ubLoopNumIdx >= 0; ubLoopNumIdx--) {264 for (int64_t ubLoopNumIdx = ubLoopNum - 1; ubLoopNumIdx >= 0; ubLoopNumIdx--) {
@@ -266,7 +266,7 @@ __aicore__ inline void GroupedDynamicBlockQuantLargeBlock<T, U, RMode>::ProcessO
266 rowNum = (ubLoopNumIdx == ubLoopNum - 1) ? blockRowNum - ubLoopNumIdx * maxUbRow_ : maxUbRow_;266 rowNum = (ubLoopNumIdx == ubLoopNum - 1) ? blockRowNum - ubLoopNumIdx * maxUbRow_ : maxUbRow_;
267 CopyIn(xGmOffset, rowNum, dataLen);267 CopyIn(xGmOffset, rowNum, dataLen);
268 inLocal = inQueue_.DeQue<T>();268 inLocal = inQueue_.DeQue<T>();
269- xLocalAddr = (__local_mem__ T*)inLocal.GetPhyAddr();269+ xLocalAddr = (__ubuf__ T*)inLocal.GetPhyAddr();
270 ComputeXTmpMax(rowNum, dataLen, xLocalAddr, xLocalMaxTmpAddr);270 ComputeXTmpMax(rowNum, dataLen, xLocalAddr, xLocalMaxTmpAddr);
271 if (ubLoopNumIdx != 0) {271 if (ubLoopNumIdx != 0) {
272 inQueue_.FreeTensor(inLocal);272 inQueue_.FreeTensor(inLocal);
@@ -277,11 +277,11 @@ __aicore__ inline void GroupedDynamicBlockQuantLargeBlock<T, U, RMode>::ProcessO
277 (AscendC::LocalTensor<uint16_t>&)xLocalMaxTmp,277 (AscendC::LocalTensor<uint16_t>&)xLocalMaxTmp,
278 (AscendC::LocalTensor<uint16_t>&)xLocalMaxTmp, xLocalMaxTmp.GetSize());278 (AscendC::LocalTensor<uint16_t>&)xLocalMaxTmp, xLocalMaxTmp.GetSize());
279 LocalTensor<float> scaleLocal = scaleQueue_.AllocTensor<float>();279 LocalTensor<float> scaleLocal = scaleQueue_.AllocTensor<float>();
280- __local_mem__ float* scaleLocalAddr = (__local_mem__ float*)scaleLocal.GetPhyAddr();280+ __ubuf__ float* scaleLocalAddr = (__ubuf__ float*)scaleLocal.GetPhyAddr();
281 ComputeScaleVF(scaleLocalAddr, xLocalMaxTmpAddr);281 ComputeScaleVF(scaleLocalAddr, xLocalMaxTmpAddr);
282 282 
283 LocalTensor<U> outLocal = outQueue_.AllocTensor<U>();283 LocalTensor<U> outLocal = outQueue_.AllocTensor<U>();
284- __local_mem__ U* outLocalAddr = (__local_mem__ U*)outLocal.GetPhyAddr();284+ __ubuf__ U* outLocalAddr = (__ubuf__ U*)outLocal.GetPhyAddr();
285 285 
286 // 分段计算Y286 // 分段计算Y
287 ComputeOutVF(rowNum * dataLen, xLocalAddr, scaleLocalAddr, outLocalAddr);287 ComputeOutVF(rowNum * dataLen, xLocalAddr, scaleLocalAddr, outLocalAddr);
@@ -296,10 +296,10 @@ __aicore__ inline void GroupedDynamicBlockQuantLargeBlock<T, U, RMode>::ProcessO
296 CopyIn(xGmOffset, rowNum, dataLen);296 CopyIn(xGmOffset, rowNum, dataLen);
297 297 
298 inLocal = inQueue_.DeQue<T>();298 inLocal = inQueue_.DeQue<T>();
299- xLocalAddr = (__local_mem__ T*)inLocal.GetPhyAddr();299+ xLocalAddr = (__ubuf__ T*)inLocal.GetPhyAddr();
300 300 
301 outLocal = outQueue_.AllocTensor<U>();301 outLocal = outQueue_.AllocTensor<U>();
302- outLocalAddr = (__local_mem__ U*)outLocal.GetPhyAddr();302+ outLocalAddr = (__ubuf__ U*)outLocal.GetPhyAddr();
303 ComputeOutVF(rowNum * dataLen, xLocalAddr, scaleLocalAddr, outLocalAddr);303 ComputeOutVF(rowNum * dataLen, xLocalAddr, scaleLocalAddr, outLocalAddr);
304 304 
305 inQueue_.FreeTensor(inLocal);305 inQueue_.FreeTensor(inLocal);
@@ -313,8 +313,8 @@ __aicore__ inline void GroupedDynamicBlockQuantLargeBlock<T, U, RMode>::ProcessO
313 313 
314template <typename T, typename U, int64_t RMode>314template <typename T, typename U, int64_t RMode>
315__aicore__ inline void GroupedDynamicBlockQuantLargeBlock<T, U, RMode>::ComputeXTmpMax(int64_t rowNum, int64_t colNum,315__aicore__ inline void GroupedDynamicBlockQuantLargeBlock<T, U, RMode>::ComputeXTmpMax(int64_t rowNum, int64_t colNum,
316- __local_mem__ T* xLocalAddr,316+ __ubuf__ T* xLocalAddr,
317- __local_mem__ T* xLocalMaxTmp)317+ __ubuf__ T* xLocalMaxTmp)
318{318{
319 uint32_t xTotalNum = rowNum * colNum;319 uint32_t xTotalNum = rowNum * colNum;
320 uint32_t dtypeSize = sizeof(T);320 uint32_t dtypeSize = sizeof(T);
@@ -332,24 +332,24 @@ __aicore__ inline void GroupedDynamicBlockQuantLargeBlock<T, U, RMode>::ComputeX
332 AscendC::MicroAPI::MaskReg332 AscendC::MicroAPI::MaskReg
333 maskAll = AscendC::MicroAPI::CreateMask<uint16_t, AscendC::MicroAPI::MaskPattern::ALL>();333 maskAll = AscendC::MicroAPI::CreateMask<uint16_t, AscendC::MicroAPI::MaskPattern::ALL>();
334 334 
335- AscendC::MicroAPI::DataCopy(vLocalTmpMaxReg, xLocalMaxTmp);335+ AscendC::MicroAPI::LoadAlign(vLocalTmpMaxReg, xLocalMaxTmp);
336 336 
337 for (uint16_t i = 0; i < vfLoop; i++) {337 for (uint16_t i = 0; i < vfLoop; i++) {
338 preg0 = AscendC::MicroAPI::UpdateMask<T>(xTotalNum);338 preg0 = AscendC::MicroAPI::UpdateMask<T>(xTotalNum);
339- AscendC::MicroAPI::DataCopy(vreg1, xLocalAddr + i * VL);339+ AscendC::MicroAPI::LoadAlign(vreg1, xLocalAddr + i * VL);
340 AscendC::MicroAPI::And((AscendC::MicroAPI::RegTensor<uint16_t>&)vreg3,340 AscendC::MicroAPI::And((AscendC::MicroAPI::RegTensor<uint16_t>&)vreg3,
341 (AscendC::MicroAPI::RegTensor<uint16_t>&)vreg1,341 (AscendC::MicroAPI::RegTensor<uint16_t>&)vreg1,
342 (AscendC::MicroAPI::RegTensor<uint16_t>&)vreg2, preg0);342 (AscendC::MicroAPI::RegTensor<uint16_t>&)vreg2, preg0);
343 AscendC::MicroAPI::Max<T, AscendC::MicroAPI::MaskMergeMode::MERGING>(vLocalTmpMaxReg, vLocalTmpMaxReg,343 AscendC::MicroAPI::Max<T, AscendC::MicroAPI::MaskMergeMode::MERGING>(vLocalTmpMaxReg, vLocalTmpMaxReg,
344 vreg3, preg0);344 vreg3, preg0);
345 }345 }
346- AscendC::MicroAPI::DataCopy(xLocalMaxTmp, vLocalTmpMaxReg, maskAll);346+ AscendC::MicroAPI::StoreAlign(xLocalMaxTmp, vLocalTmpMaxReg, maskAll);
347 }347 }
348}348}
349 349 
350template <typename T, typename U, int64_t RMode>350template <typename T, typename U, int64_t RMode>
351-__aicore__ inline void GroupedDynamicBlockQuantLargeBlock<T, U, RMode>::ComputeScaleVF(351+__aicore__ inline void GroupedDynamicBlockQuantLargeBlock<T, U, RMode>::ComputeScaleVF(__ubuf__ float* scaleLocalTmp,
352- __local_mem__ float* scaleLocalTmp, __local_mem__ T* xLocalMaxTmp)352+ __ubuf__ T* xLocalMaxTmp)
353{353{
354 static constexpr AscendC::MicroAPI::DivSpecificMode mode = {AscendC::MicroAPI::MaskMergeMode::ZEROING, false};354 static constexpr AscendC::MicroAPI::DivSpecificMode mode = {AscendC::MicroAPI::MaskMergeMode::ZEROING, false};
355 uint32_t scaleNum = 1;355 uint32_t scaleNum = 1;
@@ -373,26 +373,26 @@ __aicore__ inline void GroupedDynamicBlockQuantLargeBlock<T, U, RMode>::ComputeS
373 AscendC::MicroAPI::Duplicate(minScaleReg, minScale_);373 AscendC::MicroAPI::Duplicate(minScaleReg, minScale_);
374 AscendC::MicroAPI::Div<float, &mode>(reciprocalScale, reciprocalScale, minScaleReg, preg0);374 AscendC::MicroAPI::Div<float, &mode>(reciprocalScale, reciprocalScale, minScaleReg, preg0);
375 375 
376- AscendC::MicroAPI::DataCopy<T, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(vreg1, xLocalMaxTmp);376+ AscendC::MicroAPI::LoadAlign<T, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(vreg1, xLocalMaxTmp);
377 AscendC::MicroAPI::Cast<float, T, castTraitT2Float>(vreg2, vreg1, preg0);377 AscendC::MicroAPI::Cast<float, T, castTraitT2Float>(vreg2, vreg1, preg0);
378 378 
379 AscendC::MicroAPI::Div<float, &mode>(vreg5, vreg2, vreg3, preg0);379 AscendC::MicroAPI::Div<float, &mode>(vreg5, vreg2, vreg3, preg0);
380- AscendC::MicroAPI::CompareScalar<uint32_t, CMPMODE::LT>(380+ AscendC::MicroAPI::Compares<uint32_t, CMPMODE::LT>(scaleMaskReg, (AscendC::MicroAPI::RegTensor<uint32_t>&)vreg5,
381- scaleMaskReg, (AscendC::MicroAPI::RegTensor<uint32_t>&)vreg5, infValue_, preg0);381+ infValue_, preg0);
382 // Min(input_max / FP_MAX, 1 / minScale)382 // Min(input_max / FP_MAX, 1 / minScale)
383 AscendC::MicroAPI::Min<float, AscendC::MicroAPI::MaskMergeMode::MERGING>(vreg5, vreg5, reciprocalScale,383 AscendC::MicroAPI::Min<float, AscendC::MicroAPI::MaskMergeMode::MERGING>(vreg5, vreg5, reciprocalScale,
384 scaleMaskReg);384 scaleMaskReg);
385 385 
386- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::StoreDist::DIST_FIRST_ELEMENT_B32>(scaleLocalTmp, vreg5,386+ AscendC::MicroAPI::StoreAlign<float, AscendC::MicroAPI::StoreDist::DIST_FIRST_ELEMENT_B32>(scaleLocalTmp, vreg5,
387- preg0);387+ preg0);
388 }388 }
389}389}
390 390 
391template <typename T, typename U, int64_t RMode>391template <typename T, typename U, int64_t RMode>
392__aicore__ inline void GroupedDynamicBlockQuantLargeBlock<T, U, RMode>::ComputeOutVF(int64_t xTotalNum,392__aicore__ inline void GroupedDynamicBlockQuantLargeBlock<T, U, RMode>::ComputeOutVF(int64_t xTotalNum,
393- __local_mem__ T* xLocalAddr,393+ __ubuf__ T* xLocalAddr,
394- __local_mem__ float* scaleLocal,394+ __ubuf__ float* scaleLocal,
395- __local_mem__ U* outLocal)395+ __ubuf__ U* outLocal)
396{396{
397 uint32_t dtypeSize = sizeof(float);397 uint32_t dtypeSize = sizeof(float);
398 uint16_t VL = AscendC::VECTOR_REG_WIDTH / dtypeSize;398 uint16_t VL = AscendC::VECTOR_REG_WIDTH / dtypeSize;
@@ -414,16 +414,16 @@ __aicore__ inline void GroupedDynamicBlockQuantLargeBlock<T, U, RMode>::ComputeO
414 414 
415 preg0 = AscendC::MicroAPI::CreateMask<float, MicroAPI::MaskPattern::ALL>();415 preg0 = AscendC::MicroAPI::CreateMask<float, MicroAPI::MaskPattern::ALL>();
416 416 
417- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_BRC_B32>(vreg2, scaleLocal);417+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_BRC_B32>(vreg2, scaleLocal);
418 for (uint16_t i = 0; i < static_cast<uint16_t>(vfLoop); i++) {418 for (uint16_t i = 0; i < static_cast<uint16_t>(vfLoop); i++) {
419 preg0 = AscendC::MicroAPI::UpdateMask<float>(dataNum);419 preg0 = AscendC::MicroAPI::UpdateMask<float>(dataNum);
420- AscendC::MicroAPI::DataCopy<T, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(vreg1, xLocalAddr + i * VL);420+ AscendC::MicroAPI::LoadAlign<T, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(vreg1, xLocalAddr + i * VL);
421 AscendC::MicroAPI::Cast<float, T, castTraitT2Float>(vreg4, vreg1, preg0);421 AscendC::MicroAPI::Cast<float, T, castTraitT2Float>(vreg4, vreg1, preg0);
422 AscendC::MicroAPI::Div<float, &mode>(vreg5, vreg4, vreg2, preg0);422 AscendC::MicroAPI::Div<float, &mode>(vreg5, vreg4, vreg2, preg0);
423 423 
424 AscendC::MicroAPI::Cast<U, float, castTrait32to8>(outReg, vreg5, preg0);424 AscendC::MicroAPI::Cast<U, float, castTrait32to8>(outReg, vreg5, preg0);
425 425 
426- MicroAPI::DataCopy<U, MicroAPI::StoreDist::DIST_PACK4_B32>(outLocal + i * VL, outReg, preg0);426+ MicroAPI::StoreAlign<U, MicroAPI::StoreDist::DIST_PACK4_B32>(outLocal + i * VL, outReg, preg0);
427 }427 }
428 }428 }
429}429}
Mquant/grouped_dynamic_block_quant/op_kernel/arch35/grouped_dynamic_block_quant_small_block.h+5-4
@@ -401,7 +401,8 @@ __aicore__ inline void GroupedDynamicBlockQuantSmallBlock<T, U, RMode>::ComputeA
401 }401 }
402 pnum16 = normalBlockDataNum;402 pnum16 = normalBlockDataNum;
403 dataLenMask16 = AscendC::MicroAPI::UpdateMask<uint16_t>(pnum16);403 dataLenMask16 = AscendC::MicroAPI::UpdateMask<uint16_t>(pnum16);
404- AscendC::MicroAPI::ReduceMax<uint16_t>(expMaxRegTensor, expMaxRegTensor, dataLenMask16);404+ AscendC::MicroAPI::Reduce<MicroAPI::ReduceType::MAX, uint16_t>(expMaxRegTensor, expMaxRegTensor,
405+ dataLenMask16);
405 AscendC::MicroAPI::Cast<float, T, castTraitT2Float>(406 AscendC::MicroAPI::Cast<float, T, castTraitT2Float>(
406 expMaxRegTensorFp32, (AscendC::MicroAPI::RegTensor<T>&)expMaxRegTensor, dataLenMask16);407 expMaxRegTensorFp32, (AscendC::MicroAPI::RegTensor<T>&)expMaxRegTensor, dataLenMask16);
407 408 
@@ -409,7 +410,7 @@ __aicore__ inline void GroupedDynamicBlockQuantSmallBlock<T, U, RMode>::ComputeA
409 AscendC::MicroAPI::Duplicate(expMaxRegTensorFp32, expMaxRegTensorFp32, scaleMask32);410 AscendC::MicroAPI::Duplicate(expMaxRegTensorFp32, expMaxRegTensorFp32, scaleMask32);
410 AscendC::MicroAPI::Div((AscendC::MicroAPI::RegTensor<float>&)scaleRegTensor, expMaxRegTensorFp32,411 AscendC::MicroAPI::Div((AscendC::MicroAPI::RegTensor<float>&)scaleRegTensor, expMaxRegTensorFp32,
411 fp8MaxRegTensor, scaleMask32);412 fp8MaxRegTensor, scaleMask32);
412- AscendC::MicroAPI::CompareScalar<uint32_t, CMPMODE::LT>(413+ AscendC::MicroAPI::Compares<uint32_t, CMPMODE::LT>(
413 scaleMaskReg, (AscendC::MicroAPI::RegTensor<uint32_t>&)scaleRegTensor, infValue_, scaleMask32);414 scaleMaskReg, (AscendC::MicroAPI::RegTensor<uint32_t>&)scaleRegTensor, infValue_, scaleMask32);
414 // Min(input_max / FP_MAX, 1 / minScale)415 // Min(input_max / FP_MAX, 1 / minScale)
415 AscendC::MicroAPI::Min<float, AscendC::MicroAPI::MaskMergeMode::MERGING>(416 AscendC::MicroAPI::Min<float, AscendC::MicroAPI::MaskMergeMode::MERGING>(
@@ -449,7 +450,7 @@ __aicore__ inline void GroupedDynamicBlockQuantSmallBlock<T, U, RMode>::ComputeA
449 }450 }
450 pnum16 = tailBlockDataNum;451 pnum16 = tailBlockDataNum;
451 dataLenMask16 = AscendC::MicroAPI::UpdateMask<uint16_t>(pnum16);452 dataLenMask16 = AscendC::MicroAPI::UpdateMask<uint16_t>(pnum16);
452- AscendC::MicroAPI::ReduceMax<uint16_t>(expMaxRegTensor, expMaxRegTensor, dataLenMask16);453+ AscendC::MicroAPI::Reduce<MicroAPI::ReduceType::MAX, uint16_t>(expMaxRegTensor, expMaxRegTensor, dataLenMask16);
453 AscendC::MicroAPI::Cast<float, T, castTraitT2Float>(454 AscendC::MicroAPI::Cast<float, T, castTraitT2Float>(
454 expMaxRegTensorFp32, (AscendC::MicroAPI::RegTensor<T>&)expMaxRegTensor, dataLenMask16);455 expMaxRegTensorFp32, (AscendC::MicroAPI::RegTensor<T>&)expMaxRegTensor, dataLenMask16);
455 456 
@@ -457,7 +458,7 @@ __aicore__ inline void GroupedDynamicBlockQuantSmallBlock<T, U, RMode>::ComputeA
457 AscendC::MicroAPI::Duplicate(expMaxRegTensorFp32, expMaxRegTensorFp32, scaleMask32);458 AscendC::MicroAPI::Duplicate(expMaxRegTensorFp32, expMaxRegTensorFp32, scaleMask32);
458 AscendC::MicroAPI::Div((AscendC::MicroAPI::RegTensor<float>&)scaleRegTensor, expMaxRegTensorFp32,459 AscendC::MicroAPI::Div((AscendC::MicroAPI::RegTensor<float>&)scaleRegTensor, expMaxRegTensorFp32,
459 fp8MaxRegTensor, scaleMask32);460 fp8MaxRegTensor, scaleMask32);
460- AscendC::MicroAPI::CompareScalar<uint32_t, CMPMODE::LT>(461+ AscendC::MicroAPI::Compares<uint32_t, CMPMODE::LT>(
461 scaleMaskReg, (AscendC::MicroAPI::RegTensor<uint32_t>&)scaleRegTensor, infValue_, scaleMask32);462 scaleMaskReg, (AscendC::MicroAPI::RegTensor<uint32_t>&)scaleRegTensor, infValue_, scaleMask32);
462 // Min(input_max / FP_MAX, 1 / minScale)463 // Min(input_max / FP_MAX, 1 / minScale)
463 AscendC::MicroAPI::Min<float, AscendC::MicroAPI::MaskMergeMode::MERGING>(464 AscendC::MicroAPI::Min<float, AscendC::MicroAPI::MaskMergeMode::MERGING>(
Mquant/quant_common/op_kernel/mx_quant_common.h+17-17
@@ -199,13 +199,13 @@ __aicore__ inline void ComputeMxScaleCuBLAS(const int64_t dataLen, const uint16_
199 199 
200 Reg::ShiftRights(expMax0Reg, (Reg::RegTensor<uint32_t>&)xMax0Reg, SHR_NUM_FOR_FP32, maskAll);200 Reg::ShiftRights(expMax0Reg, (Reg::RegTensor<uint32_t>&)xMax0Reg, SHR_NUM_FOR_FP32, maskAll);
201 Reg::And(manMax0Reg, (Reg::RegTensor<uint32_t>&)xMax0Reg, manMaskReg, maskAll);201 Reg::And(manMax0Reg, (Reg::RegTensor<uint32_t>&)xMax0Reg, manMaskReg, maskAll);
202- Reg::CompareScalar<uint32_t, CMPMODE::GT>(p0, expMax0Reg, NUMBER_ZERO, maskAll);202+ Reg::Compares<uint32_t, CMPMODE::GT>(p0, expMax0Reg, NUMBER_ZERO, maskAll);
203- Reg::CompareScalar<uint32_t, CMPMODE::LT>(p0, expMax0Reg, NUMBER_TWO_FIVE_FOUR, p0);203+ Reg::Compares<uint32_t, CMPMODE::LT>(p0, expMax0Reg, NUMBER_TWO_FIVE_FOUR, p0);
204- Reg::CompareScalar<uint32_t, CMPMODE::GT>(p0, manMax0Reg, NUMBER_ZERO, p0);204+ Reg::Compares<uint32_t, CMPMODE::GT>(p0, manMax0Reg, NUMBER_ZERO, p0);
205 if constexpr (scaleAlg == TPL_SCALE_ALG_1) {205 if constexpr (scaleAlg == TPL_SCALE_ALG_1) {
206- Reg::CompareScalar<uint32_t, CMPMODE::EQ>(p1, expMax0Reg, NUMBER_ZERO, maskAll);206+ Reg::Compares<uint32_t, CMPMODE::EQ>(p1, expMax0Reg, NUMBER_ZERO, maskAll);
207- Reg::CompareScalar<uint32_t, CMPMODE::GT>(p1, manMax0Reg, NUMBER_HALF, p1);207+ Reg::Compares<uint32_t, CMPMODE::GT>(p1, manMax0Reg, NUMBER_HALF, p1);
208- Reg::MaskXor(p0, p0, p1, maskAll);208+ Reg::Xor(p0, p0, p1, maskAll);
209 }209 }
210 Reg::Adds(manMax0Reg, expMax0Reg, 1, maskAll);210 Reg::Adds(manMax0Reg, expMax0Reg, 1, maskAll);
211 Reg::Select(expMax0Reg, manMax0Reg, expMax0Reg, p0);211 Reg::Select(expMax0Reg, manMax0Reg, expMax0Reg, p0);
@@ -213,13 +213,13 @@ __aicore__ inline void ComputeMxScaleCuBLAS(const int64_t dataLen, const uint16_
213 213 
214 Reg::ShiftRights(expMax1Reg, (Reg::RegTensor<uint32_t>&)xMax1Reg, SHR_NUM_FOR_FP32, maskAll);214 Reg::ShiftRights(expMax1Reg, (Reg::RegTensor<uint32_t>&)xMax1Reg, SHR_NUM_FOR_FP32, maskAll);
215 Reg::And(manMax1Reg, (Reg::RegTensor<uint32_t>&)xMax1Reg, manMaskReg, maskAll);215 Reg::And(manMax1Reg, (Reg::RegTensor<uint32_t>&)xMax1Reg, manMaskReg, maskAll);
216- Reg::CompareScalar<uint32_t, CMPMODE::GT>(p2, expMax1Reg, NUMBER_ZERO, maskAll);216+ Reg::Compares<uint32_t, CMPMODE::GT>(p2, expMax1Reg, NUMBER_ZERO, maskAll);
217- Reg::CompareScalar<uint32_t, CMPMODE::LT>(p2, expMax1Reg, NUMBER_TWO_FIVE_FOUR, p2);217+ Reg::Compares<uint32_t, CMPMODE::LT>(p2, expMax1Reg, NUMBER_TWO_FIVE_FOUR, p2);
218- Reg::CompareScalar<uint32_t, CMPMODE::GT>(p2, manMax1Reg, NUMBER_ZERO, p2);218+ Reg::Compares<uint32_t, CMPMODE::GT>(p2, manMax1Reg, NUMBER_ZERO, p2);
219 if constexpr (scaleAlg == TPL_SCALE_ALG_1) {219 if constexpr (scaleAlg == TPL_SCALE_ALG_1) {
220- Reg::CompareScalar<uint32_t, CMPMODE::EQ>(p3, expMax1Reg, NUMBER_ZERO, maskAll);220+ Reg::Compares<uint32_t, CMPMODE::EQ>(p3, expMax1Reg, NUMBER_ZERO, maskAll);
221- Reg::CompareScalar<uint32_t, CMPMODE::GT>(p3, manMax1Reg, NUMBER_HALF, p3);221+ Reg::Compares<uint32_t, CMPMODE::GT>(p3, manMax1Reg, NUMBER_HALF, p3);
222- Reg::MaskXor(p2, p2, p3, maskAll);222+ Reg::Xor(p2, p2, p3, maskAll);
223 }223 }
224 Reg::Adds(manMax1Reg, expMax1Reg, 1, maskAll);224 Reg::Adds(manMax1Reg, expMax1Reg, 1, maskAll);
225 Reg::Select(expMax1Reg, manMax1Reg, expMax1Reg, p2);225 Reg::Select(expMax1Reg, manMax1Reg, expMax1Reg, p2);
@@ -339,7 +339,7 @@ __aicore__ inline void ComputeFP4FromFp32(Reg::RegTensor<float>& Reg)
339 Reg::Compare<int32_t, CMPMODE::EQ>(negInfMask, (Reg::RegTensor<int32_t>&)Reg, negZero, pregAll32);339 Reg::Compare<int32_t, CMPMODE::EQ>(negInfMask, (Reg::RegTensor<int32_t>&)Reg, negZero, pregAll32);
340 if constexpr (IsSameType<U, fp4x2_e1m2_t>::value) {340 if constexpr (IsSameType<U, fp4x2_e1m2_t>::value) {
341 Reg::Muls(Reg, Reg, FP4_SCALE_FACTOR, pregAll32);341 Reg::Muls(Reg, Reg, FP4_SCALE_FACTOR, pregAll32);
342- Reg::CompareScalar<float, CMPMODE::LT>(specialMask, Reg, 0, pregAll32);342+ Reg::Compares<float, CMPMODE::LT>(specialMask, Reg, 0, pregAll32);
343 Reg::Truncate<float, roundMode>(Reg, Reg, pregAll32);343 Reg::Truncate<float, roundMode>(Reg, Reg, pregAll32);
344 Reg::Muls(Reg, Reg, FP4_INV_SCALE_FACTOR, pregAll32);344 Reg::Muls(Reg, Reg, FP4_INV_SCALE_FACTOR, pregAll32);
345 } else {345 } else {
@@ -356,13 +356,13 @@ __aicore__ inline void ComputeFP4FromFp32(Reg::RegTensor<float>& Reg)
356 Reg::Mul(Reg, Reg, (Reg::RegTensor<float>&)exp1FP32, pregAll32);356 Reg::Mul(Reg, Reg, (Reg::RegTensor<float>&)exp1FP32, pregAll32);
357 Reg::Adds(exp0FP32, exp0FP32, FP32_BIAS_VALUE, pregAll32);357 Reg::Adds(exp0FP32, exp0FP32, FP32_BIAS_VALUE, pregAll32);
358 Reg::ShiftLefts(exp0FP32, exp0FP32, FP32_SHR_NUM_VAL, pregAll32);358 Reg::ShiftLefts(exp0FP32, exp0FP32, FP32_SHR_NUM_VAL, pregAll32);
359- Reg::CompareScalar<float, CMPMODE::LT>(specialMask, Reg, 0, pregAll32);359+ Reg::Compares<float, CMPMODE::LT>(specialMask, Reg, 0, pregAll32);
360 Reg::Truncate<float, roundMode>(Reg, Reg, pregAll32);360 Reg::Truncate<float, roundMode>(Reg, Reg, pregAll32);
361 Reg::Mul(Reg, Reg, (Reg::RegTensor<float>&)exp0FP32, pregAll32);361 Reg::Mul(Reg, Reg, (Reg::RegTensor<float>&)exp0FP32, pregAll32);
362 }362 }
363- Reg::CompareScalar<float, CMPMODE::EQ>(zeroMask, Reg, 0, pregAll32);363+ Reg::Compares<float, CMPMODE::EQ>(zeroMask, Reg, 0, pregAll32);
364- Reg::MaskAnd(zeroMask, specialMask, zeroMask, pregAll32);364+ Reg::And(zeroMask, specialMask, zeroMask, pregAll32);
365- Reg::MaskOr(zeroMask, negInfMask, zeroMask, pregAll32);365+ Reg::Or(zeroMask, negInfMask, zeroMask, pregAll32);
366 Reg::Select<int32_t>((Reg::RegTensor<int32_t>&)Reg, negZero, (Reg::RegTensor<int32_t>&)Reg, zeroMask);366 Reg::Select<int32_t>((Reg::RegTensor<int32_t>&)Reg, negZero, (Reg::RegTensor<int32_t>&)Reg, zeroMask);
367}367}
368 368 
Mquant/quantize/op_kernel/arch35/quantize_per_channel_nddma_no_offset_regbase.h+33-33
@@ -114,8 +114,8 @@ __aicore__ inline void QuantizePerChannelNddmaNoOffsetRegbase<T, T1, T2, U, DivM
114 int64_t sLen, int64_t sInOffset)114 int64_t sLen, int64_t sInOffset)
115{115{
116 LocalTensor<T1> sLocal = inQueueScale_.AllocTensor<T1>();116 LocalTensor<T1> sLocal = inQueueScale_.AllocTensor<T1>();
117- static constexpr AscendC::MultiCopyConfig copyConfig = {false, 0, 0, false};117+ static constexpr AscendC::NdDmaConfig copyConfig = {false, 0, 0, false};
118- MultiCopyLoopInfo<QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::MULTI_COPY_DIM> copyLoopInfo;118+ NdDmaLoopInfo<QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::MULTI_COPY_DIM> copyLoopInfo;
119 copyLoopInfo.loopSrcStride[0] = 1;119 copyLoopInfo.loopSrcStride[0] = 1;
120 copyLoopInfo.loopSrcStride[1] = 0;120 copyLoopInfo.loopSrcStride[1] = 0;
121 copyLoopInfo.loopDstStride[0] = 1;121 copyLoopInfo.loopDstStride[0] = 1;
@@ -124,8 +124,8 @@ __aicore__ inline void QuantizePerChannelNddmaNoOffsetRegbase<T, T1, T2, U, DivM
124 copyLoopInfo.loopSize[1] = tilingData_.baseN;124 copyLoopInfo.loopSize[1] = tilingData_.baseN;
125 125 
126 T1 constValue = 0;126 T1 constValue = 0;
127- AscendC::MultiCopyParams<T1, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::MULTI_COPY_DIM> copyParams =127+ AscendC::NdDmaParams<T1, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::MULTI_COPY_DIM> copyParams = {
128- {copyLoopInfo, constValue};128+ copyLoopInfo, constValue};
129 AscendC::DataCopy<T1, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::MULTI_COPY_DIM, copyConfig>(129 AscendC::DataCopy<T1, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::MULTI_COPY_DIM, copyConfig>(
130 sLocal, scaleGm_[sInOffset], copyParams);130 sLocal, scaleGm_[sInOffset], copyParams);
131 inQueueScale_.EnQue(sLocal);131 inQueueScale_.EnQue(sLocal);
@@ -176,9 +176,9 @@ __aicore__ inline void QuantizePerChannelNddmaNoOffsetRegbase<T, T1, T2, U, DivM
176 LocalTensor<T> xLocal = inQueueX_.DeQue<T>();176 LocalTensor<T> xLocal = inQueueX_.DeQue<T>();
177 LocalTensor<yCopyDtype> outLocal = outQueueY_.AllocTensor<yCopyDtype>();177 LocalTensor<yCopyDtype> outLocal = outQueueY_.AllocTensor<yCopyDtype>();
178 178 
179- __local_mem__ T* xLocalAddr = (__local_mem__ T*)xLocal.GetPhyAddr();179+ __ubuf__ T* xLocalAddr = (__ubuf__ T*)xLocal.GetPhyAddr();
180- __local_mem__ T1* scaleLocalAddr = (__local_mem__ T1*)sLocal.GetPhyAddr();180+ __ubuf__ T1* scaleLocalAddr = (__ubuf__ T1*)sLocal.GetPhyAddr();
181- __local_mem__ yCopyDtype* outLocalAddr = (__local_mem__ yCopyDtype*)outLocal.GetPhyAddr();181+ __ubuf__ yCopyDtype* outLocalAddr = (__ubuf__ yCopyDtype*)outLocal.GetPhyAddr();
182 182 
183 uint16_t VL = AscendC::VECTOR_REG_WIDTH / sizeof(float);183 uint16_t VL = AscendC::VECTOR_REG_WIDTH / sizeof(float);
184 184 
@@ -207,19 +207,19 @@ __aicore__ inline void QuantizePerChannelNddmaNoOffsetRegbase<T, T1, T2, U, DivM
207 // ld for x207 // ld for x
208 if constexpr (IsSameType<T, float>::value) {208 if constexpr (IsSameType<T, float>::value) {
209 // fp32209 // fp32
210- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(vregFloatX,210+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(vregFloatX,
211- xLocalAddr + i * VL);211+ xLocalAddr + i * VL);
212 } else if constexpr (IsSameType<T, half>::value) {212 } else if constexpr (IsSameType<T, half>::value) {
213 // fp16213 // fp16
214- AscendC::MicroAPI::DataCopy<half, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(vregX,214+ AscendC::MicroAPI::LoadAlign<half, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(vregX,
215- xLocalAddr + i * VL);215+ xLocalAddr + i * VL);
216 AscendC::MicroAPI::Cast<216 AscendC::MicroAPI::Cast<
217 float, half, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_HALF_TO_FP32>(217 float, half, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_HALF_TO_FP32>(
218 vregFloatX, vregX, mask);218 vregFloatX, vregX, mask);
219 } else if constexpr (IsSameType<T, bfloat16_t>::value) {219 } else if constexpr (IsSameType<T, bfloat16_t>::value) {
220 // bf16220 // bf16
221- AscendC::MicroAPI::DataCopy<T, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(vregX,221+ AscendC::MicroAPI::LoadAlign<T, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(vregX,
222- xLocalAddr + i * VL);222+ xLocalAddr + i * VL);
223 AscendC::MicroAPI::Cast<223 AscendC::MicroAPI::Cast<
224 float, T, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_BF16_TO_FP32>(224 float, T, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_BF16_TO_FP32>(
225 vregFloatX, vregX, mask);225 vregFloatX, vregX, mask);
@@ -228,19 +228,19 @@ __aicore__ inline void QuantizePerChannelNddmaNoOffsetRegbase<T, T1, T2, U, DivM
228 // ld for scale228 // ld for scale
229 if constexpr (IsSameType<T1, float>::value) {229 if constexpr (IsSameType<T1, float>::value) {
230 // fp32230 // fp32
231- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(vregFloatS,231+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(vregFloatS,
232- scaleLocalAddr + i * VL);232+ scaleLocalAddr + i * VL);
233 } else if (IsSameType<T1, half>::value) {233 } else if (IsSameType<T1, half>::value) {
234 // fp16234 // fp16
235- AscendC::MicroAPI::DataCopy<T1, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(vregS,235+ AscendC::MicroAPI::LoadAlign<T1, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(vregS,
236- scaleLocalAddr + i * VL);236+ scaleLocalAddr + i * VL);
237 AscendC::MicroAPI::Cast<237 AscendC::MicroAPI::Cast<
238 float, T1, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_HALF_TO_FP32>(238 float, T1, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_HALF_TO_FP32>(
239 vregFloatS, vregS, mask);239 vregFloatS, vregS, mask);
240 } else if (IsSameType<T1, bfloat16_t>::value) {240 } else if (IsSameType<T1, bfloat16_t>::value) {
241 // bf16241 // bf16
242- AscendC::MicroAPI::DataCopy<T1, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(vregS,242+ AscendC::MicroAPI::LoadAlign<T1, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(vregS,
243- scaleLocalAddr + i * VL);243+ scaleLocalAddr + i * VL);
244 AscendC::MicroAPI::Cast<244 AscendC::MicroAPI::Cast<
245 float, T1, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_BF16_TO_FP32>(245 float, T1, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_BF16_TO_FP32>(
246 vregFloatS, vregS, mask);246 vregFloatS, vregS, mask);
@@ -262,22 +262,22 @@ __aicore__ inline void QuantizePerChannelNddmaNoOffsetRegbase<T, T1, T2, U, DivM
262 AscendC::MicroAPI::Cast<262 AscendC::MicroAPI::Cast<
263 U, float, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_FP32_TO_HIFP8>(263 U, float, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_FP32_TO_HIFP8>(
264 vregY, vregFloatY, mask);264 vregY, vregFloatY, mask);
265- AscendC::MicroAPI::DataCopy<U, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(outLocalAddr + i * VL,265+ AscendC::MicroAPI::StoreAlign<U, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(outLocalAddr + i * VL,
266- vregY, mask);266+ vregY, mask);
267 } else if constexpr (IsSameType<U, fp8_e5m2_t>::value) {267 } else if constexpr (IsSameType<U, fp8_e5m2_t>::value) {
268 // fp8_e5m2268 // fp8_e5m2
269 AscendC::MicroAPI::Cast<269 AscendC::MicroAPI::Cast<
270 U, float, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_FP32_TO_FP8E5M2>(270 U, float, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_FP32_TO_FP8E5M2>(
271 vregY, vregFloatY, mask);271 vregY, vregFloatY, mask);
272- AscendC::MicroAPI::DataCopy<U, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(outLocalAddr + i * VL,272+ AscendC::MicroAPI::StoreAlign<U, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(outLocalAddr + i * VL,
273- vregY, mask);273+ vregY, mask);
274 } else if constexpr (IsSameType<U, fp8_e4m3fn_t>::value) {274 } else if constexpr (IsSameType<U, fp8_e4m3fn_t>::value) {
275 // fp8_e4m3275 // fp8_e4m3
276 AscendC::MicroAPI::Cast<276 AscendC::MicroAPI::Cast<
277 U, float, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_FP32_TO_FP8E4M3>(277 U, float, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_FP32_TO_FP8E4M3>(
278 vregY, vregFloatY, mask);278 vregY, vregFloatY, mask);
279- AscendC::MicroAPI::DataCopy<U, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(outLocalAddr + i * VL,279+ AscendC::MicroAPI::StoreAlign<U, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(outLocalAddr + i * VL,
280- vregY, mask);280+ vregY, mask);
281 } else if constexpr (IsSameType<U, int8_t>::value) {281 } else if constexpr (IsSameType<U, int8_t>::value) {
282 // int8, float->int16->half->int8282 // int8, float->int16->half->int8
283 AscendC::MicroAPI::Cast<283 AscendC::MicroAPI::Cast<
@@ -289,8 +289,8 @@ __aicore__ inline void QuantizePerChannelNddmaNoOffsetRegbase<T, T1, T2, U, DivM
289 AscendC::MicroAPI::Cast<289 AscendC::MicroAPI::Cast<
290 int8_t, half, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_HALF_TO_INT8>(290 int8_t, half, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_HALF_TO_INT8>(
291 vregY, vregHalfY, mask);291 vregY, vregHalfY, mask);
292- AscendC::MicroAPI::DataCopy<U, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(outLocalAddr + i * VL,292+ AscendC::MicroAPI::StoreAlign<U, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(outLocalAddr + i * VL,
293- vregY, mask);293+ vregY, mask);
294 } else if constexpr (IsSameType<U, uint8_t>::value) {294 } else if constexpr (IsSameType<U, uint8_t>::value) {
295 // uint8, float->int16->half->int8295 // uint8, float->int16->half->int8
296 AscendC::MicroAPI::Cast<296 AscendC::MicroAPI::Cast<
@@ -302,15 +302,15 @@ __aicore__ inline void QuantizePerChannelNddmaNoOffsetRegbase<T, T1, T2, U, DivM
302 AscendC::MicroAPI::Cast<302 AscendC::MicroAPI::Cast<
303 uint8_t, half, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_HALF_TO_UINT8>(303 uint8_t, half, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_HALF_TO_UINT8>(
304 vregY, vregHalfY, mask);304 vregY, vregHalfY, mask);
305- AscendC::MicroAPI::DataCopy<U, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(outLocalAddr + i * VL,305+ AscendC::MicroAPI::StoreAlign<U, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(outLocalAddr + i * VL,
306- vregY, mask);306+ vregY, mask);
307 } else if constexpr (IsSameType<U, int32_t>::value) {307 } else if constexpr (IsSameType<U, int32_t>::value) {
308 // int32308 // int32
309 AscendC::MicroAPI::Cast<309 AscendC::MicroAPI::Cast<
310 U, float, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_FP32_TO_INT32>(310 U, float, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_FP32_TO_INT32>(
311 vregY, vregFloatY, mask);311 vregY, vregFloatY, mask);
312- AscendC::MicroAPI::DataCopy<U, AscendC::MicroAPI::StoreDist::DIST_NORM>(outLocalAddr + i * VL, vregY,312+ AscendC::MicroAPI::StoreAlign<U, AscendC::MicroAPI::StoreDist::DIST_NORM>(outLocalAddr + i * VL, vregY,
313- mask);313+ mask);
314 } else if constexpr (IsSameType<U, int4b_t>::value) {314 } else if constexpr (IsSameType<U, int4b_t>::value) {
315 AscendC::MicroAPI::RegTensor<int16_t> vregInt16Y;315 AscendC::MicroAPI::RegTensor<int16_t> vregInt16Y;
316 AscendC::MicroAPI::RegTensor<uint16_t> vregTmp1Y;316 AscendC::MicroAPI::RegTensor<uint16_t> vregTmp1Y;
@@ -327,7 +327,7 @@ __aicore__ inline void QuantizePerChannelNddmaNoOffsetRegbase<T, T1, T2, U, DivM
327 QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_F16_TO_I8>(327 QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_F16_TO_I8>(
328 (AscendC::MicroAPI::RegTensor<int4x2_t>&)vregTmp2Y, (AscendC::MicroAPI::RegTensor<half>&)vregTmp1Y,328 (AscendC::MicroAPI::RegTensor<int4x2_t>&)vregTmp2Y, (AscendC::MicroAPI::RegTensor<half>&)vregTmp1Y,
329 mask);329 mask);
330- AscendC::MicroAPI::DataCopy<yCopyDtype, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(330+ AscendC::MicroAPI::StoreAlign<yCopyDtype, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(
331 outLocalAddr + (i * VL / 2), vregTmp2Y, mask4Int4);331 outLocalAddr + (i * VL / 2), vregTmp2Y, mask4Int4);
332 }332 }
333 }333 }
@@ -355,4 +355,4 @@ __aicore__ inline void QuantizePerChannelNddmaNoOffsetRegbase<T, T1, T2, U, DivM
355 outQueueY_.FreeTensor(outLocal);355 outQueueY_.FreeTensor(outLocal);
356}356}
357} // namespace QuantizeOp357} // namespace QuantizeOp
358-#endif358+#endif
Mquant/quantize/op_kernel/arch35/quantize_per_channel_nddma_regbase.h+49-49
@@ -126,8 +126,8 @@ __aicore__ inline void QuantizePerChannelNddmaRegbase<T, T1, T2, U, DivMode, Rou
126 int64_t sLen, int64_t sInOffset)126 int64_t sLen, int64_t sInOffset)
127{127{
128 LocalTensor<T1> sLocal = inQueueScale_.AllocTensor<T1>();128 LocalTensor<T1> sLocal = inQueueScale_.AllocTensor<T1>();
129- static constexpr AscendC::MultiCopyConfig copyConfig = {false, 0, 0, false};129+ static constexpr AscendC::NdDmaConfig copyConfig = {false, 0, 0, false};
130- MultiCopyLoopInfo<QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::MULTI_COPY_DIM> copyLoopInfo;130+ NdDmaLoopInfo<QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::MULTI_COPY_DIM> copyLoopInfo;
131 copyLoopInfo.loopSrcStride[0] = 1;131 copyLoopInfo.loopSrcStride[0] = 1;
132 copyLoopInfo.loopSrcStride[1] = 0;132 copyLoopInfo.loopSrcStride[1] = 0;
133 copyLoopInfo.loopDstStride[0] = 1;133 copyLoopInfo.loopDstStride[0] = 1;
@@ -136,8 +136,8 @@ __aicore__ inline void QuantizePerChannelNddmaRegbase<T, T1, T2, U, DivMode, Rou
136 copyLoopInfo.loopSize[1] = tilingData_.baseN;136 copyLoopInfo.loopSize[1] = tilingData_.baseN;
137 137 
138 T1 constValue = 0;138 T1 constValue = 0;
139- AscendC::MultiCopyParams<T1, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::MULTI_COPY_DIM> copyParams =139+ AscendC::NdDmaParams<T1, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::MULTI_COPY_DIM> copyParams = {
140- {copyLoopInfo, constValue};140+ copyLoopInfo, constValue};
141 AscendC::DataCopy<T1, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::MULTI_COPY_DIM, copyConfig>(141 AscendC::DataCopy<T1, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::MULTI_COPY_DIM, copyConfig>(
142 sLocal, scaleGm_[sInOffset], copyParams);142 sLocal, scaleGm_[sInOffset], copyParams);
143 inQueueScale_.EnQue(sLocal);143 inQueueScale_.EnQue(sLocal);
@@ -148,8 +148,8 @@ __aicore__ inline void QuantizePerChannelNddmaRegbase<T, T1, T2, U, DivMode, Rou
148 int64_t sLen, int64_t sInOffset)148 int64_t sLen, int64_t sInOffset)
149{149{
150 LocalTensor<T2> oLocal = inQueueOffset_.AllocTensor<T2>();150 LocalTensor<T2> oLocal = inQueueOffset_.AllocTensor<T2>();
151- static constexpr AscendC::MultiCopyConfig copyConfig = {false, 0, 0, false};151+ static constexpr AscendC::NdDmaConfig copyConfig = {false, 0, 0, false};
152- MultiCopyLoopInfo<QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::MULTI_COPY_DIM> copyLoopInfo;152+ NdDmaLoopInfo<QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::MULTI_COPY_DIM> copyLoopInfo;
153 copyLoopInfo.loopSrcStride[0] = 1;153 copyLoopInfo.loopSrcStride[0] = 1;
154 copyLoopInfo.loopSrcStride[1] = 0;154 copyLoopInfo.loopSrcStride[1] = 0;
155 copyLoopInfo.loopDstStride[0] = 1;155 copyLoopInfo.loopDstStride[0] = 1;
@@ -158,8 +158,8 @@ __aicore__ inline void QuantizePerChannelNddmaRegbase<T, T1, T2, U, DivMode, Rou
158 copyLoopInfo.loopSize[1] = tilingData_.baseN;158 copyLoopInfo.loopSize[1] = tilingData_.baseN;
159 159 
160 T2 constValue = 0;160 T2 constValue = 0;
161- AscendC::MultiCopyParams<T2, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::MULTI_COPY_DIM> copyParams =161+ AscendC::NdDmaParams<T2, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::MULTI_COPY_DIM> copyParams = {
162- {copyLoopInfo, constValue};162+ copyLoopInfo, constValue};
163 AscendC::DataCopy<T2, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::MULTI_COPY_DIM, copyConfig>(163 AscendC::DataCopy<T2, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::MULTI_COPY_DIM, copyConfig>(
164 oLocal, offsetGm_[sInOffset], copyParams);164 oLocal, offsetGm_[sInOffset], copyParams);
165 inQueueOffset_.EnQue(oLocal);165 inQueueOffset_.EnQue(oLocal);
@@ -208,12 +208,12 @@ __aicore__ inline void QuantizePerChannelNddmaRegbase<T, T1, T2, U, DivMode, Rou
208 LocalTensor<T> xLocal = inQueueX_.DeQue<T>();208 LocalTensor<T> xLocal = inQueueX_.DeQue<T>();
209 LocalTensor<yCopyDtype> outLocal = outQueueY_.AllocTensor<yCopyDtype>();209 LocalTensor<yCopyDtype> outLocal = outQueueY_.AllocTensor<yCopyDtype>();
210 210 
211- __local_mem__ T* xLocalAddr = (__local_mem__ T*)xLocal.GetPhyAddr();211+ __ubuf__ T* xLocalAddr = (__ubuf__ T*)xLocal.GetPhyAddr();
212- __local_mem__ T1* scaleLocalAddr = (__local_mem__ T1*)sLocal.GetPhyAddr();212+ __ubuf__ T1* scaleLocalAddr = (__ubuf__ T1*)sLocal.GetPhyAddr();
213- __local_mem__ yCopyDtype* outLocalAddr = (__local_mem__ yCopyDtype*)outLocal.GetPhyAddr();213+ __ubuf__ yCopyDtype* outLocalAddr = (__ubuf__ yCopyDtype*)outLocal.GetPhyAddr();
214 214 
215 uint16_t VL = AscendC::VECTOR_REG_WIDTH / sizeof(float);215 uint16_t VL = AscendC::VECTOR_REG_WIDTH / sizeof(float);
216- __local_mem__ T2* offsetLocalAddr = (__local_mem__ T2*)oLocal.GetPhyAddr();216+ __ubuf__ T2* offsetLocalAddr = (__ubuf__ T2*)oLocal.GetPhyAddr();
217 217 
218 // has offset218 // has offset
219 __VEC_SCOPE__219 __VEC_SCOPE__
@@ -247,19 +247,19 @@ __aicore__ inline void QuantizePerChannelNddmaRegbase<T, T1, T2, U, DivMode, Rou
247 // ld and cast for x247 // ld and cast for x
248 if constexpr (IsSameType<T, float>::value) {248 if constexpr (IsSameType<T, float>::value) {
249 // fp32249 // fp32
250- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(vregFloatX,250+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(vregFloatX,
251- xLocalAddr + i * VL);251+ xLocalAddr + i * VL);
252 } else if constexpr (IsSameType<T, half>::value) {252 } else if constexpr (IsSameType<T, half>::value) {
253 // fp16253 // fp16
254- AscendC::MicroAPI::DataCopy<half, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(vregX,254+ AscendC::MicroAPI::LoadAlign<half, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(vregX,
255- xLocalAddr + i * VL);255+ xLocalAddr + i * VL);
256 AscendC::MicroAPI::Cast<256 AscendC::MicroAPI::Cast<
257 float, half, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_HALF_TO_FP32>(257 float, half, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_HALF_TO_FP32>(
258 vregFloatX, vregX, mask);258 vregFloatX, vregX, mask);
259 } else if constexpr (IsSameType<T, bfloat16_t>::value) {259 } else if constexpr (IsSameType<T, bfloat16_t>::value) {
260 // bf16260 // bf16
261- AscendC::MicroAPI::DataCopy<T, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(vregX,261+ AscendC::MicroAPI::LoadAlign<T, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(vregX,
262- xLocalAddr + i * VL);262+ xLocalAddr + i * VL);
263 AscendC::MicroAPI::Cast<263 AscendC::MicroAPI::Cast<
264 float, T, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_BF16_TO_FP32>(264 float, T, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_BF16_TO_FP32>(
265 vregFloatX, vregX, mask);265 vregFloatX, vregX, mask);
@@ -268,19 +268,19 @@ __aicore__ inline void QuantizePerChannelNddmaRegbase<T, T1, T2, U, DivMode, Rou
268 // ld and cast for scale268 // ld and cast for scale
269 if constexpr (IsSameType<T1, float>::value) {269 if constexpr (IsSameType<T1, float>::value) {
270 // fp32270 // fp32
271- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(vregFloatS,271+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(vregFloatS,
272- scaleLocalAddr + i * VL);272+ scaleLocalAddr + i * VL);
273 } else if constexpr (IsSameType<T1, half>::value) {273 } else if constexpr (IsSameType<T1, half>::value) {
274 // fp16274 // fp16
275- AscendC::MicroAPI::DataCopy<half, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(275+ AscendC::MicroAPI::LoadAlign<half, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(
276 vregS, scaleLocalAddr + i * VL);276 vregS, scaleLocalAddr + i * VL);
277 AscendC::MicroAPI::Cast<277 AscendC::MicroAPI::Cast<
278 float, T1, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_HALF_TO_FP32>(278 float, T1, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_HALF_TO_FP32>(
279 vregFloatS, vregS, mask);279 vregFloatS, vregS, mask);
280 } else if constexpr (IsSameType<T1, bfloat16_t>::value) {280 } else if constexpr (IsSameType<T1, bfloat16_t>::value) {
281 // bf16281 // bf16
282- AscendC::MicroAPI::DataCopy<T1, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(vregS,282+ AscendC::MicroAPI::LoadAlign<T1, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(vregS,
283- scaleLocalAddr + i * VL);283+ scaleLocalAddr + i * VL);
284 AscendC::MicroAPI::Cast<284 AscendC::MicroAPI::Cast<
285 float, T1, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_BF16_TO_FP32>(285 float, T1, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_BF16_TO_FP32>(
286 vregFloatS, vregS, mask);286 vregFloatS, vregS, mask);
@@ -288,15 +288,15 @@ __aicore__ inline void QuantizePerChannelNddmaRegbase<T, T1, T2, U, DivMode, Rou
288 // ld and cast for offset288 // ld and cast for offset
289 if constexpr (IsSameType<T2, int32_t>::value) {289 if constexpr (IsSameType<T2, int32_t>::value) {
290 // int32290 // int32
291- AscendC::MicroAPI::DataCopy<T2, AscendC::MicroAPI::LoadDist::DIST_NORM>(vregO,291+ AscendC::MicroAPI::LoadAlign<T2, AscendC::MicroAPI::LoadDist::DIST_NORM>(vregO,
292- offsetLocalAddr + i * VL);292+ offsetLocalAddr + i * VL);
293 AscendC::MicroAPI::Cast<293 AscendC::MicroAPI::Cast<
294 float, T2, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_INT32_TO_FP32>(294 float, T2, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_INT32_TO_FP32>(
295 vregFloatO, vregO, mask);295 vregFloatO, vregO, mask);
296 } else if constexpr (IsSameType<T2, int8_t>::value) {296 } else if constexpr (IsSameType<T2, int8_t>::value) {
297 // int8297 // int8
298- AscendC::MicroAPI::DataCopy<T2, AscendC::MicroAPI::LoadDist::DIST_UNPACK4_B8>(vregO,298+ AscendC::MicroAPI::LoadAlign<T2, AscendC::MicroAPI::LoadDist::DIST_UNPACK4_B8>(
299- offsetLocalAddr + i * VL);299+ vregO, offsetLocalAddr + i * VL);
300 AscendC::MicroAPI::Cast<300 AscendC::MicroAPI::Cast<
301 half, T2, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_INT8_TO_HALF>(301 half, T2, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_INT8_TO_HALF>(
302 vregHalfO, vregO, mask);302 vregHalfO, vregO, mask);
@@ -305,8 +305,8 @@ __aicore__ inline void QuantizePerChannelNddmaRegbase<T, T1, T2, U, DivMode, Rou
305 vregFloatO, vregHalfO, mask);305 vregFloatO, vregHalfO, mask);
306 } else if constexpr (IsSameType<T2, uint8_t>::value) {306 } else if constexpr (IsSameType<T2, uint8_t>::value) {
307 // uint8307 // uint8
308- AscendC::MicroAPI::DataCopy<T2, AscendC::MicroAPI::LoadDist::DIST_UNPACK4_B8>(vregO,308+ AscendC::MicroAPI::LoadAlign<T2, AscendC::MicroAPI::LoadDist::DIST_UNPACK4_B8>(
309- offsetLocalAddr + i * VL);309+ vregO, offsetLocalAddr + i * VL);
310 AscendC::MicroAPI::Cast<310 AscendC::MicroAPI::Cast<
311 half, T2, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_UINT8_TO_HALF>(311 half, T2, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_UINT8_TO_HALF>(
312 vregHalfO, vregO, mask);312 vregHalfO, vregO, mask);
@@ -315,22 +315,22 @@ __aicore__ inline void QuantizePerChannelNddmaRegbase<T, T1, T2, U, DivMode, Rou
315 vregFloatO, vregHalfO, mask);315 vregFloatO, vregHalfO, mask);
316 } else if constexpr (IsSameType<T2, bfloat16_t>::value) {316 } else if constexpr (IsSameType<T2, bfloat16_t>::value) {
317 // bf16317 // bf16
318- AscendC::MicroAPI::DataCopy<T2, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(vregO,318+ AscendC::MicroAPI::LoadAlign<T2, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(
319- offsetLocalAddr + i * VL);319+ vregO, offsetLocalAddr + i * VL);
320 AscendC::MicroAPI::Cast<320 AscendC::MicroAPI::Cast<
321 float, T2, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_BF16_TO_FP32>(321 float, T2, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_BF16_TO_FP32>(
322 vregFloatO, vregO, mask);322 vregFloatO, vregO, mask);
323 } else if constexpr (IsSameType<T2, half>::value) {323 } else if constexpr (IsSameType<T2, half>::value) {
324 // fp16324 // fp16
325- AscendC::MicroAPI::DataCopy<T2, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(vregO,325+ AscendC::MicroAPI::LoadAlign<T2, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(
326- offsetLocalAddr + i * VL);326+ vregO, offsetLocalAddr + i * VL);
327 AscendC::MicroAPI::Cast<327 AscendC::MicroAPI::Cast<
328 float, T2, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_HALF_TO_FP32>(328 float, T2, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_HALF_TO_FP32>(
329 vregFloatO, vregO, mask);329 vregFloatO, vregO, mask);
330 } else if constexpr (IsSameType<T2, float>::value) {330 } else if constexpr (IsSameType<T2, float>::value) {
331 // fp32331 // fp32
332- AscendC::MicroAPI::DataCopy<T2, AscendC::MicroAPI::LoadDist::DIST_NORM>(vregFloatO,332+ AscendC::MicroAPI::LoadAlign<T2, AscendC::MicroAPI::LoadDist::DIST_NORM>(vregFloatO,
333- offsetLocalAddr + i * VL);333+ offsetLocalAddr + i * VL);
334 }334 }
335 if constexpr (SqrtMode == TPL_SQRT_MODE) {335 if constexpr (SqrtMode == TPL_SQRT_MODE) {
336 AscendC::MicroAPI::Mul(vregFloatS, vregFloatS, vregFloatS, mask);336 AscendC::MicroAPI::Mul(vregFloatS, vregFloatS, vregFloatS, mask);
@@ -351,22 +351,22 @@ __aicore__ inline void QuantizePerChannelNddmaRegbase<T, T1, T2, U, DivMode, Rou
351 AscendC::MicroAPI::Cast<351 AscendC::MicroAPI::Cast<
352 U, float, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_FP32_TO_HIFP8>(352 U, float, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_FP32_TO_HIFP8>(
353 vregY, vregFloatY, mask);353 vregY, vregFloatY, mask);
354- AscendC::MicroAPI::DataCopy<U, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(outLocalAddr + i * VL,354+ AscendC::MicroAPI::StoreAlign<U, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(outLocalAddr + i * VL,
355- vregY, mask);355+ vregY, mask);
356 } else if constexpr (IsSameType<U, fp8_e5m2_t>::value) {356 } else if constexpr (IsSameType<U, fp8_e5m2_t>::value) {
357 // fp8_e5m2357 // fp8_e5m2
358 AscendC::MicroAPI::Cast<358 AscendC::MicroAPI::Cast<
359 U, float, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_FP32_TO_FP8E5M2>(359 U, float, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_FP32_TO_FP8E5M2>(
360 vregY, vregFloatY, mask);360 vregY, vregFloatY, mask);
361- AscendC::MicroAPI::DataCopy<U, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(outLocalAddr + i * VL,361+ AscendC::MicroAPI::StoreAlign<U, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(outLocalAddr + i * VL,
362- vregY, mask);362+ vregY, mask);
363 } else if constexpr (IsSameType<U, fp8_e4m3fn_t>::value) {363 } else if constexpr (IsSameType<U, fp8_e4m3fn_t>::value) {
364 // fp8_e4m3364 // fp8_e4m3
365 AscendC::MicroAPI::Cast<365 AscendC::MicroAPI::Cast<
366 U, float, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_FP32_TO_FP8E4M3>(366 U, float, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_FP32_TO_FP8E4M3>(
367 vregY, vregFloatY, mask);367 vregY, vregFloatY, mask);
368- AscendC::MicroAPI::DataCopy<U, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(outLocalAddr + i * VL,368+ AscendC::MicroAPI::StoreAlign<U, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(outLocalAddr + i * VL,
369- vregY, mask);369+ vregY, mask);
370 } else if constexpr (IsSameType<U, int8_t>::value) {370 } else if constexpr (IsSameType<U, int8_t>::value) {
371 // int8371 // int8
372 AscendC::MicroAPI::Cast<372 AscendC::MicroAPI::Cast<
@@ -378,8 +378,8 @@ __aicore__ inline void QuantizePerChannelNddmaRegbase<T, T1, T2, U, DivMode, Rou
378 AscendC::MicroAPI::Cast<378 AscendC::MicroAPI::Cast<
379 int8_t, half, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_HALF_TO_INT8>(379 int8_t, half, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_HALF_TO_INT8>(
380 vregY, vregHalfY, mask);380 vregY, vregHalfY, mask);
381- AscendC::MicroAPI::DataCopy<U, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(outLocalAddr + i * VL,381+ AscendC::MicroAPI::StoreAlign<U, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(outLocalAddr + i * VL,
382- vregY, mask);382+ vregY, mask);
383 } else if constexpr (IsSameType<U, uint8_t>::value) {383 } else if constexpr (IsSameType<U, uint8_t>::value) {
384 // uint8384 // uint8
385 AscendC::MicroAPI::Cast<385 AscendC::MicroAPI::Cast<
@@ -391,15 +391,15 @@ __aicore__ inline void QuantizePerChannelNddmaRegbase<T, T1, T2, U, DivMode, Rou
391 AscendC::MicroAPI::Cast<391 AscendC::MicroAPI::Cast<
392 uint8_t, half, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_HALF_TO_UINT8>(392 uint8_t, half, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_HALF_TO_UINT8>(
393 vregY, vregHalfY, mask);393 vregY, vregHalfY, mask);
394- AscendC::MicroAPI::DataCopy<U, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(outLocalAddr + i * VL,394+ AscendC::MicroAPI::StoreAlign<U, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(outLocalAddr + i * VL,
395- vregY, mask);395+ vregY, mask);
396 } else if constexpr (IsSameType<U, int32_t>::value) {396 } else if constexpr (IsSameType<U, int32_t>::value) {
397 // int32397 // int32
398 AscendC::MicroAPI::Cast<398 AscendC::MicroAPI::Cast<
399 U, float, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_FP32_TO_INT32>(399 U, float, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_FP32_TO_INT32>(
400 vregY, vregFloatY, mask);400 vregY, vregFloatY, mask);
401- AscendC::MicroAPI::DataCopy<U, AscendC::MicroAPI::StoreDist::DIST_NORM>(outLocalAddr + i * VL, vregY,401+ AscendC::MicroAPI::StoreAlign<U, AscendC::MicroAPI::StoreDist::DIST_NORM>(outLocalAddr + i * VL, vregY,
402- mask);402+ mask);
403 } else if constexpr (IsSameType<U, int4b_t>::value) {403 } else if constexpr (IsSameType<U, int4b_t>::value) {
404 AscendC::MicroAPI::RegTensor<int16_t> vregInt16Y;404 AscendC::MicroAPI::RegTensor<int16_t> vregInt16Y;
405 AscendC::MicroAPI::RegTensor<uint16_t> vregTmp1Y;405 AscendC::MicroAPI::RegTensor<uint16_t> vregTmp1Y;
@@ -415,7 +415,7 @@ __aicore__ inline void QuantizePerChannelNddmaRegbase<T, T1, T2, U, DivMode, Rou
415 QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_F16_TO_I8>(415 QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_F16_TO_I8>(
416 (AscendC::MicroAPI::RegTensor<int4x2_t>&)vregTmp2Y, (AscendC::MicroAPI::RegTensor<half>&)vregTmp1Y,416 (AscendC::MicroAPI::RegTensor<int4x2_t>&)vregTmp2Y, (AscendC::MicroAPI::RegTensor<half>&)vregTmp1Y,
417 mask);417 mask);
418- AscendC::MicroAPI::DataCopy<yCopyDtype, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(418+ AscendC::MicroAPI::StoreAlign<yCopyDtype, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(
419 outLocalAddr + (i * VL / 2), vregTmp2Y, mask4Int4);419 outLocalAddr + (i * VL / 2), vregTmp2Y, mask4Int4);
420 }420 }
421 }421 }
@@ -443,4 +443,4 @@ __aicore__ inline void QuantizePerChannelNddmaRegbase<T, T1, T2, U, DivMode, Rou
443 outQueueY_.FreeTensor(outLocal);443 outQueueY_.FreeTensor(outLocal);
444}444}
445} // namespace QuantizeOp445} // namespace QuantizeOp
446-#endif446+#endif
Mquant/quantize/op_kernel/arch35/quantize_per_channel_no_offset_regbase.h+18-18
@@ -161,9 +161,9 @@ __aicore__ inline void QuantizePerChannelNoOffsetRegbase<T, T1, T2, U, DivMode,
161 LocalTensor<T> xLocal = inQueueX_.DeQue<T>();161 LocalTensor<T> xLocal = inQueueX_.DeQue<T>();
162 LocalTensor<yCopyDtype> outLocal = outQueueY_.AllocTensor<yCopyDtype>();162 LocalTensor<yCopyDtype> outLocal = outQueueY_.AllocTensor<yCopyDtype>();
163 163 
164- __local_mem__ T* xLocalAddr = (__local_mem__ T*)xLocal.GetPhyAddr();164+ __ubuf__ T* xLocalAddr = (__ubuf__ T*)xLocal.GetPhyAddr();
165- __local_mem__ T1* scaleLocalAddr = (__local_mem__ T1*)sLocal.GetPhyAddr();165+ __ubuf__ T1* scaleLocalAddr = (__ubuf__ T1*)sLocal.GetPhyAddr();
166- __local_mem__ yCopyDtype* outLocalAddr = (__local_mem__ yCopyDtype*)outLocal.GetPhyAddr();166+ __ubuf__ yCopyDtype* outLocalAddr = (__ubuf__ yCopyDtype*)outLocal.GetPhyAddr();
167 167 
168 uint16_t VL = AscendC::VECTOR_REG_WIDTH / sizeof(float);168 uint16_t VL = AscendC::VECTOR_REG_WIDTH / sizeof(float);
169 uint32_t xLocalOffset = static_cast<uint32_t>(tilingData_.baseLen);169 uint32_t xLocalOffset = static_cast<uint32_t>(tilingData_.baseLen);
@@ -194,18 +194,18 @@ __aicore__ inline void QuantizePerChannelNoOffsetRegbase<T, T1, T2, U, DivMode,
194 // ld for x194 // ld for x
195 if constexpr (IsSameType<T, float>::value) {195 if constexpr (IsSameType<T, float>::value) {
196 // fp32196 // fp32
197- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(197+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(
198 vregFloatX, xLocalAddr + i * VL + j * xLocalOffset);198 vregFloatX, xLocalAddr + i * VL + j * xLocalOffset);
199 } else if constexpr (IsSameType<T, half>::value) {199 } else if constexpr (IsSameType<T, half>::value) {
200 // fp16200 // fp16
201- AscendC::MicroAPI::DataCopy<half, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(201+ AscendC::MicroAPI::LoadAlign<half, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(
202 vregX, xLocalAddr + i * VL + j * xLocalOffset);202 vregX, xLocalAddr + i * VL + j * xLocalOffset);
203 AscendC::MicroAPI::Cast<203 AscendC::MicroAPI::Cast<
204 float, half, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_HALF_TO_FP32>(204 float, half, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_HALF_TO_FP32>(
205 vregFloatX, vregX, mask);205 vregFloatX, vregX, mask);
206 } else if constexpr (IsSameType<T, bfloat16_t>::value) {206 } else if constexpr (IsSameType<T, bfloat16_t>::value) {
207 // bf16207 // bf16
208- AscendC::MicroAPI::DataCopy<T, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(208+ AscendC::MicroAPI::LoadAlign<T, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(
209 vregX, xLocalAddr + i * VL + j * xLocalOffset);209 vregX, xLocalAddr + i * VL + j * xLocalOffset);
210 AscendC::MicroAPI::Cast<210 AscendC::MicroAPI::Cast<
211 float, T, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_BF16_TO_FP32>(211 float, T, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_BF16_TO_FP32>(
@@ -215,18 +215,18 @@ __aicore__ inline void QuantizePerChannelNoOffsetRegbase<T, T1, T2, U, DivMode,
215 // ld for scale215 // ld for scale
216 if constexpr (IsSameType<T1, float>::value) {216 if constexpr (IsSameType<T1, float>::value) {
217 // fp32217 // fp32
218- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(vregFloatS,218+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(
219- scaleLocalAddr + i * VL);219+ vregFloatS, scaleLocalAddr + i * VL);
220 } else if (IsSameType<T1, half>::value) {220 } else if (IsSameType<T1, half>::value) {
221 // fp16221 // fp16
222- AscendC::MicroAPI::DataCopy<T1, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(222+ AscendC::MicroAPI::LoadAlign<T1, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(
223 vregS, scaleLocalAddr + i * VL);223 vregS, scaleLocalAddr + i * VL);
224 AscendC::MicroAPI::Cast<224 AscendC::MicroAPI::Cast<
225 float, T1, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_HALF_TO_FP32>(225 float, T1, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_HALF_TO_FP32>(
226 vregFloatS, vregS, mask);226 vregFloatS, vregS, mask);
227 } else if (IsSameType<T1, bfloat16_t>::value) {227 } else if (IsSameType<T1, bfloat16_t>::value) {
228 // bf16228 // bf16
229- AscendC::MicroAPI::DataCopy<T1, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(229+ AscendC::MicroAPI::LoadAlign<T1, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(
230 vregS, scaleLocalAddr + i * VL);230 vregS, scaleLocalAddr + i * VL);
231 AscendC::MicroAPI::Cast<231 AscendC::MicroAPI::Cast<
232 float, T1, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_BF16_TO_FP32>(232 float, T1, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_BF16_TO_FP32>(
@@ -249,21 +249,21 @@ __aicore__ inline void QuantizePerChannelNoOffsetRegbase<T, T1, T2, U, DivMode,
249 AscendC::MicroAPI::Cast<249 AscendC::MicroAPI::Cast<
250 U, float, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_FP32_TO_HIFP8>(250 U, float, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_FP32_TO_HIFP8>(
251 vregY, vregFloatY, mask);251 vregY, vregFloatY, mask);
252- AscendC::MicroAPI::DataCopy<U, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(252+ AscendC::MicroAPI::StoreAlign<U, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(
253 outLocalAddr + i * VL + j * xLocalOffset, vregY, mask);253 outLocalAddr + i * VL + j * xLocalOffset, vregY, mask);
254 } else if constexpr (IsSameType<U, fp8_e5m2_t>::value) {254 } else if constexpr (IsSameType<U, fp8_e5m2_t>::value) {
255 // fp8_e5m2255 // fp8_e5m2
256 AscendC::MicroAPI::Cast<256 AscendC::MicroAPI::Cast<
257 U, float, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_FP32_TO_FP8E5M2>(257 U, float, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_FP32_TO_FP8E5M2>(
258 vregY, vregFloatY, mask);258 vregY, vregFloatY, mask);
259- AscendC::MicroAPI::DataCopy<U, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(259+ AscendC::MicroAPI::StoreAlign<U, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(
260 outLocalAddr + i * VL + j * xLocalOffset, vregY, mask);260 outLocalAddr + i * VL + j * xLocalOffset, vregY, mask);
261 } else if constexpr (IsSameType<U, fp8_e4m3fn_t>::value) {261 } else if constexpr (IsSameType<U, fp8_e4m3fn_t>::value) {
262 // fp8_e4m3262 // fp8_e4m3
263 AscendC::MicroAPI::Cast<263 AscendC::MicroAPI::Cast<
264 U, float, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_FP32_TO_FP8E4M3>(264 U, float, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_FP32_TO_FP8E4M3>(
265 vregY, vregFloatY, mask);265 vregY, vregFloatY, mask);
266- AscendC::MicroAPI::DataCopy<U, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(266+ AscendC::MicroAPI::StoreAlign<U, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(
267 outLocalAddr + i * VL + j * xLocalOffset, vregY, mask);267 outLocalAddr + i * VL + j * xLocalOffset, vregY, mask);
268 } else if constexpr (IsSameType<U, int8_t>::value) {268 } else if constexpr (IsSameType<U, int8_t>::value) {
269 // int8, float->int16->half->int8269 // int8, float->int16->half->int8
@@ -279,7 +279,7 @@ __aicore__ inline void QuantizePerChannelNoOffsetRegbase<T, T1, T2, U, DivMode,
279 int8_t, half,279 int8_t, half,
280 QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_HALF_TO_INT8>(280 QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_HALF_TO_INT8>(
281 vregY, vregHalfY, mask);281 vregY, vregHalfY, mask);
282- AscendC::MicroAPI::DataCopy<U, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(282+ AscendC::MicroAPI::StoreAlign<U, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(
283 outLocalAddr + i * VL + j * xLocalOffset, vregY, mask);283 outLocalAddr + i * VL + j * xLocalOffset, vregY, mask);
284 } else if constexpr (IsSameType<U, uint8_t>::value) {284 } else if constexpr (IsSameType<U, uint8_t>::value) {
285 // uint8, float->int16->half->int8285 // uint8, float->int16->half->int8
@@ -295,14 +295,14 @@ __aicore__ inline void QuantizePerChannelNoOffsetRegbase<T, T1, T2, U, DivMode,
295 uint8_t, half,295 uint8_t, half,
296 QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_HALF_TO_UINT8>(296 QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_HALF_TO_UINT8>(
297 vregY, vregHalfY, mask);297 vregY, vregHalfY, mask);
298- AscendC::MicroAPI::DataCopy<U, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(298+ AscendC::MicroAPI::StoreAlign<U, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(
299 outLocalAddr + i * VL + j * xLocalOffset, vregY, mask);299 outLocalAddr + i * VL + j * xLocalOffset, vregY, mask);
300 } else if constexpr (IsSameType<U, int32_t>::value) {300 } else if constexpr (IsSameType<U, int32_t>::value) {
301 // int32301 // int32
302 AscendC::MicroAPI::Cast<302 AscendC::MicroAPI::Cast<
303 U, float, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_FP32_TO_INT32>(303 U, float, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_FP32_TO_INT32>(
304 vregY, vregFloatY, mask);304 vregY, vregFloatY, mask);
305- AscendC::MicroAPI::DataCopy<U, AscendC::MicroAPI::StoreDist::DIST_NORM>(305+ AscendC::MicroAPI::StoreAlign<U, AscendC::MicroAPI::StoreDist::DIST_NORM>(
306 outLocalAddr + i * VL + j * xLocalOffset, vregY, mask);306 outLocalAddr + i * VL + j * xLocalOffset, vregY, mask);
307 } else if constexpr (IsSameType<U, int4b_t>::value) {307 } else if constexpr (IsSameType<U, int4b_t>::value) {
308 AscendC::MicroAPI::RegTensor<int16_t> vregInt16Y;308 AscendC::MicroAPI::RegTensor<int16_t> vregInt16Y;
@@ -322,7 +322,7 @@ __aicore__ inline void QuantizePerChannelNoOffsetRegbase<T, T1, T2, U, DivMode,
322 int4x2_t, half, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_F16_TO_I8>(322 int4x2_t, half, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_F16_TO_I8>(
323 (AscendC::MicroAPI::RegTensor<int4x2_t>&)vregTmp2Y,323 (AscendC::MicroAPI::RegTensor<int4x2_t>&)vregTmp2Y,
324 (AscendC::MicroAPI::RegTensor<half>&)vregTmp1Y, mask);324 (AscendC::MicroAPI::RegTensor<half>&)vregTmp1Y, mask);
325- AscendC::MicroAPI::DataCopy<yCopyDtype, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(325+ AscendC::MicroAPI::StoreAlign<yCopyDtype, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(
326 outLocalAddr + (i * VL / 2 + j * tilingData_.baseLen), vregTmp2Y, mask4Int4);326 outLocalAddr + (i * VL / 2 + j * tilingData_.baseLen), vregTmp2Y, mask4Int4);
327 }327 }
328 }328 }
@@ -346,4 +346,4 @@ __aicore__ inline void QuantizePerChannelNoOffsetRegbase<T, T1, T2, U, DivMode,
346 outQueueY_.FreeTensor(outLocal);346 outQueueY_.FreeTensor(outLocal);
347}347}
348} // namespace QuantizeOp348} // namespace QuantizeOp
349-#endif349+#endif
Mquant/quantize/op_kernel/arch35/quantize_per_channel_regbase.h+27-27
@@ -188,12 +188,12 @@ __aicore__ inline void QuantizePerChannelRegbase<T, T1, T2, U, DivMode, RoundMod
188 LocalTensor<T> xLocal = inQueueX_.DeQue<T>();188 LocalTensor<T> xLocal = inQueueX_.DeQue<T>();
189 LocalTensor<yCopyDtype> outLocal = outQueueY_.AllocTensor<yCopyDtype>();189 LocalTensor<yCopyDtype> outLocal = outQueueY_.AllocTensor<yCopyDtype>();
190 190 
191- __local_mem__ T* xLocalAddr = (__local_mem__ T*)xLocal.GetPhyAddr();191+ __ubuf__ T* xLocalAddr = (__ubuf__ T*)xLocal.GetPhyAddr();
192- __local_mem__ T1* scaleLocalAddr = (__local_mem__ T1*)sLocal.GetPhyAddr();192+ __ubuf__ T1* scaleLocalAddr = (__ubuf__ T1*)sLocal.GetPhyAddr();
193- __local_mem__ yCopyDtype* outLocalAddr = (__local_mem__ yCopyDtype*)outLocal.GetPhyAddr();193+ __ubuf__ yCopyDtype* outLocalAddr = (__ubuf__ yCopyDtype*)outLocal.GetPhyAddr();
194 194 
195 uint16_t VL = AscendC::VECTOR_REG_WIDTH / sizeof(float);195 uint16_t VL = AscendC::VECTOR_REG_WIDTH / sizeof(float);
196- __local_mem__ T2* offsetLocalAddr = (__local_mem__ T2*)oLocal.GetPhyAddr();196+ __ubuf__ T2* offsetLocalAddr = (__ubuf__ T2*)oLocal.GetPhyAddr();
197 uint32_t xLocalOffset = static_cast<uint32_t>(tilingData_.baseLen);197 uint32_t xLocalOffset = static_cast<uint32_t>(tilingData_.baseLen);
198 198 
199 // has offset199 // has offset
@@ -229,18 +229,18 @@ __aicore__ inline void QuantizePerChannelRegbase<T, T1, T2, U, DivMode, RoundMod
229 // ld and cast for x229 // ld and cast for x
230 if constexpr (IsSameType<T, float>::value) {230 if constexpr (IsSameType<T, float>::value) {
231 // fp32231 // fp32
232- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(232+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(
233 vregFloatX, xLocalAddr + i * VL + j * xLocalOffset);233 vregFloatX, xLocalAddr + i * VL + j * xLocalOffset);
234 } else if constexpr (IsSameType<T, half>::value) {234 } else if constexpr (IsSameType<T, half>::value) {
235 // fp16235 // fp16
236- AscendC::MicroAPI::DataCopy<half, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(236+ AscendC::MicroAPI::LoadAlign<half, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(
237 vregX, xLocalAddr + i * VL + j * xLocalOffset);237 vregX, xLocalAddr + i * VL + j * xLocalOffset);
238 AscendC::MicroAPI::Cast<238 AscendC::MicroAPI::Cast<
239 float, half, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_HALF_TO_FP32>(239 float, half, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_HALF_TO_FP32>(
240 vregFloatX, vregX, mask);240 vregFloatX, vregX, mask);
241 } else if constexpr (IsSameType<T, bfloat16_t>::value) {241 } else if constexpr (IsSameType<T, bfloat16_t>::value) {
242 // bf16242 // bf16
243- AscendC::MicroAPI::DataCopy<T, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(243+ AscendC::MicroAPI::LoadAlign<T, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(
244 vregX, xLocalAddr + i * VL + j * xLocalOffset);244 vregX, xLocalAddr + i * VL + j * xLocalOffset);
245 AscendC::MicroAPI::Cast<245 AscendC::MicroAPI::Cast<
246 float, T, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_BF16_TO_FP32>(246 float, T, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_BF16_TO_FP32>(
@@ -250,18 +250,18 @@ __aicore__ inline void QuantizePerChannelRegbase<T, T1, T2, U, DivMode, RoundMod
250 // ld and cast for scale250 // ld and cast for scale
251 if constexpr (IsSameType<T1, float>::value) {251 if constexpr (IsSameType<T1, float>::value) {
252 // fp32252 // fp32
253- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(vregFloatS,253+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(
254- scaleLocalAddr + i * VL);254+ vregFloatS, scaleLocalAddr + i * VL);
255 } else if constexpr (IsSameType<T1, half>::value) {255 } else if constexpr (IsSameType<T1, half>::value) {
256 // fp16256 // fp16
257- AscendC::MicroAPI::DataCopy<half, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(257+ AscendC::MicroAPI::LoadAlign<half, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(
258 vregS, scaleLocalAddr + i * VL);258 vregS, scaleLocalAddr + i * VL);
259 AscendC::MicroAPI::Cast<259 AscendC::MicroAPI::Cast<
260 float, T1, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_HALF_TO_FP32>(260 float, T1, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_HALF_TO_FP32>(
261 vregFloatS, vregS, mask);261 vregFloatS, vregS, mask);
262 } else if constexpr (IsSameType<T1, bfloat16_t>::value) {262 } else if constexpr (IsSameType<T1, bfloat16_t>::value) {
263 // bf16263 // bf16
264- AscendC::MicroAPI::DataCopy<T1, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(264+ AscendC::MicroAPI::LoadAlign<T1, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(
265 vregS, scaleLocalAddr + i * VL);265 vregS, scaleLocalAddr + i * VL);
266 AscendC::MicroAPI::Cast<266 AscendC::MicroAPI::Cast<
267 float, T1, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_BF16_TO_FP32>(267 float, T1, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_BF16_TO_FP32>(
@@ -270,14 +270,14 @@ __aicore__ inline void QuantizePerChannelRegbase<T, T1, T2, U, DivMode, RoundMod
270 // ld and cast for offset270 // ld and cast for offset
271 if constexpr (IsSameType<T2, int32_t>::value) {271 if constexpr (IsSameType<T2, int32_t>::value) {
272 // int32272 // int32
273- AscendC::MicroAPI::DataCopy<T2, AscendC::MicroAPI::LoadDist::DIST_NORM>(vregO,273+ AscendC::MicroAPI::LoadAlign<T2, AscendC::MicroAPI::LoadDist::DIST_NORM>(vregO,
274- offsetLocalAddr + i * VL);274+ offsetLocalAddr + i * VL);
275 AscendC::MicroAPI::Cast<275 AscendC::MicroAPI::Cast<
276 float, T2, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_INT32_TO_FP32>(276 float, T2, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_INT32_TO_FP32>(
277 vregFloatO, vregO, mask);277 vregFloatO, vregO, mask);
278 } else if constexpr (IsSameType<T2, int8_t>::value) {278 } else if constexpr (IsSameType<T2, int8_t>::value) {
279 // int8279 // int8
280- AscendC::MicroAPI::DataCopy<T2, AscendC::MicroAPI::LoadDist::DIST_UNPACK4_B8>(280+ AscendC::MicroAPI::LoadAlign<T2, AscendC::MicroAPI::LoadDist::DIST_UNPACK4_B8>(
281 vregO, offsetLocalAddr + i * VL);281 vregO, offsetLocalAddr + i * VL);
282 AscendC::MicroAPI::Cast<282 AscendC::MicroAPI::Cast<
283 half, T2, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_INT8_TO_HALF>(283 half, T2, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_INT8_TO_HALF>(
@@ -287,7 +287,7 @@ __aicore__ inline void QuantizePerChannelRegbase<T, T1, T2, U, DivMode, RoundMod
287 vregFloatO, vregHalfO, mask);287 vregFloatO, vregHalfO, mask);
288 } else if constexpr (IsSameType<T2, uint8_t>::value) {288 } else if constexpr (IsSameType<T2, uint8_t>::value) {
289 // uint8289 // uint8
290- AscendC::MicroAPI::DataCopy<T2, AscendC::MicroAPI::LoadDist::DIST_UNPACK4_B8>(290+ AscendC::MicroAPI::LoadAlign<T2, AscendC::MicroAPI::LoadDist::DIST_UNPACK4_B8>(
291 vregO, offsetLocalAddr + i * VL);291 vregO, offsetLocalAddr + i * VL);
292 AscendC::MicroAPI::Cast<292 AscendC::MicroAPI::Cast<
293 half, T2, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_UINT8_TO_HALF>(293 half, T2, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_UINT8_TO_HALF>(
@@ -297,22 +297,22 @@ __aicore__ inline void QuantizePerChannelRegbase<T, T1, T2, U, DivMode, RoundMod
297 vregFloatO, vregHalfO, mask);297 vregFloatO, vregHalfO, mask);
298 } else if constexpr (IsSameType<T2, bfloat16_t>::value) {298 } else if constexpr (IsSameType<T2, bfloat16_t>::value) {
299 // bf16299 // bf16
300- AscendC::MicroAPI::DataCopy<T2, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(300+ AscendC::MicroAPI::LoadAlign<T2, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(
301 vregO, offsetLocalAddr + i * VL);301 vregO, offsetLocalAddr + i * VL);
302 AscendC::MicroAPI::Cast<302 AscendC::MicroAPI::Cast<
303 float, T2, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_BF16_TO_FP32>(303 float, T2, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_BF16_TO_FP32>(
304 vregFloatO, vregO, mask);304 vregFloatO, vregO, mask);
305 } else if constexpr (IsSameType<T2, half>::value) {305 } else if constexpr (IsSameType<T2, half>::value) {
306 // fp16306 // fp16
307- AscendC::MicroAPI::DataCopy<T2, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(307+ AscendC::MicroAPI::LoadAlign<T2, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(
308 vregO, offsetLocalAddr + i * VL);308 vregO, offsetLocalAddr + i * VL);
309 AscendC::MicroAPI::Cast<309 AscendC::MicroAPI::Cast<
310 float, T2, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_HALF_TO_FP32>(310 float, T2, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_HALF_TO_FP32>(
311 vregFloatO, vregO, mask);311 vregFloatO, vregO, mask);
312 } else if constexpr (IsSameType<T2, float>::value) {312 } else if constexpr (IsSameType<T2, float>::value) {
313 // fp32313 // fp32
314- AscendC::MicroAPI::DataCopy<T2, AscendC::MicroAPI::LoadDist::DIST_NORM>(vregFloatO,314+ AscendC::MicroAPI::LoadAlign<T2, AscendC::MicroAPI::LoadDist::DIST_NORM>(vregFloatO,
315- offsetLocalAddr + i * VL);315+ offsetLocalAddr + i * VL);
316 }316 }
317 if constexpr (SqrtMode == TPL_SQRT_MODE) {317 if constexpr (SqrtMode == TPL_SQRT_MODE) {
318 AscendC::MicroAPI::Mul(vregFloatS, vregFloatS, vregFloatS, mask);318 AscendC::MicroAPI::Mul(vregFloatS, vregFloatS, vregFloatS, mask);
@@ -333,21 +333,21 @@ __aicore__ inline void QuantizePerChannelRegbase<T, T1, T2, U, DivMode, RoundMod
333 AscendC::MicroAPI::Cast<333 AscendC::MicroAPI::Cast<
334 U, float, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_FP32_TO_HIFP8>(334 U, float, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_FP32_TO_HIFP8>(
335 vregY, vregFloatY, mask);335 vregY, vregFloatY, mask);
336- AscendC::MicroAPI::DataCopy<U, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(336+ AscendC::MicroAPI::StoreAlign<U, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(
337 outLocalAddr + i * VL + j * xLocalOffset, vregY, mask);337 outLocalAddr + i * VL + j * xLocalOffset, vregY, mask);
338 } else if constexpr (IsSameType<U, fp8_e5m2_t>::value) {338 } else if constexpr (IsSameType<U, fp8_e5m2_t>::value) {
339 // fp8_e5m2339 // fp8_e5m2
340 AscendC::MicroAPI::Cast<340 AscendC::MicroAPI::Cast<
341 U, float, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_FP32_TO_FP8E5M2>(341 U, float, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_FP32_TO_FP8E5M2>(
342 vregY, vregFloatY, mask);342 vregY, vregFloatY, mask);
343- AscendC::MicroAPI::DataCopy<U, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(343+ AscendC::MicroAPI::StoreAlign<U, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(
344 outLocalAddr + i * VL + j * xLocalOffset, vregY, mask);344 outLocalAddr + i * VL + j * xLocalOffset, vregY, mask);
345 } else if constexpr (IsSameType<U, fp8_e4m3fn_t>::value) {345 } else if constexpr (IsSameType<U, fp8_e4m3fn_t>::value) {
346 // fp8_e4m3346 // fp8_e4m3
347 AscendC::MicroAPI::Cast<347 AscendC::MicroAPI::Cast<
348 U, float, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_FP32_TO_FP8E4M3>(348 U, float, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_FP32_TO_FP8E4M3>(
349 vregY, vregFloatY, mask);349 vregY, vregFloatY, mask);
350- AscendC::MicroAPI::DataCopy<U, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(350+ AscendC::MicroAPI::StoreAlign<U, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(
351 outLocalAddr + i * VL + j * xLocalOffset, vregY, mask);351 outLocalAddr + i * VL + j * xLocalOffset, vregY, mask);
352 } else if constexpr (IsSameType<U, int8_t>::value) {352 } else if constexpr (IsSameType<U, int8_t>::value) {
353 // int8353 // int8
@@ -363,7 +363,7 @@ __aicore__ inline void QuantizePerChannelRegbase<T, T1, T2, U, DivMode, RoundMod
363 int8_t, half,363 int8_t, half,
364 QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_HALF_TO_INT8>(364 QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_HALF_TO_INT8>(
365 vregY, vregHalfY, mask);365 vregY, vregHalfY, mask);
366- AscendC::MicroAPI::DataCopy<U, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(366+ AscendC::MicroAPI::StoreAlign<U, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(
367 outLocalAddr + i * VL + j * xLocalOffset, vregY, mask);367 outLocalAddr + i * VL + j * xLocalOffset, vregY, mask);
368 } else if constexpr (IsSameType<U, uint8_t>::value) {368 } else if constexpr (IsSameType<U, uint8_t>::value) {
369 // uint8369 // uint8
@@ -379,14 +379,14 @@ __aicore__ inline void QuantizePerChannelRegbase<T, T1, T2, U, DivMode, RoundMod
379 uint8_t, half,379 uint8_t, half,
380 QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_HALF_TO_UINT8>(380 QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_HALF_TO_UINT8>(
381 vregY, vregHalfY, mask);381 vregY, vregHalfY, mask);
382- AscendC::MicroAPI::DataCopy<U, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(382+ AscendC::MicroAPI::StoreAlign<U, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(
383 outLocalAddr + i * VL + j * xLocalOffset, vregY, mask);383 outLocalAddr + i * VL + j * xLocalOffset, vregY, mask);
384 } else if constexpr (IsSameType<U, int32_t>::value) {384 } else if constexpr (IsSameType<U, int32_t>::value) {
385 // int32385 // int32
386 AscendC::MicroAPI::Cast<386 AscendC::MicroAPI::Cast<
387 U, float, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_FP32_TO_INT32>(387 U, float, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_FP32_TO_INT32>(
388 vregY, vregFloatY, mask);388 vregY, vregFloatY, mask);
389- AscendC::MicroAPI::DataCopy<U, AscendC::MicroAPI::StoreDist::DIST_NORM>(389+ AscendC::MicroAPI::StoreAlign<U, AscendC::MicroAPI::StoreDist::DIST_NORM>(
390 outLocalAddr + i * VL + j * xLocalOffset, vregY, mask);390 outLocalAddr + i * VL + j * xLocalOffset, vregY, mask);
391 } else if constexpr (IsSameType<U, int4b_t>::value) {391 } else if constexpr (IsSameType<U, int4b_t>::value) {
392 AscendC::MicroAPI::RegTensor<int16_t> vregInt16Y;392 AscendC::MicroAPI::RegTensor<int16_t> vregInt16Y;
@@ -405,7 +405,7 @@ __aicore__ inline void QuantizePerChannelRegbase<T, T1, T2, U, DivMode, RoundMod
405 int4x2_t, half, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_F16_TO_I8>(405 int4x2_t, half, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_F16_TO_I8>(
406 (AscendC::MicroAPI::RegTensor<int4x2_t>&)vregTmp2Y,406 (AscendC::MicroAPI::RegTensor<int4x2_t>&)vregTmp2Y,
407 (AscendC::MicroAPI::RegTensor<half>&)vregTmp1Y, mask);407 (AscendC::MicroAPI::RegTensor<half>&)vregTmp1Y, mask);
408- AscendC::MicroAPI::DataCopy<yCopyDtype, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(408+ AscendC::MicroAPI::StoreAlign<yCopyDtype, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(
409 outLocalAddr + (i * VL / 2 + j * tilingData_.baseLen), vregTmp2Y, mask4Int4);409 outLocalAddr + (i * VL / 2 + j * tilingData_.baseLen), vregTmp2Y, mask4Int4);
410 }410 }
411 }411 }
@@ -429,4 +429,4 @@ __aicore__ inline void QuantizePerChannelRegbase<T, T1, T2, U, DivMode, RoundMod
429 outQueueY_.FreeTensor(outLocal);429 outQueueY_.FreeTensor(outLocal);
430}430}
431} // namespace QuantizeOp431} // namespace QuantizeOp
432-#endif432+#endif
Mquant/quantize/op_kernel/arch35/quantize_per_head_no_offset_regbase.h+21-21
@@ -222,8 +222,8 @@ private:
222 int64_t paramLen, int64_t paramOffset)222 int64_t paramLen, int64_t paramOffset)
223 {223 {
224 auto paramLocal = inQueue.AllocTensor<T1>();224 auto paramLocal = inQueue.AllocTensor<T1>();
225- static constexpr AscendC::MultiCopyConfig copyConfig = {false, 0, 0, false};225+ static constexpr AscendC::NdDmaConfig copyConfig = {false, 0, 0, false};
226- MultiCopyLoopInfo<QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::MULTI_COPY_DIM> multiCopyParams;226+ NdDmaLoopInfo<QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::MULTI_COPY_DIM> multiCopyParams;
227 // src stride info per loop.227 // src stride info per loop.
228 multiCopyParams.loopSrcStride[0] = 0;228 multiCopyParams.loopSrcStride[0] = 0;
229 multiCopyParams.loopSrcStride[1] = 1;229 multiCopyParams.loopSrcStride[1] = 1;
@@ -234,8 +234,8 @@ private:
234 multiCopyParams.loopSize[0] = tilingData_.baseLen;234 multiCopyParams.loopSize[0] = tilingData_.baseLen;
235 multiCopyParams.loopSize[1] = paramLen;235 multiCopyParams.loopSize[1] = paramLen;
236 T1 constValue = 0;236 T1 constValue = 0;
237- AscendC::MultiCopyParams<T1, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::MULTI_COPY_DIM>237+ AscendC::NdDmaParams<T1, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::MULTI_COPY_DIM> paramsMain =
238- paramsMain = {multiCopyParams, constValue};238+ {multiCopyParams, constValue};
239 AscendC::DataCopy<T1, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::MULTI_COPY_DIM, copyConfig>(239 AscendC::DataCopy<T1, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::MULTI_COPY_DIM, copyConfig>(
240 paramLocal, inGm[paramOffset], paramsMain);240 paramLocal, inGm[paramOffset], paramsMain);
241 inQueue.EnQue(paramLocal);241 inQueue.EnQue(paramLocal);
@@ -268,9 +268,9 @@ private:
268 auto inLocal = inQueueX_.DeQue<T>();268 auto inLocal = inQueueX_.DeQue<T>();
269 auto outLocal = outQueueY_.AllocTensor<yCopyDtype>();269 auto outLocal = outQueueY_.AllocTensor<yCopyDtype>();
270 270 
271- __local_mem__ T* xLocalAddr = (__local_mem__ T*)inLocal.GetPhyAddr();271+ __ubuf__ T* xLocalAddr = (__ubuf__ T*)inLocal.GetPhyAddr();
272- __local_mem__ T1* scaleLocalAddr = (__local_mem__ T1*)scaleLocal.GetPhyAddr();272+ __ubuf__ T1* scaleLocalAddr = (__ubuf__ T1*)scaleLocal.GetPhyAddr();
273- __local_mem__ yCopyDtype* outLocalAddr = (__local_mem__ yCopyDtype*)outLocal.GetPhyAddr();273+ __ubuf__ yCopyDtype* outLocalAddr = (__ubuf__ yCopyDtype*)outLocal.GetPhyAddr();
274 uint16_t VL = AscendC::VECTOR_REG_WIDTH / sizeof(float);274 uint16_t VL = AscendC::VECTOR_REG_WIDTH / sizeof(float);
275 275 
276 // no offset276 // no offset
@@ -301,11 +301,11 @@ private:
301 // ld and cast for x301 // ld and cast for x
302 if constexpr (IsSameType<T, float>::value) {302 if constexpr (IsSameType<T, float>::value) {
303 // fp32303 // fp32
304- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(304+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(
305 vregFloatX, xLocalAddr + i * VL + j * tilingData_.baseLen);305 vregFloatX, xLocalAddr + i * VL + j * tilingData_.baseLen);
306 } else if constexpr (IsSameType<T, half>::value) {306 } else if constexpr (IsSameType<T, half>::value) {
307 // fp16307 // fp16
308- AscendC::MicroAPI::DataCopy<half, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(308+ AscendC::MicroAPI::LoadAlign<half, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(
309 vregX, xLocalAddr + i * VL + j * tilingData_.baseLen);309 vregX, xLocalAddr + i * VL + j * tilingData_.baseLen);
310 AscendC::MicroAPI::Cast<310 AscendC::MicroAPI::Cast<
311 float, half,311 float, half,
@@ -313,7 +313,7 @@ private:
313 vregFloatX, vregX, mask);313 vregFloatX, vregX, mask);
314 } else {314 } else {
315 // bf16315 // bf16
316- AscendC::MicroAPI::DataCopy<T, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(316+ AscendC::MicroAPI::LoadAlign<T, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(
317 vregX, xLocalAddr + i * VL + j * tilingData_.baseLen);317 vregX, xLocalAddr + i * VL + j * tilingData_.baseLen);
318 AscendC::MicroAPI::Cast<318 AscendC::MicroAPI::Cast<
319 float, T,319 float, T,
@@ -324,11 +324,11 @@ private:
324 // ld and cast for scale324 // ld and cast for scale
325 if constexpr (IsSameType<T1, float>::value) {325 if constexpr (IsSameType<T1, float>::value) {
326 // fp32326 // fp32
327- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(327+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(
328 vregFloatS, scaleLocalAddr + i * VL + j * tilingData_.baseLen);328 vregFloatS, scaleLocalAddr + i * VL + j * tilingData_.baseLen);
329 } else if constexpr (IsSameType<T1, half>::value) {329 } else if constexpr (IsSameType<T1, half>::value) {
330 // fp16330 // fp16
331- AscendC::MicroAPI::DataCopy<half, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(331+ AscendC::MicroAPI::LoadAlign<half, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(
332 vregS, scaleLocalAddr + i * VL + j * tilingData_.baseLen);332 vregS, scaleLocalAddr + i * VL + j * tilingData_.baseLen);
333 AscendC::MicroAPI::Cast<333 AscendC::MicroAPI::Cast<
334 float, half,334 float, half,
@@ -336,7 +336,7 @@ private:
336 vregFloatS, vregS, mask);336 vregFloatS, vregS, mask);
337 } else {337 } else {
338 // bf16338 // bf16
339- AscendC::MicroAPI::DataCopy<T1, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(339+ AscendC::MicroAPI::LoadAlign<T1, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(
340 vregS, scaleLocalAddr + i * VL + j * tilingData_.baseLen);340 vregS, scaleLocalAddr + i * VL + j * tilingData_.baseLen);
341 AscendC::MicroAPI::Cast<341 AscendC::MicroAPI::Cast<
342 float, T1,342 float, T1,
@@ -364,7 +364,7 @@ private:
364 U, float,364 U, float,
365 QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_FP32_TO_HIFP8>(365 QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_FP32_TO_HIFP8>(
366 vregY, vregFloatY, mask);366 vregY, vregFloatY, mask);
367- AscendC::MicroAPI::DataCopy<U, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(367+ AscendC::MicroAPI::StoreAlign<U, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(
368 outLocalAddr + i * VL + j * tilingData_.baseLen, vregY, mask);368 outLocalAddr + i * VL + j * tilingData_.baseLen, vregY, mask);
369 } else if constexpr (IsSameType<U, fp8_e5m2_t>::value) {369 } else if constexpr (IsSameType<U, fp8_e5m2_t>::value) {
370 // fp8_e5m2 支持370 // fp8_e5m2 支持
@@ -372,7 +372,7 @@ private:
372 U, float,372 U, float,
373 QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_FP32_TO_FP8E5M2>(373 QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_FP32_TO_FP8E5M2>(
374 vregY, vregFloatY, mask);374 vregY, vregFloatY, mask);
375- AscendC::MicroAPI::DataCopy<U, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(375+ AscendC::MicroAPI::StoreAlign<U, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(
376 outLocalAddr + i * VL + j * tilingData_.baseLen, vregY, mask);376 outLocalAddr + i * VL + j * tilingData_.baseLen, vregY, mask);
377 } else if constexpr (IsSameType<U, fp8_e4m3fn_t>::value) {377 } else if constexpr (IsSameType<U, fp8_e4m3fn_t>::value) {
378 // fp8_e4m3 支持378 // fp8_e4m3 支持
@@ -380,7 +380,7 @@ private:
380 U, float,380 U, float,
381 QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_FP32_TO_FP8E4M3>(381 QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_FP32_TO_FP8E4M3>(
382 vregY, vregFloatY, mask);382 vregY, vregFloatY, mask);
383- AscendC::MicroAPI::DataCopy<U, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(383+ AscendC::MicroAPI::StoreAlign<U, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(
384 outLocalAddr + i * VL + j * tilingData_.baseLen, vregY, mask);384 outLocalAddr + i * VL + j * tilingData_.baseLen, vregY, mask);
385 } else if constexpr (IsSameType<U, int8_t>::value) {385 } else if constexpr (IsSameType<U, int8_t>::value) {
386 // int8 支持386 // int8 支持
@@ -397,7 +397,7 @@ private:
397 int8_t, half,397 int8_t, half,
398 QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_HALF_TO_INT8>(398 QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_HALF_TO_INT8>(
399 vregY, vregHalfY, mask);399 vregY, vregHalfY, mask);
400- AscendC::MicroAPI::DataCopy<U, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(400+ AscendC::MicroAPI::StoreAlign<U, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(
401 outLocalAddr + i * VL + j * tilingData_.baseLen, vregY, mask);401 outLocalAddr + i * VL + j * tilingData_.baseLen, vregY, mask);
402 } else if constexpr (IsSameType<U, uint8_t>::value) {402 } else if constexpr (IsSameType<U, uint8_t>::value) {
403 // uint8403 // uint8
@@ -414,7 +414,7 @@ private:
414 uint8_t, half,414 uint8_t, half,
415 QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_HALF_TO_UINT8>(415 QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_HALF_TO_UINT8>(
416 vregY, vregHalfY, mask);416 vregY, vregHalfY, mask);
417- AscendC::MicroAPI::DataCopy<U, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(417+ AscendC::MicroAPI::StoreAlign<U, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(
418 outLocalAddr + i * VL + j * tilingData_.baseLen, vregY, mask);418 outLocalAddr + i * VL + j * tilingData_.baseLen, vregY, mask);
419 } else if constexpr (IsSameType<U, int32_t>::value) {419 } else if constexpr (IsSameType<U, int32_t>::value) {
420 // int32420 // int32
@@ -422,7 +422,7 @@ private:
422 U, float,422 U, float,
423 QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_FP32_TO_INT32>(423 QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_FP32_TO_INT32>(
424 vregY, vregFloatY, mask);424 vregY, vregFloatY, mask);
425- AscendC::MicroAPI::DataCopy<U, AscendC::MicroAPI::StoreDist::DIST_NORM>(425+ AscendC::MicroAPI::StoreAlign<U, AscendC::MicroAPI::StoreDist::DIST_NORM>(
426 outLocalAddr + i * VL + j * tilingData_.baseLen, vregY, mask);426 outLocalAddr + i * VL + j * tilingData_.baseLen, vregY, mask);
427 } else if constexpr (IsSameType<U, int4b_t>::value) {427 } else if constexpr (IsSameType<U, int4b_t>::value) {
428 AscendC::MicroAPI::RegTensor<int16_t> vregInt16Y;428 AscendC::MicroAPI::RegTensor<int16_t> vregInt16Y;
@@ -443,7 +443,7 @@ private:
443 QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_F16_TO_I8>(443 QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_F16_TO_I8>(
444 (AscendC::MicroAPI::RegTensor<int4x2_t>&)vregTmp2Y,444 (AscendC::MicroAPI::RegTensor<int4x2_t>&)vregTmp2Y,
445 (AscendC::MicroAPI::RegTensor<half>&)vregTmp1Y, mask);445 (AscendC::MicroAPI::RegTensor<half>&)vregTmp1Y, mask);
446- AscendC::MicroAPI::DataCopy<yCopyDtype, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(446+ AscendC::MicroAPI::StoreAlign<yCopyDtype, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(
447 outLocalAddr + (i * VL / 2 + j * tilingData_.baseLen), vregTmp2Y, mask4Int4);447 outLocalAddr + (i * VL / 2 + j * tilingData_.baseLen), vregTmp2Y, mask4Int4);
448 }448 }
449 }449 }
@@ -454,4 +454,4 @@ private:
454 }454 }
455};455};
456} // namespace QuantizeOp456} // namespace QuantizeOp
457-#endif // QUANTIZE_PERHEAD_REGBASE_H457+#endif // QUANTIZE_PERHEAD_REGBASE_H
Mquant/quantize/op_kernel/arch35/quantize_per_head_regbase.h+27-27
@@ -237,8 +237,8 @@ private:
237 int64_t paramLen, int64_t paramOffset)237 int64_t paramLen, int64_t paramOffset)
238 {238 {
239 auto paramLocal = inQueue.AllocTensor<dtypeCopyIn>();239 auto paramLocal = inQueue.AllocTensor<dtypeCopyIn>();
240- static constexpr AscendC::MultiCopyConfig copyConfig = {false, 0, 0, false};240+ static constexpr AscendC::NdDmaConfig copyConfig = {false, 0, 0, false};
241- MultiCopyLoopInfo<QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::MULTI_COPY_DIM> multiCopyParams;241+ NdDmaLoopInfo<QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::MULTI_COPY_DIM> multiCopyParams;
242 // src stride info per loop.242 // src stride info per loop.
243 multiCopyParams.loopSrcStride[0] = 0;243 multiCopyParams.loopSrcStride[0] = 0;
244 multiCopyParams.loopSrcStride[1] = 1;244 multiCopyParams.loopSrcStride[1] = 1;
@@ -249,7 +249,7 @@ private:
249 multiCopyParams.loopSize[0] = tilingData_.baseLen;249 multiCopyParams.loopSize[0] = tilingData_.baseLen;
250 multiCopyParams.loopSize[1] = paramLen;250 multiCopyParams.loopSize[1] = paramLen;
251 dtypeCopyIn constValue = 0;251 dtypeCopyIn constValue = 0;
252- AscendC::MultiCopyParams<dtypeCopyIn, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::MULTI_COPY_DIM>252+ AscendC::NdDmaParams<dtypeCopyIn, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::MULTI_COPY_DIM>
253 paramsMain = {multiCopyParams, constValue};253 paramsMain = {multiCopyParams, constValue};
254 AscendC::DataCopy<dtypeCopyIn, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::MULTI_COPY_DIM,254 AscendC::DataCopy<dtypeCopyIn, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::MULTI_COPY_DIM,
255 copyConfig>(paramLocal, inGm[paramOffset], paramsMain);255 copyConfig>(paramLocal, inGm[paramOffset], paramsMain);
@@ -284,10 +284,10 @@ private:
284 auto inLocal = inQueueX_.DeQue<T>();284 auto inLocal = inQueueX_.DeQue<T>();
285 auto outLocal = outQueueY_.AllocTensor<yCopyDtype>();285 auto outLocal = outQueueY_.AllocTensor<yCopyDtype>();
286 286 
287- __local_mem__ T* xLocalAddr = (__local_mem__ T*)inLocal.GetPhyAddr();287+ __ubuf__ T* xLocalAddr = (__ubuf__ T*)inLocal.GetPhyAddr();
288- __local_mem__ T1* scaleLocalAddr = (__local_mem__ T1*)scaleLocal.GetPhyAddr();288+ __ubuf__ T1* scaleLocalAddr = (__ubuf__ T1*)scaleLocal.GetPhyAddr();
289- __local_mem__ T2* offsetLocalAddr = (__local_mem__ T2*)offsetLocal.GetPhyAddr();289+ __ubuf__ T2* offsetLocalAddr = (__ubuf__ T2*)offsetLocal.GetPhyAddr();
290- __local_mem__ yCopyDtype* outLocalAddr = (__local_mem__ yCopyDtype*)outLocal.GetPhyAddr();290+ __ubuf__ yCopyDtype* outLocalAddr = (__ubuf__ yCopyDtype*)outLocal.GetPhyAddr();
291 uint16_t VL = AscendC::VECTOR_REG_WIDTH / sizeof(float);291 uint16_t VL = AscendC::VECTOR_REG_WIDTH / sizeof(float);
292 __VEC_SCOPE__292 __VEC_SCOPE__
293 {293 {
@@ -321,11 +321,11 @@ private:
321 // ld and cast for x321 // ld and cast for x
322 if constexpr (IsSameType<T, float>::value) {322 if constexpr (IsSameType<T, float>::value) {
323 // fp32323 // fp32
324- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(324+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(
325 vregFloatX, xLocalAddr + i * VL + j * tilingData_.baseLen);325 vregFloatX, xLocalAddr + i * VL + j * tilingData_.baseLen);
326 } else if constexpr (IsSameType<T, half>::value) {326 } else if constexpr (IsSameType<T, half>::value) {
327 // fp16327 // fp16
328- AscendC::MicroAPI::DataCopy<half, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(328+ AscendC::MicroAPI::LoadAlign<half, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(
329 vregX, xLocalAddr + i * VL + j * tilingData_.baseLen);329 vregX, xLocalAddr + i * VL + j * tilingData_.baseLen);
330 AscendC::MicroAPI::Cast<330 AscendC::MicroAPI::Cast<
331 float, half,331 float, half,
@@ -333,7 +333,7 @@ private:
333 vregFloatX, vregX, mask);333 vregFloatX, vregX, mask);
334 } else {334 } else {
335 // bf16335 // bf16
336- AscendC::MicroAPI::DataCopy<T, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(336+ AscendC::MicroAPI::LoadAlign<T, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(
337 vregX, xLocalAddr + i * VL + j * tilingData_.baseLen);337 vregX, xLocalAddr + i * VL + j * tilingData_.baseLen);
338 AscendC::MicroAPI::Cast<338 AscendC::MicroAPI::Cast<
339 float, T,339 float, T,
@@ -344,11 +344,11 @@ private:
344 // ld and cast for scale344 // ld and cast for scale
345 if constexpr (IsSameType<T1, float>::value) {345 if constexpr (IsSameType<T1, float>::value) {
346 // fp32346 // fp32
347- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(347+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(
348 vregFloatS, scaleLocalAddr + i * VL + j * tilingData_.baseLen);348 vregFloatS, scaleLocalAddr + i * VL + j * tilingData_.baseLen);
349 } else if constexpr (IsSameType<T1, half>::value) {349 } else if constexpr (IsSameType<T1, half>::value) {
350 // fp16350 // fp16
351- AscendC::MicroAPI::DataCopy<half, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(351+ AscendC::MicroAPI::LoadAlign<half, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(
352 vregS, scaleLocalAddr + i * VL + j * tilingData_.baseLen);352 vregS, scaleLocalAddr + i * VL + j * tilingData_.baseLen);
353 AscendC::MicroAPI::Cast<353 AscendC::MicroAPI::Cast<
354 float, half,354 float, half,
@@ -356,7 +356,7 @@ private:
356 vregFloatS, vregS, mask);356 vregFloatS, vregS, mask);
357 } else {357 } else {
358 // bf16358 // bf16
359- AscendC::MicroAPI::DataCopy<T1, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(359+ AscendC::MicroAPI::LoadAlign<T1, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(
360 vregS, scaleLocalAddr + i * VL + j * tilingData_.baseLen);360 vregS, scaleLocalAddr + i * VL + j * tilingData_.baseLen);
361 AscendC::MicroAPI::Cast<361 AscendC::MicroAPI::Cast<
362 float, T1,362 float, T1,
@@ -367,7 +367,7 @@ private:
367 // ld and cast for offset367 // ld and cast for offset
368 if constexpr (IsSameType<T2, int32_t>::value) {368 if constexpr (IsSameType<T2, int32_t>::value) {
369 // int32369 // int32
370- AscendC::MicroAPI::DataCopy<T2, AscendC::MicroAPI::LoadDist::DIST_NORM>(370+ AscendC::MicroAPI::LoadAlign<T2, AscendC::MicroAPI::LoadDist::DIST_NORM>(
371 vregO, offsetLocalAddr + i * VL + j * tilingData_.baseLen);371 vregO, offsetLocalAddr + i * VL + j * tilingData_.baseLen);
372 AscendC::MicroAPI::Cast<372 AscendC::MicroAPI::Cast<
373 float, T2,373 float, T2,
@@ -375,7 +375,7 @@ private:
375 vregFloatO, vregO, mask);375 vregFloatO, vregO, mask);
376 } else if constexpr (IsSameType<T2, int8_t>::value) {376 } else if constexpr (IsSameType<T2, int8_t>::value) {
377 // int8377 // int8
378- AscendC::MicroAPI::DataCopy<T2, AscendC::MicroAPI::LoadDist::DIST_UNPACK4_B8>(378+ AscendC::MicroAPI::LoadAlign<T2, AscendC::MicroAPI::LoadDist::DIST_UNPACK4_B8>(
379 vregO, offsetLocalAddr + i * VL + j * tilingData_.baseLen);379 vregO, offsetLocalAddr + i * VL + j * tilingData_.baseLen);
380 AscendC::MicroAPI::Cast<380 AscendC::MicroAPI::Cast<
381 half, T2,381 half, T2,
@@ -387,7 +387,7 @@ private:
387 vregFloatO, vregHalfO, mask);387 vregFloatO, vregHalfO, mask);
388 } else if constexpr (IsSameType<T2, uint8_t>::value) {388 } else if constexpr (IsSameType<T2, uint8_t>::value) {
389 // uint8389 // uint8
390- AscendC::MicroAPI::DataCopy<T2, AscendC::MicroAPI::LoadDist::DIST_UNPACK4_B8>(390+ AscendC::MicroAPI::LoadAlign<T2, AscendC::MicroAPI::LoadDist::DIST_UNPACK4_B8>(
391 vregO, offsetLocalAddr + i * VL + j * tilingData_.baseLen);391 vregO, offsetLocalAddr + i * VL + j * tilingData_.baseLen);
392 AscendC::MicroAPI::Cast<392 AscendC::MicroAPI::Cast<
393 half, T2,393 half, T2,
@@ -399,11 +399,11 @@ private:
399 vregFloatO, vregHalfO, mask);399 vregFloatO, vregHalfO, mask);
400 } else if constexpr (IsSameType<T2, float>::value) {400 } else if constexpr (IsSameType<T2, float>::value) {
401 // fp32401 // fp32
402- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(402+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(
403 vregFloatO, offsetLocalAddr + i * VL + j * tilingData_.baseLen);403 vregFloatO, offsetLocalAddr + i * VL + j * tilingData_.baseLen);
404 } else if constexpr (IsSameType<T2, half>::value) {404 } else if constexpr (IsSameType<T2, half>::value) {
405 // fp16405 // fp16
406- AscendC::MicroAPI::DataCopy<half, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(406+ AscendC::MicroAPI::LoadAlign<half, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(
407 vregO, offsetLocalAddr + i * VL + j * tilingData_.baseLen);407 vregO, offsetLocalAddr + i * VL + j * tilingData_.baseLen);
408 AscendC::MicroAPI::Cast<408 AscendC::MicroAPI::Cast<
409 float, half,409 float, half,
@@ -411,7 +411,7 @@ private:
411 vregFloatO, vregO, mask);411 vregFloatO, vregO, mask);
412 } else {412 } else {
413 // bf16413 // bf16
414- AscendC::MicroAPI::DataCopy<T2, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(414+ AscendC::MicroAPI::LoadAlign<T2, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(
415 vregO, offsetLocalAddr + i * VL + j * tilingData_.baseLen);415 vregO, offsetLocalAddr + i * VL + j * tilingData_.baseLen);
416 AscendC::MicroAPI::Cast<416 AscendC::MicroAPI::Cast<
417 float, T2,417 float, T2,
@@ -442,7 +442,7 @@ private:
442 U, float,442 U, float,
443 QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_FP32_TO_HIFP8>(443 QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_FP32_TO_HIFP8>(
444 vregY, vregFloatY, mask);444 vregY, vregFloatY, mask);
445- AscendC::MicroAPI::DataCopy<U, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(445+ AscendC::MicroAPI::StoreAlign<U, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(
446 outLocalAddr + i * VL + j * tilingData_.baseLen, vregY, mask);446 outLocalAddr + i * VL + j * tilingData_.baseLen, vregY, mask);
447 } else if constexpr (IsSameType<U, fp8_e5m2_t>::value) {447 } else if constexpr (IsSameType<U, fp8_e5m2_t>::value) {
448 // fp8_e5m2448 // fp8_e5m2
@@ -450,7 +450,7 @@ private:
450 U, float,450 U, float,
451 QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_FP32_TO_FP8E5M2>(451 QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_FP32_TO_FP8E5M2>(
452 vregY, vregFloatY, mask);452 vregY, vregFloatY, mask);
453- AscendC::MicroAPI::DataCopy<U, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(453+ AscendC::MicroAPI::StoreAlign<U, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(
454 outLocalAddr + i * VL + j * tilingData_.baseLen, vregY, mask);454 outLocalAddr + i * VL + j * tilingData_.baseLen, vregY, mask);
455 } else if constexpr (IsSameType<U, fp8_e4m3fn_t>::value) {455 } else if constexpr (IsSameType<U, fp8_e4m3fn_t>::value) {
456 // fp8_e4m3456 // fp8_e4m3
@@ -458,7 +458,7 @@ private:
458 U, float,458 U, float,
459 QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_FP32_TO_FP8E4M3>(459 QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_FP32_TO_FP8E4M3>(
460 vregY, vregFloatY, mask);460 vregY, vregFloatY, mask);
461- AscendC::MicroAPI::DataCopy<U, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(461+ AscendC::MicroAPI::StoreAlign<U, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(
462 outLocalAddr + i * VL + j * tilingData_.baseLen, vregY, mask);462 outLocalAddr + i * VL + j * tilingData_.baseLen, vregY, mask);
463 } else if constexpr (IsSameType<U, int8_t>::value) {463 } else if constexpr (IsSameType<U, int8_t>::value) {
464 // int8 支持464 // int8 支持
@@ -475,7 +475,7 @@ private:
475 int8_t, half,475 int8_t, half,
476 QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_HALF_TO_INT8>(476 QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_HALF_TO_INT8>(
477 vregY, vregHalfY, mask);477 vregY, vregHalfY, mask);
478- AscendC::MicroAPI::DataCopy<U, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(478+ AscendC::MicroAPI::StoreAlign<U, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(
479 outLocalAddr + i * VL + j * tilingData_.baseLen, vregY, mask);479 outLocalAddr + i * VL + j * tilingData_.baseLen, vregY, mask);
480 } else if constexpr (IsSameType<U, uint8_t>::value) {480 } else if constexpr (IsSameType<U, uint8_t>::value) {
481 // uint8481 // uint8
@@ -492,7 +492,7 @@ private:
492 uint8_t, half,492 uint8_t, half,
493 QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_HALF_TO_UINT8>(493 QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_HALF_TO_UINT8>(
494 vregY, vregHalfY, mask);494 vregY, vregHalfY, mask);
495- AscendC::MicroAPI::DataCopy<U, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(495+ AscendC::MicroAPI::StoreAlign<U, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(
496 outLocalAddr + i * VL + j * tilingData_.baseLen, vregY, mask);496 outLocalAddr + i * VL + j * tilingData_.baseLen, vregY, mask);
497 } else if constexpr (IsSameType<U, int32_t>::value) {497 } else if constexpr (IsSameType<U, int32_t>::value) {
498 // int32498 // int32
@@ -500,7 +500,7 @@ private:
500 U, float,500 U, float,
501 QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_FP32_TO_INT32>(501 QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_FP32_TO_INT32>(
502 vregY, vregFloatY, mask);502 vregY, vregFloatY, mask);
503- AscendC::MicroAPI::DataCopy<U, AscendC::MicroAPI::StoreDist::DIST_NORM>(503+ AscendC::MicroAPI::StoreAlign<U, AscendC::MicroAPI::StoreDist::DIST_NORM>(
504 outLocalAddr + i * VL + j * tilingData_.baseLen, vregY, mask);504 outLocalAddr + i * VL + j * tilingData_.baseLen, vregY, mask);
505 } else if constexpr (IsSameType<U, int4b_t>::value) {505 } else if constexpr (IsSameType<U, int4b_t>::value) {
506 AscendC::MicroAPI::RegTensor<int16_t> vregInt16Y;506 AscendC::MicroAPI::RegTensor<int16_t> vregInt16Y;
@@ -521,7 +521,7 @@ private:
521 QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_F16_TO_I8>(521 QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_F16_TO_I8>(
522 (AscendC::MicroAPI::RegTensor<int4x2_t>&)vregTmp2Y,522 (AscendC::MicroAPI::RegTensor<int4x2_t>&)vregTmp2Y,
523 (AscendC::MicroAPI::RegTensor<half>&)vregTmp1Y, mask);523 (AscendC::MicroAPI::RegTensor<half>&)vregTmp1Y, mask);
524- AscendC::MicroAPI::DataCopy<yCopyDtype, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(524+ AscendC::MicroAPI::StoreAlign<yCopyDtype, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(
525 outLocalAddr + (i * VL / 2 + j * tilingData_.baseLen), vregTmp2Y, mask4Int4);525 outLocalAddr + (i * VL / 2 + j * tilingData_.baseLen), vregTmp2Y, mask4Int4);
526 }526 }
527 }527 }
@@ -532,4 +532,4 @@ private:
532 }532 }
533};533};
534} // namespace QuantizeOp534} // namespace QuantizeOp
535-#endif // QUANTIZE_PERHEAD_REGBASE_H535+#endif // QUANTIZE_PERHEAD_REGBASE_H
Mquant/quantize/op_kernel/arch35/quantize_per_tensor_no_offset_regbase.h+19-18
@@ -157,9 +157,9 @@ __aicore__ inline void QuantizePerTensorNoOffsetRegbase<T, T1, T2, U, DivMode, R
157 LocalTensor<T> xLocal = inQueueX_.DeQue<T>();157 LocalTensor<T> xLocal = inQueueX_.DeQue<T>();
158 LocalTensor<yCopyDtype> outLocal = outQueueY_.AllocTensor<yCopyDtype>();158 LocalTensor<yCopyDtype> outLocal = outQueueY_.AllocTensor<yCopyDtype>();
159 159 
160- __local_mem__ T* xLocalAddr = (__local_mem__ T*)xLocal.GetPhyAddr();160+ __ubuf__ T* xLocalAddr = (__ubuf__ T*)xLocal.GetPhyAddr();
161- __local_mem__ T1* scaleLocalAddr = (__local_mem__ T1*)sLocal.GetPhyAddr();161+ __ubuf__ T1* scaleLocalAddr = (__ubuf__ T1*)sLocal.GetPhyAddr();
162- __local_mem__ yCopyDtype* outLocalAddr = (__local_mem__ yCopyDtype*)outLocal.GetPhyAddr();162+ __ubuf__ yCopyDtype* outLocalAddr = (__ubuf__ yCopyDtype*)outLocal.GetPhyAddr();
163 163 
164 uint16_t VL = AscendC::VECTOR_REG_WIDTH / sizeof(float);164 uint16_t VL = AscendC::VECTOR_REG_WIDTH / sizeof(float);
165 uint32_t xLocalOffset = static_cast<uint32_t>(tilingData_.baseLen);165 uint32_t xLocalOffset = static_cast<uint32_t>(tilingData_.baseLen);
@@ -190,18 +190,18 @@ __aicore__ inline void QuantizePerTensorNoOffsetRegbase<T, T1, T2, U, DivMode, R
190 // ld for x190 // ld for x
191 if constexpr (IsSameType<T, float>::value) {191 if constexpr (IsSameType<T, float>::value) {
192 // fp32192 // fp32
193- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(193+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(
194 vregFloatX, xLocalAddr + i * VL + j * xLocalOffset);194 vregFloatX, xLocalAddr + i * VL + j * xLocalOffset);
195 } else if constexpr (IsSameType<T, half>::value) {195 } else if constexpr (IsSameType<T, half>::value) {
196 // fp16196 // fp16
197- AscendC::MicroAPI::DataCopy<half, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(197+ AscendC::MicroAPI::LoadAlign<half, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(
198 vregX, xLocalAddr + i * VL + j * xLocalOffset);198 vregX, xLocalAddr + i * VL + j * xLocalOffset);
199 AscendC::MicroAPI::Cast<199 AscendC::MicroAPI::Cast<
200 float, half, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_HALF_TO_FP32>(200 float, half, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_HALF_TO_FP32>(
201 vregFloatX, vregX, mask);201 vregFloatX, vregX, mask);
202 } else if constexpr (IsSameType<T, bfloat16_t>::value) {202 } else if constexpr (IsSameType<T, bfloat16_t>::value) {
203 // bf16203 // bf16
204- AscendC::MicroAPI::DataCopy<T, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(204+ AscendC::MicroAPI::LoadAlign<T, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(
205 vregX, xLocalAddr + i * VL + j * xLocalOffset);205 vregX, xLocalAddr + i * VL + j * xLocalOffset);
206 AscendC::MicroAPI::Cast<206 AscendC::MicroAPI::Cast<
207 float, T, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_BF16_TO_FP32>(207 float, T, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_BF16_TO_FP32>(
@@ -211,17 +211,18 @@ __aicore__ inline void QuantizePerTensorNoOffsetRegbase<T, T1, T2, U, DivMode, R
211 // ld for scale211 // ld for scale
212 if constexpr (IsSameType<T1, float>::value) {212 if constexpr (IsSameType<T1, float>::value) {
213 // fp32213 // fp32
214- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_BRC_B32>(vregFloatS,214+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_BRC_B32>(vregFloatS,
215- scaleLocalAddr);215+ scaleLocalAddr);
216 } else if constexpr (IsSameType<T1, half>::value) {216 } else if constexpr (IsSameType<T1, half>::value) {
217 // fp16217 // fp16
218- AscendC::MicroAPI::DataCopy<half, AscendC::MicroAPI::LoadDist::DIST_BRC_B16>(vregS, scaleLocalAddr);218+ AscendC::MicroAPI::LoadAlign<half, AscendC::MicroAPI::LoadDist::DIST_BRC_B16>(vregS,
219+ scaleLocalAddr);
219 AscendC::MicroAPI::Cast<220 AscendC::MicroAPI::Cast<
220 float, T1, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_HALF_TO_FP32>(221 float, T1, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_HALF_TO_FP32>(
221 vregFloatS, vregS, mask);222 vregFloatS, vregS, mask);
222 } else if constexpr (IsSameType<T1, bfloat16_t>::value) {223 } else if constexpr (IsSameType<T1, bfloat16_t>::value) {
223 // bf16224 // bf16
224- AscendC::MicroAPI::DataCopy<T1, AscendC::MicroAPI::LoadDist::DIST_BRC_B16>(vregS, scaleLocalAddr);225+ AscendC::MicroAPI::LoadAlign<T1, AscendC::MicroAPI::LoadDist::DIST_BRC_B16>(vregS, scaleLocalAddr);
225 AscendC::MicroAPI::Cast<226 AscendC::MicroAPI::Cast<
226 float, T1, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_BF16_TO_FP32>(227 float, T1, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_BF16_TO_FP32>(
227 vregFloatS, vregS, mask);228 vregFloatS, vregS, mask);
@@ -243,21 +244,21 @@ __aicore__ inline void QuantizePerTensorNoOffsetRegbase<T, T1, T2, U, DivMode, R
243 AscendC::MicroAPI::Cast<244 AscendC::MicroAPI::Cast<
244 U, float, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_FP32_TO_HIFP8>(245 U, float, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_FP32_TO_HIFP8>(
245 vregY, vregFloatY, mask);246 vregY, vregFloatY, mask);
246- AscendC::MicroAPI::DataCopy<U, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(247+ AscendC::MicroAPI::StoreAlign<U, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(
247 outLocalAddr + i * VL + j * xLocalOffset, vregY, mask);248 outLocalAddr + i * VL + j * xLocalOffset, vregY, mask);
248 } else if constexpr (IsSameType<U, fp8_e5m2_t>::value) {249 } else if constexpr (IsSameType<U, fp8_e5m2_t>::value) {
249 // fp8_e5m2250 // fp8_e5m2
250 AscendC::MicroAPI::Cast<251 AscendC::MicroAPI::Cast<
251 U, float, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_FP32_TO_FP8E5M2>(252 U, float, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_FP32_TO_FP8E5M2>(
252 vregY, vregFloatY, mask);253 vregY, vregFloatY, mask);
253- AscendC::MicroAPI::DataCopy<U, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(254+ AscendC::MicroAPI::StoreAlign<U, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(
254 outLocalAddr + i * VL + j * xLocalOffset, vregY, mask);255 outLocalAddr + i * VL + j * xLocalOffset, vregY, mask);
255 } else if constexpr (IsSameType<U, fp8_e4m3fn_t>::value) {256 } else if constexpr (IsSameType<U, fp8_e4m3fn_t>::value) {
256 // fp8_e4m3257 // fp8_e4m3
257 AscendC::MicroAPI::Cast<258 AscendC::MicroAPI::Cast<
258 U, float, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_FP32_TO_FP8E4M3>(259 U, float, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_FP32_TO_FP8E4M3>(
259 vregY, vregFloatY, mask);260 vregY, vregFloatY, mask);
260- AscendC::MicroAPI::DataCopy<U, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(261+ AscendC::MicroAPI::StoreAlign<U, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(
261 outLocalAddr + i * VL + j * xLocalOffset, vregY, mask);262 outLocalAddr + i * VL + j * xLocalOffset, vregY, mask);
262 } else if constexpr (IsSameType<U, int8_t>::value) {263 } else if constexpr (IsSameType<U, int8_t>::value) {
263 // int8264 // int8
@@ -273,7 +274,7 @@ __aicore__ inline void QuantizePerTensorNoOffsetRegbase<T, T1, T2, U, DivMode, R
273 int8_t, half,274 int8_t, half,
274 QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_HALF_TO_INT8>(275 QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_HALF_TO_INT8>(
275 vregY, vregHalfY, mask);276 vregY, vregHalfY, mask);
276- AscendC::MicroAPI::DataCopy<U, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(277+ AscendC::MicroAPI::StoreAlign<U, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(
277 outLocalAddr + i * VL + j * xLocalOffset, vregY, mask);278 outLocalAddr + i * VL + j * xLocalOffset, vregY, mask);
278 } else if constexpr (IsSameType<U, uint8_t>::value) {279 } else if constexpr (IsSameType<U, uint8_t>::value) {
279 // uint8280 // uint8
@@ -289,14 +290,14 @@ __aicore__ inline void QuantizePerTensorNoOffsetRegbase<T, T1, T2, U, DivMode, R
289 uint8_t, half,290 uint8_t, half,
290 QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_HALF_TO_UINT8>(291 QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_HALF_TO_UINT8>(
291 vregY, vregHalfY, mask);292 vregY, vregHalfY, mask);
292- AscendC::MicroAPI::DataCopy<U, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(293+ AscendC::MicroAPI::StoreAlign<U, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(
293 outLocalAddr + i * VL + j * xLocalOffset, vregY, mask);294 outLocalAddr + i * VL + j * xLocalOffset, vregY, mask);
294 } else if constexpr (IsSameType<U, int32_t>::value) {295 } else if constexpr (IsSameType<U, int32_t>::value) {
295 // int32296 // int32
296 AscendC::MicroAPI::Cast<297 AscendC::MicroAPI::Cast<
297 U, float, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_FP32_TO_INT32>(298 U, float, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_FP32_TO_INT32>(
298 vregY, vregFloatY, mask);299 vregY, vregFloatY, mask);
299- AscendC::MicroAPI::DataCopy<U, AscendC::MicroAPI::StoreDist::DIST_NORM>(300+ AscendC::MicroAPI::StoreAlign<U, AscendC::MicroAPI::StoreDist::DIST_NORM>(
300 outLocalAddr + i * VL + j * xLocalOffset, vregY, mask);301 outLocalAddr + i * VL + j * xLocalOffset, vregY, mask);
301 } else if constexpr (IsSameType<U, int4b_t>::value) {302 } else if constexpr (IsSameType<U, int4b_t>::value) {
302 AscendC::MicroAPI::RegTensor<int16_t> vregInt16Y;303 AscendC::MicroAPI::RegTensor<int16_t> vregInt16Y;
@@ -316,7 +317,7 @@ __aicore__ inline void QuantizePerTensorNoOffsetRegbase<T, T1, T2, U, DivMode, R
316 int4x2_t, half, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_F16_TO_I8>(317 int4x2_t, half, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_F16_TO_I8>(
317 (AscendC::MicroAPI::RegTensor<int4x2_t>&)vregTmp2Y,318 (AscendC::MicroAPI::RegTensor<int4x2_t>&)vregTmp2Y,
318 (AscendC::MicroAPI::RegTensor<half>&)vregTmp1Y, mask);319 (AscendC::MicroAPI::RegTensor<half>&)vregTmp1Y, mask);
319- AscendC::MicroAPI::DataCopy<yCopyDtype, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(320+ AscendC::MicroAPI::StoreAlign<yCopyDtype, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(
320 outLocalAddr + (i * VL / 2 + j * tilingData_.baseLen), vregTmp2Y, mask4Int4);321 outLocalAddr + (i * VL / 2 + j * tilingData_.baseLen), vregTmp2Y, mask4Int4);
321 }322 }
322 }323 }
@@ -340,4 +341,4 @@ __aicore__ inline void QuantizePerTensorNoOffsetRegbase<T, T1, T2, U, DivMode, R
340 outQueueY_.FreeTensor(outLocal);341 outQueueY_.FreeTensor(outLocal);
341}342}
342} // namespace QuantizeOp343} // namespace QuantizeOp
343-#endif344+#endif
Mquant/quantize/op_kernel/arch35/quantize_per_tensor_regbase.h+27-26
@@ -180,13 +180,13 @@ __aicore__ inline void QuantizePerTensorRegbase<T, T1, T2, U, DivMode, RoundMode
180 LocalTensor<T> xLocal = inQueueX_.DeQue<T>();180 LocalTensor<T> xLocal = inQueueX_.DeQue<T>();
181 LocalTensor<yCopyDtype> outLocal = outQueueY_.AllocTensor<yCopyDtype>();181 LocalTensor<yCopyDtype> outLocal = outQueueY_.AllocTensor<yCopyDtype>();
182 182 
183- __local_mem__ T* xLocalAddr = (__local_mem__ T*)xLocal.GetPhyAddr();183+ __ubuf__ T* xLocalAddr = (__ubuf__ T*)xLocal.GetPhyAddr();
184- __local_mem__ T1* scaleLocalAddr = (__local_mem__ T1*)sLocal.GetPhyAddr();184+ __ubuf__ T1* scaleLocalAddr = (__ubuf__ T1*)sLocal.GetPhyAddr();
185- __local_mem__ yCopyDtype* outLocalAddr = (__local_mem__ yCopyDtype*)outLocal.GetPhyAddr();185+ __ubuf__ yCopyDtype* outLocalAddr = (__ubuf__ yCopyDtype*)outLocal.GetPhyAddr();
186 186 
187 uint16_t VL = AscendC::VECTOR_REG_WIDTH / sizeof(float);187 uint16_t VL = AscendC::VECTOR_REG_WIDTH / sizeof(float);
188 188 
189- __local_mem__ T2* offsetLocalAddr = (__local_mem__ T2*)oLocal.GetPhyAddr();189+ __ubuf__ T2* offsetLocalAddr = (__ubuf__ T2*)oLocal.GetPhyAddr();
190 uint32_t xLocalOffset = static_cast<uint32_t>(tilingData_.baseLen);190 uint32_t xLocalOffset = static_cast<uint32_t>(tilingData_.baseLen);
191 191 
192 // has offset192 // has offset
@@ -222,18 +222,18 @@ __aicore__ inline void QuantizePerTensorRegbase<T, T1, T2, U, DivMode, RoundMode
222 // ld and cast for x222 // ld and cast for x
223 if constexpr (IsSameType<T, float>::value) {223 if constexpr (IsSameType<T, float>::value) {
224 // fp32224 // fp32
225- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(225+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(
226 vregFloatX, xLocalAddr + i * VL + j * xLocalOffset);226 vregFloatX, xLocalAddr + i * VL + j * xLocalOffset);
227 } else if constexpr (IsSameType<T, half>::value) {227 } else if constexpr (IsSameType<T, half>::value) {
228 // fp16228 // fp16
229- AscendC::MicroAPI::DataCopy<half, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(229+ AscendC::MicroAPI::LoadAlign<half, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(
230 vregX, xLocalAddr + i * VL + j * xLocalOffset);230 vregX, xLocalAddr + i * VL + j * xLocalOffset);
231 AscendC::MicroAPI::Cast<231 AscendC::MicroAPI::Cast<
232 float, half, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_HALF_TO_FP32>(232 float, half, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_HALF_TO_FP32>(
233 vregFloatX, vregX, mask);233 vregFloatX, vregX, mask);
234 } else if constexpr (IsSameType<T, bfloat16_t>::value) {234 } else if constexpr (IsSameType<T, bfloat16_t>::value) {
235 // bf16235 // bf16
236- AscendC::MicroAPI::DataCopy<T, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(236+ AscendC::MicroAPI::LoadAlign<T, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(
237 vregX, xLocalAddr + i * VL + j * xLocalOffset);237 vregX, xLocalAddr + i * VL + j * xLocalOffset);
238 AscendC::MicroAPI::Cast<238 AscendC::MicroAPI::Cast<
239 float, T, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_BF16_TO_FP32>(239 float, T, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_BF16_TO_FP32>(
@@ -243,17 +243,18 @@ __aicore__ inline void QuantizePerTensorRegbase<T, T1, T2, U, DivMode, RoundMode
243 // ld and cast for scale243 // ld and cast for scale
244 if constexpr (IsSameType<T1, float>::value) {244 if constexpr (IsSameType<T1, float>::value) {
245 // fp32245 // fp32
246- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_BRC_B32>(vregFloatS,246+ AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_BRC_B32>(vregFloatS,
247- scaleLocalAddr);247+ scaleLocalAddr);
248 } else if constexpr (IsSameType<T1, half>::value) {248 } else if constexpr (IsSameType<T1, half>::value) {
249 // fp16249 // fp16
250- AscendC::MicroAPI::DataCopy<half, AscendC::MicroAPI::LoadDist::DIST_BRC_B16>(vregS, scaleLocalAddr);250+ AscendC::MicroAPI::LoadAlign<half, AscendC::MicroAPI::LoadDist::DIST_BRC_B16>(vregS,
251+ scaleLocalAddr);
251 AscendC::MicroAPI::Cast<252 AscendC::MicroAPI::Cast<
252 float, T1, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_HALF_TO_FP32>(253 float, T1, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_HALF_TO_FP32>(
253 vregFloatS, vregS, mask);254 vregFloatS, vregS, mask);
254 } else if constexpr (IsSameType<T1, bfloat16_t>::value) {255 } else if constexpr (IsSameType<T1, bfloat16_t>::value) {
255 // bf16256 // bf16
256- AscendC::MicroAPI::DataCopy<T1, AscendC::MicroAPI::LoadDist::DIST_BRC_B16>(vregS, scaleLocalAddr);257+ AscendC::MicroAPI::LoadAlign<T1, AscendC::MicroAPI::LoadDist::DIST_BRC_B16>(vregS, scaleLocalAddr);
257 AscendC::MicroAPI::Cast<258 AscendC::MicroAPI::Cast<
258 float, T1, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_BF16_TO_FP32>(259 float, T1, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_BF16_TO_FP32>(
259 vregFloatS, vregS, mask);260 vregFloatS, vregS, mask);
@@ -262,13 +263,13 @@ __aicore__ inline void QuantizePerTensorRegbase<T, T1, T2, U, DivMode, RoundMode
262 // ld and cast for offset263 // ld and cast for offset
263 if constexpr (IsSameType<T2, int32_t>::value) {264 if constexpr (IsSameType<T2, int32_t>::value) {
264 // int32265 // int32
265- AscendC::MicroAPI::DataCopy<T2, AscendC::MicroAPI::LoadDist::DIST_BRC_B32>(vregO, offsetLocalAddr);266+ AscendC::MicroAPI::LoadAlign<T2, AscendC::MicroAPI::LoadDist::DIST_BRC_B32>(vregO, offsetLocalAddr);
266 AscendC::MicroAPI::Cast<267 AscendC::MicroAPI::Cast<
267 float, T2, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_INT32_TO_FP32>(268 float, T2, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_INT32_TO_FP32>(
268 vregFloatO, vregO, mask);269 vregFloatO, vregO, mask);
269 } else if constexpr (IsSameType<T2, int8_t>::value) {270 } else if constexpr (IsSameType<T2, int8_t>::value) {
270 // int8271 // int8
271- AscendC::MicroAPI::DataCopy<T2, AscendC::MicroAPI::LoadDist::DIST_BRC_B8>(vregO, offsetLocalAddr);272+ AscendC::MicroAPI::LoadAlign<T2, AscendC::MicroAPI::LoadDist::DIST_BRC_B8>(vregO, offsetLocalAddr);
272 AscendC::MicroAPI::Cast<273 AscendC::MicroAPI::Cast<
273 half, T2, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_INT8_TO_HALF>(274 half, T2, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_INT8_TO_HALF>(
274 vregHalfO, vregO, mask);275 vregHalfO, vregO, mask);
@@ -277,7 +278,7 @@ __aicore__ inline void QuantizePerTensorRegbase<T, T1, T2, U, DivMode, RoundMode
277 vregFloatO, vregHalfO, mask);278 vregFloatO, vregHalfO, mask);
278 } else if constexpr (IsSameType<T2, uint8_t>::value) {279 } else if constexpr (IsSameType<T2, uint8_t>::value) {
279 // uint8280 // uint8
280- AscendC::MicroAPI::DataCopy<T2, AscendC::MicroAPI::LoadDist::DIST_BRC_B8>(vregO, offsetLocalAddr);281+ AscendC::MicroAPI::LoadAlign<T2, AscendC::MicroAPI::LoadDist::DIST_BRC_B8>(vregO, offsetLocalAddr);
281 AscendC::MicroAPI::Cast<282 AscendC::MicroAPI::Cast<
282 half, T2, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_UINT8_TO_HALF>(283 half, T2, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_UINT8_TO_HALF>(
283 vregHalfO, vregO, mask);284 vregHalfO, vregO, mask);
@@ -286,20 +287,20 @@ __aicore__ inline void QuantizePerTensorRegbase<T, T1, T2, U, DivMode, RoundMode
286 vregFloatO, vregHalfO, mask);287 vregFloatO, vregHalfO, mask);
287 } else if constexpr (IsSameType<T2, bfloat16_t>::value) {288 } else if constexpr (IsSameType<T2, bfloat16_t>::value) {
288 // bf16289 // bf16
289- AscendC::MicroAPI::DataCopy<T2, AscendC::MicroAPI::LoadDist::DIST_BRC_B16>(vregO, offsetLocalAddr);290+ AscendC::MicroAPI::LoadAlign<T2, AscendC::MicroAPI::LoadDist::DIST_BRC_B16>(vregO, offsetLocalAddr);
290 AscendC::MicroAPI::Cast<291 AscendC::MicroAPI::Cast<
291 float, T2, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_BF16_TO_FP32>(292 float, T2, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_BF16_TO_FP32>(
292 vregFloatO, vregO, mask);293 vregFloatO, vregO, mask);
293 } else if constexpr (IsSameType<T2, half>::value) {294 } else if constexpr (IsSameType<T2, half>::value) {
294 // fp16295 // fp16
295- AscendC::MicroAPI::DataCopy<T2, AscendC::MicroAPI::LoadDist::DIST_BRC_B16>(vregO, offsetLocalAddr);296+ AscendC::MicroAPI::LoadAlign<T2, AscendC::MicroAPI::LoadDist::DIST_BRC_B16>(vregO, offsetLocalAddr);
296 AscendC::MicroAPI::Cast<297 AscendC::MicroAPI::Cast<
297 float, T2, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_HALF_TO_FP32>(298 float, T2, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_HALF_TO_FP32>(
298 vregFloatO, vregO, mask);299 vregFloatO, vregO, mask);
299 } else if constexpr (IsSameType<T2, float>::value) {300 } else if constexpr (IsSameType<T2, float>::value) {
300 // fp32301 // fp32
301- AscendC::MicroAPI::DataCopy<T2, AscendC::MicroAPI::LoadDist::DIST_BRC_B32>(vregFloatO,302+ AscendC::MicroAPI::LoadAlign<T2, AscendC::MicroAPI::LoadDist::DIST_BRC_B32>(vregFloatO,
302- offsetLocalAddr);303+ offsetLocalAddr);
303 }304 }
304 if constexpr (SqrtMode == TPL_SQRT_MODE) {305 if constexpr (SqrtMode == TPL_SQRT_MODE) {
305 AscendC::MicroAPI::Mul(vregFloatS, vregFloatS, vregFloatS, mask);306 AscendC::MicroAPI::Mul(vregFloatS, vregFloatS, vregFloatS, mask);
@@ -320,21 +321,21 @@ __aicore__ inline void QuantizePerTensorRegbase<T, T1, T2, U, DivMode, RoundMode
320 AscendC::MicroAPI::Cast<321 AscendC::MicroAPI::Cast<
321 U, float, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_FP32_TO_HIFP8>(322 U, float, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_FP32_TO_HIFP8>(
322 vregY, vregFloatY, mask);323 vregY, vregFloatY, mask);
323- AscendC::MicroAPI::DataCopy<U, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(324+ AscendC::MicroAPI::StoreAlign<U, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(
324 outLocalAddr + i * VL + j * xLocalOffset, vregY, mask);325 outLocalAddr + i * VL + j * xLocalOffset, vregY, mask);
325 } else if constexpr (IsSameType<U, fp8_e5m2_t>::value) {326 } else if constexpr (IsSameType<U, fp8_e5m2_t>::value) {
326 // fp8_e5m2327 // fp8_e5m2
327 AscendC::MicroAPI::Cast<328 AscendC::MicroAPI::Cast<
328 U, float, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_FP32_TO_FP8E5M2>(329 U, float, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_FP32_TO_FP8E5M2>(
329 vregY, vregFloatY, mask);330 vregY, vregFloatY, mask);
330- AscendC::MicroAPI::DataCopy<U, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(331+ AscendC::MicroAPI::StoreAlign<U, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(
331 outLocalAddr + i * VL + j * xLocalOffset, vregY, mask);332 outLocalAddr + i * VL + j * xLocalOffset, vregY, mask);
332 } else if constexpr (IsSameType<U, fp8_e4m3fn_t>::value) {333 } else if constexpr (IsSameType<U, fp8_e4m3fn_t>::value) {
333 // fp8_e4m3334 // fp8_e4m3
334 AscendC::MicroAPI::Cast<335 AscendC::MicroAPI::Cast<
335 U, float, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_FP32_TO_FP8E4M3>(336 U, float, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_FP32_TO_FP8E4M3>(
336 vregY, vregFloatY, mask);337 vregY, vregFloatY, mask);
337- AscendC::MicroAPI::DataCopy<U, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(338+ AscendC::MicroAPI::StoreAlign<U, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(
338 outLocalAddr + i * VL + j * xLocalOffset, vregY, mask);339 outLocalAddr + i * VL + j * xLocalOffset, vregY, mask);
339 } else if constexpr (IsSameType<U, int8_t>::value) {340 } else if constexpr (IsSameType<U, int8_t>::value) {
340 // int8341 // int8
@@ -350,7 +351,7 @@ __aicore__ inline void QuantizePerTensorRegbase<T, T1, T2, U, DivMode, RoundMode
350 int8_t, half,351 int8_t, half,
351 QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_HALF_TO_INT8>(352 QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_HALF_TO_INT8>(
352 vregY, vregHalfY, mask);353 vregY, vregHalfY, mask);
353- AscendC::MicroAPI::DataCopy<U, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(354+ AscendC::MicroAPI::StoreAlign<U, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(
354 outLocalAddr + i * VL + j * xLocalOffset, vregY, mask);355 outLocalAddr + i * VL + j * xLocalOffset, vregY, mask);
355 } else if constexpr (IsSameType<U, uint8_t>::value) {356 } else if constexpr (IsSameType<U, uint8_t>::value) {
356 // uint8357 // uint8
@@ -366,14 +367,14 @@ __aicore__ inline void QuantizePerTensorRegbase<T, T1, T2, U, DivMode, RoundMode
366 uint8_t, half,367 uint8_t, half,
367 QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_HALF_TO_UINT8>(368 QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_HALF_TO_UINT8>(
368 vregY, vregHalfY, mask);369 vregY, vregHalfY, mask);
369- AscendC::MicroAPI::DataCopy<U, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(370+ AscendC::MicroAPI::StoreAlign<U, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(
370 outLocalAddr + i * VL + j * xLocalOffset, vregY, mask);371 outLocalAddr + i * VL + j * xLocalOffset, vregY, mask);
371 } else if constexpr (IsSameType<U, int32_t>::value) {372 } else if constexpr (IsSameType<U, int32_t>::value) {
372 // int32373 // int32
373 AscendC::MicroAPI::Cast<374 AscendC::MicroAPI::Cast<
374 U, float, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_FP32_TO_INT32>(375 U, float, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_FP32_TO_INT32>(
375 vregY, vregFloatY, mask);376 vregY, vregFloatY, mask);
376- AscendC::MicroAPI::DataCopy<U, AscendC::MicroAPI::StoreDist::DIST_NORM>(377+ AscendC::MicroAPI::StoreAlign<U, AscendC::MicroAPI::StoreDist::DIST_NORM>(
377 outLocalAddr + i * VL + j * xLocalOffset, vregY, mask);378 outLocalAddr + i * VL + j * xLocalOffset, vregY, mask);
378 } else if constexpr (IsSameType<U, int4b_t>::value) {379 } else if constexpr (IsSameType<U, int4b_t>::value) {
379 AscendC::MicroAPI::RegTensor<int16_t> vregInt16Y;380 AscendC::MicroAPI::RegTensor<int16_t> vregInt16Y;
@@ -393,7 +394,7 @@ __aicore__ inline void QuantizePerTensorRegbase<T, T1, T2, U, DivMode, RoundMode
393 int4x2_t, half, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_F16_TO_I8>(394 int4x2_t, half, QuantizeBase<T, T1, T2, U, DivMode, RoundMode, SqrtMode>::CAST_TRAIT_F16_TO_I8>(
394 (AscendC::MicroAPI::RegTensor<int4x2_t>&)vregTmp2Y,395 (AscendC::MicroAPI::RegTensor<int4x2_t>&)vregTmp2Y,
395 (AscendC::MicroAPI::RegTensor<half>&)vregTmp1Y, mask);396 (AscendC::MicroAPI::RegTensor<half>&)vregTmp1Y, mask);
396- AscendC::MicroAPI::DataCopy<yCopyDtype, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(397+ AscendC::MicroAPI::StoreAlign<yCopyDtype, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(
397 outLocalAddr + (i * VL / 2 + j * tilingData_.baseLen), vregTmp2Y, mask4Int4);398 outLocalAddr + (i * VL / 2 + j * tilingData_.baseLen), vregTmp2Y, mask4Int4);
398 }399 }
399 }400 }
@@ -417,4 +418,4 @@ __aicore__ inline void QuantizePerTensorRegbase<T, T1, T2, U, DivMode, RoundMode
417 outQueueY_.FreeTensor(outLocal);418 outQueueY_.FreeTensor(outLocal);
418}419}
419} // namespace QuantizeOp420} // namespace QuantizeOp
420-#endif421+#endif