Merged
feat(layer_norm): reuse LayerNormV3 two-pass perf template #13207
rkAdd on 22 days ago
feat(layer_norm): reuse LayerNormV3 two-pass perf template #13207
Merged
rkAdd on 22 days ago
Total: 4 files changed+53-40
@@ -36,7 +36,7 @@ protected:
36 static void TearDownTestCase() { std::cout << "LayerNormTiling TearDown" << std::endl; }36 static void TearDownTestCase() { std::cout << "LayerNormTiling TearDown" << std::endl; }
37};37};
38 38 
39-TEST_F(LayerNormTilingTest, layer_norm_tiling_regbase_two_pass)39+TEST_F(LayerNormTilingTest, layer_norm_tiling_regbase_two_pass_perf)
40{40{
41 gert::StorageShape x_shape = {{64, 4096}, {64, 4096}};41 gert::StorageShape x_shape = {{64, 4096}, {64, 4096}};
42 gert::StorageShape gamma_shape = {{4096}, {4096}};42 gert::StorageShape gamma_shape = {{4096}, {4096}};
@@ -129,7 +129,7 @@ TEST_F(LayerNormTilingTest, layer_norm_tiling_regbase_two_pass)
129 EXPECT_EQ(tiling_func(tiling_context), ge::GRAPH_SUCCESS);129 EXPECT_EQ(tiling_func(tiling_context), ge::GRAPH_SUCCESS);
130 // todo check tiling result130 // todo check tiling result
131 auto tiling_key = tiling_context->GetTilingKey();131 auto tiling_key = tiling_context->GetTilingKey();
132- // ASSERT_EQ(tiling_key, 300);132+ ASSERT_EQ(tiling_key, 500);
133 auto tilingData = tiling_context->GetRawTilingData();133 auto tilingData = tiling_context->GetRawTilingData();
134 ASSERT_NE(tilingData, nullptr);134 ASSERT_NE(tilingData, nullptr);
135 // EXPECT_EQ(to_string<int32_t>(tilingData->GetData(), tilingData->GetDataSize()),135 // EXPECT_EQ(to_string<int32_t>(tilingData->GetData(), tilingData->GetDataSize()),
@@ -51,8 +51,11 @@ bool LayerNormV3RegBaseTwoPassPerfTiling::CanFitInBuffer(int64_t curA)
51 int64_t ubCanUseSize = GetUBCanUseSize();51 int64_t ubCanUseSize = GetUBCanUseSize();
52 int64_t rowWeight = GetRowWeight();52 int64_t rowWeight = GetRowWeight();
53 53 
54- return curA * static_cast<int64_t>(commonParams.rowAlign) * rowWeight <=54+ int64_t outputBufferSize = LNV3_NUM_TWO * LNV3_DOUBLE_BUFFER * curAAlign * sizeof(float);
fanzhaoyun001
fanzhaoyun001fanzhaoyun00116 days ago

[Major] 这个 UB 预算修正同时改了 LayerNormV3 的 aUbFactor,建议补 V3 的回归数据:kernel 里 meanQueue_/lastoutQueue_ 都是 DOUBLE_BUFFER * aUbFactorAlignB32 * sizeof(float),实际 16B/元素,而原式 LNV3_NUM_TWO * LNV3_DOUBLE_BUFFER * curAAlign 只扣了 4B/元素(少算 12B)。乘上 sizeof(float) 后是对的,但 V3 走的是同一份 CanFitInBuffer,它的 aUbFactor 也会跟着变小、UB loop 变多。PR 描述写"未提供性能数据",建议补 V3(大 A / 大 rowAlign 场景)的精度与性能对比,确认这次修正没有把 V3 拖慢。

likedislike
55- ubCanUseSize - LNV3_NUM_TWO * LNV3_DOUBLE_BUFFER * curAAlign;55+ if (commonParams.isV1) {
56+ outputBufferSize += curAAlign * sizeof(float);
57+ }
58+ return curA * static_cast<int64_t>(commonParams.rowAlign) * rowWeight <= ubCanUseSize - outputBufferSize;
56}59}
57 60 
58bool LayerNormV3RegBaseTwoPassPerfTiling::IsCapable()61bool LayerNormV3RegBaseTwoPassPerfTiling::IsCapable()
@@ -162,4 +165,5 @@ ge::graphStatus LayerNormV3RegBaseTwoPassPerfTiling::PostTiling()
162}165}
163 166 
164REGISTER_OPS_TILING_TEMPLATE(LayerNormV3, LayerNormV3RegBaseTwoPassPerfTiling, 150);167REGISTER_OPS_TILING_TEMPLATE(LayerNormV3, LayerNormV3RegBaseTwoPassPerfTiling, 150);
168+REGISTER_OPS_TILING_TEMPLATE(LayerNorm, LayerNormV3RegBaseTwoPassPerfTiling, 150);
165} // namespace optiling169} // namespace optiling
@@ -36,7 +36,7 @@ using AscendC::Reg::UpdateMask;
36using NormCommon::NormCommonRegbase::LoadRegForDtype;36using NormCommon::NormCommonRegbase::LoadRegForDtype;
37using NormCommon::NormCommonRegbase::StoreRegForDtype;37using NormCommon::NormCommonRegbase::StoreRegForDtype;
38 38 
39-template <typename T, typename U, typename M>39+template <typename T, typename U, typename M, bool IsOutRstd>
40class LayerNormV3RegbaseTwoPassPerf {40class LayerNormV3RegbaseTwoPassPerf {
41public:41public:
42 __aicore__ inline LayerNormV3RegbaseTwoPassPerf(const LayerNormV3TilingDataRegBaseTwoPassPerf* tilingData,42 __aicore__ inline LayerNormV3RegbaseTwoPassPerf(const LayerNormV3TilingDataRegBaseTwoPassPerf* tilingData,
@@ -48,7 +48,7 @@ public:
48 hasBeta_ = (tilingData->nullptrBeta == 0);48 hasBeta_ = (tilingData->nullptrBeta == 0);
49 }49 }
50 50 
51- __aicore__ inline void Init(GM_ADDR x, GM_ADDR gamma, GM_ADDR beta, GM_ADDR y, GM_ADDR mean, GM_ADDR rstd)51+ __aicore__ inline void Init(GM_ADDR x, GM_ADDR gamma, GM_ADDR beta, GM_ADDR y, GM_ADDR mean, GM_ADDR lastout)
52 {52 {
53 // copy in gamma and beta53 // copy in gamma and beta
54 gammaGm_.SetGlobalBuffer((__gm__ U*)gamma);54 gammaGm_.SetGlobalBuffer((__gm__ U*)gamma);
@@ -60,10 +60,13 @@ public:
60 xGm_.SetGlobalBuffer((__gm__ T*)x + xBlockOffset);60 xGm_.SetGlobalBuffer((__gm__ T*)x + xBlockOffset);
61 yGm_.SetGlobalBuffer((__gm__ T*)y + xBlockOffset);61 yGm_.SetGlobalBuffer((__gm__ T*)y + xBlockOffset);
62 meanGm_.SetGlobalBuffer((__gm__ M*)mean + meanBlockOffset);62 meanGm_.SetGlobalBuffer((__gm__ M*)mean + meanBlockOffset);
63- rstdGm_.SetGlobalBuffer((__gm__ M*)rstd + meanBlockOffset);63+ lastoutGm_.SetGlobalBuffer((__gm__ M*)lastout + meanBlockOffset);
64 64 
65 pipe_->InitBuffer(meanQueue_, DOUBLE_BUFFER, tl_->aUbFactorAlignB32 * sizeof(float));65 pipe_->InitBuffer(meanQueue_, DOUBLE_BUFFER, tl_->aUbFactorAlignB32 * sizeof(float));
66- pipe_->InitBuffer(rstdQueue_, DOUBLE_BUFFER, tl_->aUbFactorAlignB32 * sizeof(float));66+ pipe_->InitBuffer(lastoutQueue_, DOUBLE_BUFFER, tl_->aUbFactorAlignB32 * sizeof(float));
67+ if constexpr (!IsOutRstd) {
68+ pipe_->InitBuffer(rstdBuf_, tl_->aUbFactorAlignB32 * sizeof(float));
69+ }
67 70 
68 elemNum_ = tl_->aUbFactor * tl_->rAlign;71 elemNum_ = tl_->aUbFactor * tl_->rAlign;
69 pipe_->InitBuffer(xQueue_, DOUBLE_BUFFER, elemNum_ * sizeof(T));72 pipe_->InitBuffer(xQueue_, DOUBLE_BUFFER, elemNum_ * sizeof(T));
@@ -87,7 +90,7 @@ public:
87 CopyInX(offset, currentA);90 CopyInX(offset, currentA);
88 Caculate(currentA);91 Caculate(currentA);
89 CopyOutY(offset, currentA);92 CopyOutY(offset, currentA);
90- CopyOutMeanRstd(aOffset, currentA);93+ CopyOutMeanLastout(aOffset, currentA);
91 }94 }
92 95 
93 gammaBetaQueue_.FreeTensor(gammaBetaInUb_);96 gammaBetaQueue_.FreeTensor(gammaBetaInUb_);
@@ -137,12 +140,12 @@ private:
137 {140 {
138 LocalTensor<T> xInUb = xQueue_.template DeQue<T>();141 LocalTensor<T> xInUb = xQueue_.template DeQue<T>();
139 meanOutUb_ = meanQueue_.AllocTensor<float>();142 meanOutUb_ = meanQueue_.AllocTensor<float>();
140- rstdOutUb_ = rstdQueue_.AllocTensor<float>();143+ lastoutOutUb_ = lastoutQueue_.AllocTensor<float>();
141 LocalTensor<float> tmpTensor = tmpBuf.Get<float>();144 LocalTensor<float> tmpTensor = tmpBuf.Get<float>();
142 145 
143 __ubuf__ T* xInUbAddr = (__ubuf__ T*)xInUb.GetPhyAddr();146 __ubuf__ T* xInUbAddr = (__ubuf__ T*)xInUb.GetPhyAddr();
144 __ubuf__ float* meanOutUbAddr = (__ubuf__ float*)meanOutUb_.GetPhyAddr();147 __ubuf__ float* meanOutUbAddr = (__ubuf__ float*)meanOutUb_.GetPhyAddr();
145- __ubuf__ float* rstdOutUbAddr = (__ubuf__ float*)rstdOutUb_.GetPhyAddr();148+ __ubuf__ float* rstdOutUbAddr = (__ubuf__ float*)lastoutOutUb_.GetPhyAddr();
146 __ubuf__ float* xSubMeanUbAddr = (__ubuf__ float*)tmpTensor.GetPhyAddr();149 __ubuf__ float* xSubMeanUbAddr = (__ubuf__ float*)tmpTensor.GetPhyAddr();
147 __ubuf__ float* tmpUbAddr = (__ubuf__ float*)tmpTensor.GetPhyAddr() + elemNum_;150 __ubuf__ float* tmpUbAddr = (__ubuf__ float*)tmpTensor.GetPhyAddr() + elemNum_;
148 151 
@@ -156,7 +159,11 @@ private:
156 CalculateMeanVarRCommon<NUM_TWO>(xInUbAddr, meanOutUbAddr, rstdOutUbAddr, xSubMeanUbAddr, tmpUbAddr,159 CalculateMeanVarRCommon<NUM_TWO>(xInUbAddr, meanOutUbAddr, rstdOutUbAddr, xSubMeanUbAddr, tmpUbAddr,
157 currentANum);160 currentANum);
158 }161 }
159- CalculateRstdVF(rstdOutUbAddr, currentANum);162+ __ubuf__ float* varianceUbAddr = rstdOutUbAddr;
163+ if constexpr (!IsOutRstd) {
164+ rstdOutUbAddr = (__ubuf__ float*)rstdBuf_.Get<float>().GetPhyAddr();
165+ }
166+ CalculateRstdVF(rstdOutUbAddr, varianceUbAddr, currentANum);
160 167 
161 LocalTensor<T> yOutUb = yQueue_.AllocTensor<T>();168 LocalTensor<T> yOutUb = yQueue_.AllocTensor<T>();
162 __ubuf__ U* gammaInUbAddr = (__ubuf__ U*)gammaBetaInUb_.GetPhyAddr();169 __ubuf__ U* gammaInUbAddr = (__ubuf__ U*)gammaBetaInUb_.GetPhyAddr();
@@ -505,10 +512,10 @@ private:
505 }512 }
506 }513 }
507 514 
508- __aicore__ inline void CalculateRstdVF(__ubuf__ float* rstdOutUb, uint16_t currentANum)515+ __aicore__ inline void CalculateRstdVF(__ubuf__ float* rstdOutUb, __ubuf__ float* varianceUb, uint16_t currentANum)
509 {516 {
510 float epsilonLocal = tl_->epsilon;517 float epsilonLocal = tl_->epsilon;
511- NormCommon::ComputeRstdNewtonRaphson<false>(rstdOutUb, rstdOutUb, static_cast<uint32_t>(currentANum),518+ NormCommon::ComputeRstdNewtonRaphson<false>(varianceUb, rstdOutUb, static_cast<uint32_t>(currentANum),
512 epsilonLocal, 1.0f, VL_B32);519 epsilonLocal, 1.0f, VL_B32);
513 }520 }
514 521 
@@ -603,12 +610,12 @@ private:
603 }610 }
604 }611 }
605 612 
606- __aicore__ inline void CastMeanRstd(int64_t currentANum)613+ __aicore__ inline void CastMeanLastout(int64_t currentANum)
607 {614 {
608 __ubuf__ float* meanInAddr = (__ubuf__ float*)meanOutUb_.GetPhyAddr();615 __ubuf__ float* meanInAddr = (__ubuf__ float*)meanOutUb_.GetPhyAddr();
609- __ubuf__ float* rstdInAddr = (__ubuf__ float*)rstdOutUb_.GetPhyAddr();616+ __ubuf__ float* rstdInAddr = (__ubuf__ float*)lastoutOutUb_.GetPhyAddr();
610 __ubuf__ M* meanOutAddr = (__ubuf__ M*)meanOutUb_.GetPhyAddr();617 __ubuf__ M* meanOutAddr = (__ubuf__ M*)meanOutUb_.GetPhyAddr();
611- __ubuf__ M* rstdOutAddr = (__ubuf__ M*)rstdOutUb_.GetPhyAddr();618+ __ubuf__ M* rstdOutAddr = (__ubuf__ M*)lastoutOutUb_.GetPhyAddr();
612 619 
613 uint32_t castCount = static_cast<uint32_t>(currentANum);620 uint32_t castCount = static_cast<uint32_t>(currentANum);
614 uint16_t castLoops = static_cast<uint32_t>((castCount + VL_B32 - 1) / VL_B32);621 uint16_t castLoops = static_cast<uint32_t>((castCount + VL_B32 - 1) / VL_B32);
@@ -631,15 +638,15 @@ private:
631 }638 }
632 }639 }
633 640 
634- __aicore__ inline void CopyOutMeanRstd(int64_t offset, int64_t currentANum)641+ __aicore__ inline void CopyOutMeanLastout(int64_t offset, int64_t currentANum)
635 {642 {
636 if constexpr (!IsSameType<M, float>::value) {643 if constexpr (!IsSameType<M, float>::value) {
637 // float to bfloat16 or float16, input continue and output each repeat have only half value644 // float to bfloat16 or float16, input continue and output each repeat have only half value
638- CastMeanRstd(currentANum);645+ CastMeanLastout(currentANum);
639 meanQueue_.EnQue(meanOutUb_);646 meanQueue_.EnQue(meanOutUb_);
640- rstdQueue_.EnQue(rstdOutUb_);647+ lastoutQueue_.EnQue(lastoutOutUb_);
641 LocalTensor<M> meanInUb = meanQueue_.template DeQue<M>();648 LocalTensor<M> meanInUb = meanQueue_.template DeQue<M>();
642- LocalTensor<M> rstdInUb = rstdQueue_.template DeQue<M>();649+ LocalTensor<M> rstdInUb = lastoutQueue_.template DeQue<M>();
643 650 
644 uint32_t castDmaCount = static_cast<uint32_t>(currentANum);651 uint32_t castDmaCount = static_cast<uint32_t>(currentANum);
645 uint32_t castDmaLoops = static_cast<uint32_t>(castDmaCount / VL_B32);652 uint32_t castDmaLoops = static_cast<uint32_t>(castDmaCount / VL_B32);
@@ -650,7 +657,7 @@ private:
650 copyInParams.srcStride = (VECTOR_REG_WIDTH - VL_B32 * sizeof(M)) / BLOCK_SIZE;657 copyInParams.srcStride = (VECTOR_REG_WIDTH - VL_B32 * sizeof(M)) / BLOCK_SIZE;
651 copyInParams.dstStride = 0;658 copyInParams.dstStride = 0;
652 DataCopyPad(meanGm_[offset], meanInUb, copyInParams);659 DataCopyPad(meanGm_[offset], meanInUb, copyInParams);
653- DataCopyPad(rstdGm_[offset], rstdInUb, copyInParams);660+ DataCopyPad(lastoutGm_[offset], rstdInUb, copyInParams);
654 }661 }
655 // tail662 // tail
656 uint32_t tailSize = static_cast<uint32_t>(castDmaCount % VL_B32);663 uint32_t tailSize = static_cast<uint32_t>(castDmaCount % VL_B32);
@@ -661,24 +668,25 @@ private:
661 copyInParamsTail.srcStride = 0;668 copyInParamsTail.srcStride = 0;
662 copyInParamsTail.dstStride = 0;669 copyInParamsTail.dstStride = 0;
663 DataCopyPad(meanGm_[offset + castDmaLoops * VL_B32], meanInUb[castDmaLoops * VL_B16], copyInParamsTail);670 DataCopyPad(meanGm_[offset + castDmaLoops * VL_B32], meanInUb[castDmaLoops * VL_B16], copyInParamsTail);
664- DataCopyPad(rstdGm_[offset + castDmaLoops * VL_B32], rstdInUb[castDmaLoops * VL_B16], copyInParamsTail);671+ DataCopyPad(lastoutGm_[offset + castDmaLoops * VL_B32], rstdInUb[castDmaLoops * VL_B16],
672+ copyInParamsTail);
665 }673 }
666 meanQueue_.FreeTensor(meanInUb);674 meanQueue_.FreeTensor(meanInUb);
667- rstdQueue_.FreeTensor(rstdInUb);675+ lastoutQueue_.FreeTensor(rstdInUb);
668 } else {676 } else {
669 meanQueue_.EnQue(meanOutUb_);677 meanQueue_.EnQue(meanOutUb_);
670- rstdQueue_.EnQue(rstdOutUb_);678+ lastoutQueue_.EnQue(lastoutOutUb_);
671 LocalTensor<float> meanInUb = meanQueue_.template DeQue<float>();679 LocalTensor<float> meanInUb = meanQueue_.template DeQue<float>();
672- LocalTensor<float> rstdInUb = rstdQueue_.template DeQue<float>();680+ LocalTensor<float> rstdInUb = lastoutQueue_.template DeQue<float>();
673 DataCopyExtParams copyInParams;681 DataCopyExtParams copyInParams;
674 copyInParams.blockCount = 1;682 copyInParams.blockCount = 1;
675 copyInParams.blockLen = currentANum * sizeof(float);683 copyInParams.blockLen = currentANum * sizeof(float);
676 copyInParams.srcStride = 0;684 copyInParams.srcStride = 0;
677 copyInParams.dstStride = 0;685 copyInParams.dstStride = 0;
678 DataCopyPad(meanGm_[offset], meanInUb, copyInParams);686 DataCopyPad(meanGm_[offset], meanInUb, copyInParams);
679- DataCopyPad(rstdGm_[offset], rstdInUb, copyInParams);687+ DataCopyPad(lastoutGm_[offset], rstdInUb, copyInParams);
680 meanQueue_.FreeTensor(meanInUb);688 meanQueue_.FreeTensor(meanInUb);
681- rstdQueue_.FreeTensor(rstdInUb);689+ lastoutQueue_.FreeTensor(rstdInUb);
682 }690 }
683 }691 }
684 692 
@@ -701,7 +709,7 @@ private:
701 709 
702 GlobalTensor<T> yGm_;710 GlobalTensor<T> yGm_;
703 GlobalTensor<M> meanGm_;711 GlobalTensor<M> meanGm_;
704- GlobalTensor<M> rstdGm_;712+ GlobalTensor<M> lastoutGm_;
705 713 
706 /* variable */714 /* variable */
707 bool hasGamma_ = true;715 bool hasGamma_ = true;
@@ -717,13 +725,14 @@ private:
717 725 
718 TQue<QuePosition::VECOUT, 1> yQueue_;726 TQue<QuePosition::VECOUT, 1> yQueue_;
719 TQue<QuePosition::VECOUT, 1> meanQueue_;727 TQue<QuePosition::VECOUT, 1> meanQueue_;
720- TQue<QuePosition::VECOUT, 1> rstdQueue_;728+ TQue<QuePosition::VECOUT, 1> lastoutQueue_;
721 729 
722 TBuf<TPosition::VECCALC> tmpBuf;730 TBuf<TPosition::VECCALC> tmpBuf;
731+ TBuf<TPosition::VECCALC> rstdBuf_;
723 732 
724 LocalTensor<U> gammaBetaInUb_;733 LocalTensor<U> gammaBetaInUb_;
725 LocalTensor<float> meanOutUb_;734 LocalTensor<float> meanOutUb_;
726- LocalTensor<float> rstdOutUb_;735+ LocalTensor<float> lastoutOutUb_;
727};736};
728} // namespace LayerNormV3737} // namespace LayerNormV3
729 738 
@@ -105,15 +105,15 @@ __aicore__ inline void RegbaseWelfordImpl(GM_ADDR x, GM_ADDR gamma, GM_ADDR beta
105 op.Process();105 op.Process();
106}106}
107 107 
108-template <typename Tfm, typename Tweight, typename Tmean>108+template <typename Tfm, typename Tweight, typename Tmean, bool IsOutRstd>
109__aicore__ inline void RegbaseTwoPassPerfImpl(GM_ADDR x, GM_ADDR gamma, GM_ADDR beta, GM_ADDR y, GM_ADDR mean,109__aicore__ inline void RegbaseTwoPassPerfImpl(GM_ADDR x, GM_ADDR gamma, GM_ADDR beta, GM_ADDR y, GM_ADDR mean,
110- GM_ADDR rstd, GM_ADDR tiling)110+ GM_ADDR lastout, GM_ADDR tiling)
111{111{
112 TPipe pipeIn;112 TPipe pipeIn;
113 GET_TILING_DATA_WITH_STRUCT(LayerNormV3TilingDataRegBaseTwoPassPerf, tiling_data_in, tiling);113 GET_TILING_DATA_WITH_STRUCT(LayerNormV3TilingDataRegBaseTwoPassPerf, tiling_data_in, tiling);
114 const LayerNormV3TilingDataRegBaseTwoPassPerf* __restrict tilingData = &tiling_data_in;114 const LayerNormV3TilingDataRegBaseTwoPassPerf* __restrict tilingData = &tiling_data_in;
115- LayerNormV3RegbaseTwoPassPerf<Tfm, Tweight, Tmean> op(tilingData, &pipeIn);115+ LayerNormV3RegbaseTwoPassPerf<Tfm, Tweight, Tmean, IsOutRstd> op(tilingData, &pipeIn);
116- op.Init(x, gamma, beta, y, mean, rstd);116+ op.Init(x, gamma, beta, y, mean, lastout);
117 op.Process();117 op.Process();
118}118}
119template <typename Tfm, typename Tweight, typename Tmean, bool IsOutRstd>119template <typename Tfm, typename Tweight, typename Tmean, bool IsOutRstd>
@@ -190,19 +190,19 @@ static __aicore__ inline void layer_norm_impl(GM_ADDR x, GM_ADDR gamma, GM_ADDR
190 RegbaseWelfordImpl<bfloat16_t, bfloat16_t, DTYPE_MEAN, IsOutRstd>(x, gamma, beta, y, mean, lastout, tiling);190 RegbaseWelfordImpl<bfloat16_t, bfloat16_t, DTYPE_MEAN, IsOutRstd>(x, gamma, beta, y, mean, lastout, tiling);
191 return;191 return;
192 } else if (TILING_KEY_IS(LNV3_REGBASE_TWO_PASS_PERF_FLOAT_FLOAT)) {192 } else if (TILING_KEY_IS(LNV3_REGBASE_TWO_PASS_PERF_FLOAT_FLOAT)) {
193- RegbaseTwoPassPerfImpl<float, float, DTYPE_MEAN>(x, gamma, beta, y, mean, lastout, tiling);193+ RegbaseTwoPassPerfImpl<float, float, DTYPE_MEAN, IsOutRstd>(x, gamma, beta, y, mean, lastout, tiling);
194 return;194 return;
195 } else if (TILING_KEY_IS(LNV3_REGBASE_TWO_PASS_PERF_HALF_FLOAT)) {195 } else if (TILING_KEY_IS(LNV3_REGBASE_TWO_PASS_PERF_HALF_FLOAT)) {
196- RegbaseTwoPassPerfImpl<half, float, DTYPE_MEAN>(x, gamma, beta, y, mean, lastout, tiling);196+ RegbaseTwoPassPerfImpl<half, float, DTYPE_MEAN, IsOutRstd>(x, gamma, beta, y, mean, lastout, tiling);
197 return;197 return;
198 } else if (TILING_KEY_IS(LNV3_REGBASE_TWO_PASS_PERF_HALF_HALF)) {198 } else if (TILING_KEY_IS(LNV3_REGBASE_TWO_PASS_PERF_HALF_HALF)) {
199- RegbaseTwoPassPerfImpl<half, half, DTYPE_MEAN>(x, gamma, beta, y, mean, lastout, tiling);199+ RegbaseTwoPassPerfImpl<half, half, DTYPE_MEAN, IsOutRstd>(x, gamma, beta, y, mean, lastout, tiling);
200 return;200 return;
201 } else if (TILING_KEY_IS(LNV3_REGBASE_TWO_PASS_PERF_BF16_FLOAT)) {201 } else if (TILING_KEY_IS(LNV3_REGBASE_TWO_PASS_PERF_BF16_FLOAT)) {
202- RegbaseTwoPassPerfImpl<bfloat16_t, float, DTYPE_MEAN>(x, gamma, beta, y, mean, lastout, tiling);202+ RegbaseTwoPassPerfImpl<bfloat16_t, float, DTYPE_MEAN, IsOutRstd>(x, gamma, beta, y, mean, lastout, tiling);
203 return;203 return;
204 } else if (TILING_KEY_IS(LNV3_REGBASE_TWO_PASS_PERF_BF16_BF16)) {204 } else if (TILING_KEY_IS(LNV3_REGBASE_TWO_PASS_PERF_BF16_BF16)) {
205- RegbaseTwoPassPerfImpl<bfloat16_t, bfloat16_t, DTYPE_MEAN>(x, gamma, beta, y, mean, lastout, tiling);205+ RegbaseTwoPassPerfImpl<bfloat16_t, bfloat16_t, DTYPE_MEAN, IsOutRstd>(x, gamma, beta, y, mean, lastout, tiling);
206 return;206 return;
207 } else if (TILING_KEY_IS(LNV3_REGBASE_NO_REDUCE_FLOAT_FLOAT)) {207 } else if (TILING_KEY_IS(LNV3_REGBASE_NO_REDUCE_FLOAT_FLOAT)) {
208 RegbaseNoReduceImpl<float, float, DTYPE_MEAN, IsOutRstd>(x, gamma, beta, y, mean, lastout, tiling);208 RegbaseNoReduceImpl<float, float, DTYPE_MEAN, IsOutRstd>(x, gamma, beta, y, mean, lastout, tiling);