Merged
feat(layer_norm): reuse LayerNormV3 two-pass perf template #13207
rkAdd on 22 days ago
feat(layer_norm): reuse LayerNormV3 two-pass perf template #13207
Merged
Merged fromwangpengbo26/ops-nnln_prefintocann/ops-nnmaster
Total: 4 files changed+53-40
| @@ -36,7 +36,7 @@ protected: | |||
| 36 | static void TearDownTestCase() { std::cout << "LayerNormTiling TearDown" << std::endl; } | 36 | static void TearDownTestCase() { std::cout << "LayerNormTiling TearDown" << std::endl; } |
| 37 | }; | 37 | }; |
| 38 | 38 | ||
| 39 | -TEST_F(LayerNormTilingTest, layer_norm_tiling_regbase_two_pass) | 39 | +TEST_F(LayerNormTilingTest, layer_norm_tiling_regbase_two_pass_perf) |
| 40 | { | 40 | { |
| 41 | gert::StorageShape x_shape = {{64, 4096}, {64, 4096}}; | 41 | gert::StorageShape x_shape = {{64, 4096}, {64, 4096}}; |
| 42 | gert::StorageShape gamma_shape = {{4096}, {4096}}; | 42 | gert::StorageShape gamma_shape = {{4096}, {4096}}; |
| @@ -129,7 +129,7 @@ TEST_F(LayerNormTilingTest, layer_norm_tiling_regbase_two_pass) | |||
| 129 | EXPECT_EQ(tiling_func(tiling_context), ge::GRAPH_SUCCESS); | 129 | EXPECT_EQ(tiling_func(tiling_context), ge::GRAPH_SUCCESS); |
| 130 | // todo check tiling result | 130 | // todo check tiling result |
| 131 | auto tiling_key = tiling_context->GetTilingKey(); | 131 | auto tiling_key = tiling_context->GetTilingKey(); |
| 132 | - // ASSERT_EQ(tiling_key, 300); | 132 | + ASSERT_EQ(tiling_key, 500); |
| 133 | auto tilingData = tiling_context->GetRawTilingData(); | 133 | auto tilingData = tiling_context->GetRawTilingData(); |
| 134 | ASSERT_NE(tilingData, nullptr); | 134 | ASSERT_NE(tilingData, nullptr); |
| 135 | // EXPECT_EQ(to_string<int32_t>(tilingData->GetData(), tilingData->GetDataSize()), | 135 | // EXPECT_EQ(to_string<int32_t>(tilingData->GetData(), tilingData->GetDataSize()), |
| @@ -51,8 +51,11 @@ bool LayerNormV3RegBaseTwoPassPerfTiling::CanFitInBuffer(int64_t curA) | |||
| 51 | int64_t ubCanUseSize = GetUBCanUseSize(); | 51 | int64_t ubCanUseSize = GetUBCanUseSize(); |
| 52 | int64_t rowWeight = GetRowWeight(); | 52 | int64_t rowWeight = GetRowWeight(); |
| 53 | 53 | ||
| 54 | - return curA * static_cast<int64_t>(commonParams.rowAlign) * rowWeight <= | 54 | + int64_t outputBufferSize = LNV3_NUM_TWO * LNV3_DOUBLE_BUFFER * curAAlign * sizeof(float); |
| 55 | - ubCanUseSize - LNV3_NUM_TWO * LNV3_DOUBLE_BUFFER * curAAlign; | 55 | + if (commonParams.isV1) { |
| 56 | + outputBufferSize += curAAlign * sizeof(float); | ||
| 57 | + } | ||
| 58 | + return curA * static_cast<int64_t>(commonParams.rowAlign) * rowWeight <= ubCanUseSize - outputBufferSize; | ||
| 56 | } | 59 | } |
| 57 | 60 | ||
| 58 | bool LayerNormV3RegBaseTwoPassPerfTiling::IsCapable() | 61 | bool LayerNormV3RegBaseTwoPassPerfTiling::IsCapable() |
| @@ -162,4 +165,5 @@ ge::graphStatus LayerNormV3RegBaseTwoPassPerfTiling::PostTiling() | |||
| 162 | } | 165 | } |
| 163 | 166 | ||
| 164 | REGISTER_OPS_TILING_TEMPLATE(LayerNormV3, LayerNormV3RegBaseTwoPassPerfTiling, 150); | 167 | REGISTER_OPS_TILING_TEMPLATE(LayerNormV3, LayerNormV3RegBaseTwoPassPerfTiling, 150); |
| 168 | +REGISTER_OPS_TILING_TEMPLATE(LayerNorm, LayerNormV3RegBaseTwoPassPerfTiling, 150); | ||
| 165 | } // namespace optiling | 169 | } // namespace optiling |
| @@ -36,7 +36,7 @@ using AscendC::Reg::UpdateMask; | |||
| 36 | using NormCommon::NormCommonRegbase::LoadRegForDtype; | 36 | using NormCommon::NormCommonRegbase::LoadRegForDtype; |
| 37 | using NormCommon::NormCommonRegbase::StoreRegForDtype; | 37 | using NormCommon::NormCommonRegbase::StoreRegForDtype; |
| 38 | 38 | ||
| 39 | -template <typename T, typename U, typename M> | 39 | +template <typename T, typename U, typename M, bool IsOutRstd> |
| 40 | class LayerNormV3RegbaseTwoPassPerf { | 40 | class LayerNormV3RegbaseTwoPassPerf { |
| 41 | public: | 41 | public: |
| 42 | __aicore__ inline LayerNormV3RegbaseTwoPassPerf(const LayerNormV3TilingDataRegBaseTwoPassPerf* tilingData, | 42 | __aicore__ inline LayerNormV3RegbaseTwoPassPerf(const LayerNormV3TilingDataRegBaseTwoPassPerf* tilingData, |
| @@ -48,7 +48,7 @@ public: | |||
| 48 | hasBeta_ = (tilingData->nullptrBeta == 0); | 48 | hasBeta_ = (tilingData->nullptrBeta == 0); |
| 49 | } | 49 | } |
| 50 | 50 | ||
| 51 | - __aicore__ inline void Init(GM_ADDR x, GM_ADDR gamma, GM_ADDR beta, GM_ADDR y, GM_ADDR mean, GM_ADDR rstd) | 51 | + __aicore__ inline void Init(GM_ADDR x, GM_ADDR gamma, GM_ADDR beta, GM_ADDR y, GM_ADDR mean, GM_ADDR lastout) |
| 52 | { | 52 | { |
| 53 | // copy in gamma and beta | 53 | // copy in gamma and beta |
| 54 | gammaGm_.SetGlobalBuffer((__gm__ U*)gamma); | 54 | gammaGm_.SetGlobalBuffer((__gm__ U*)gamma); |
| @@ -60,10 +60,13 @@ public: | |||
| 60 | xGm_.SetGlobalBuffer((__gm__ T*)x + xBlockOffset); | 60 | xGm_.SetGlobalBuffer((__gm__ T*)x + xBlockOffset); |
| 61 | yGm_.SetGlobalBuffer((__gm__ T*)y + xBlockOffset); | 61 | yGm_.SetGlobalBuffer((__gm__ T*)y + xBlockOffset); |
| 62 | meanGm_.SetGlobalBuffer((__gm__ M*)mean + meanBlockOffset); | 62 | meanGm_.SetGlobalBuffer((__gm__ M*)mean + meanBlockOffset); |
| 63 | - rstdGm_.SetGlobalBuffer((__gm__ M*)rstd + meanBlockOffset); | 63 | + lastoutGm_.SetGlobalBuffer((__gm__ M*)lastout + meanBlockOffset); |
| 64 | 64 | ||
| 65 | pipe_->InitBuffer(meanQueue_, DOUBLE_BUFFER, tl_->aUbFactorAlignB32 * sizeof(float)); | 65 | pipe_->InitBuffer(meanQueue_, DOUBLE_BUFFER, tl_->aUbFactorAlignB32 * sizeof(float)); |
| 66 | - pipe_->InitBuffer(rstdQueue_, DOUBLE_BUFFER, tl_->aUbFactorAlignB32 * sizeof(float)); | 66 | + pipe_->InitBuffer(lastoutQueue_, DOUBLE_BUFFER, tl_->aUbFactorAlignB32 * sizeof(float)); |
| 67 | + if constexpr (!IsOutRstd) { | ||
| 68 | + pipe_->InitBuffer(rstdBuf_, tl_->aUbFactorAlignB32 * sizeof(float)); | ||
| 69 | + } | ||
| 67 | 70 | ||
| 68 | elemNum_ = tl_->aUbFactor * tl_->rAlign; | 71 | elemNum_ = tl_->aUbFactor * tl_->rAlign; |
| 69 | pipe_->InitBuffer(xQueue_, DOUBLE_BUFFER, elemNum_ * sizeof(T)); | 72 | pipe_->InitBuffer(xQueue_, DOUBLE_BUFFER, elemNum_ * sizeof(T)); |
| @@ -87,7 +90,7 @@ public: | |||
| 87 | CopyInX(offset, currentA); | 90 | CopyInX(offset, currentA); |
| 88 | Caculate(currentA); | 91 | Caculate(currentA); |
| 89 | CopyOutY(offset, currentA); | 92 | CopyOutY(offset, currentA); |
| 90 | - CopyOutMeanRstd(aOffset, currentA); | 93 | + CopyOutMeanLastout(aOffset, currentA); |
| 91 | } | 94 | } |
| 92 | 95 | ||
| 93 | gammaBetaQueue_.FreeTensor(gammaBetaInUb_); | 96 | gammaBetaQueue_.FreeTensor(gammaBetaInUb_); |
| @@ -137,12 +140,12 @@ private: | |||
| 137 | { | 140 | { |
| 138 | LocalTensor<T> xInUb = xQueue_.template DeQue<T>(); | 141 | LocalTensor<T> xInUb = xQueue_.template DeQue<T>(); |
| 139 | meanOutUb_ = meanQueue_.AllocTensor<float>(); | 142 | meanOutUb_ = meanQueue_.AllocTensor<float>(); |
| 140 | - rstdOutUb_ = rstdQueue_.AllocTensor<float>(); | 143 | + lastoutOutUb_ = lastoutQueue_.AllocTensor<float>(); |
| 141 | LocalTensor<float> tmpTensor = tmpBuf.Get<float>(); | 144 | LocalTensor<float> tmpTensor = tmpBuf.Get<float>(); |
| 142 | 145 | ||
| 143 | __ubuf__ T* xInUbAddr = (__ubuf__ T*)xInUb.GetPhyAddr(); | 146 | __ubuf__ T* xInUbAddr = (__ubuf__ T*)xInUb.GetPhyAddr(); |
| 144 | __ubuf__ float* meanOutUbAddr = (__ubuf__ float*)meanOutUb_.GetPhyAddr(); | 147 | __ubuf__ float* meanOutUbAddr = (__ubuf__ float*)meanOutUb_.GetPhyAddr(); |
| 145 | - __ubuf__ float* rstdOutUbAddr = (__ubuf__ float*)rstdOutUb_.GetPhyAddr(); | 148 | + __ubuf__ float* rstdOutUbAddr = (__ubuf__ float*)lastoutOutUb_.GetPhyAddr(); |
| 146 | __ubuf__ float* xSubMeanUbAddr = (__ubuf__ float*)tmpTensor.GetPhyAddr(); | 149 | __ubuf__ float* xSubMeanUbAddr = (__ubuf__ float*)tmpTensor.GetPhyAddr(); |
| 147 | __ubuf__ float* tmpUbAddr = (__ubuf__ float*)tmpTensor.GetPhyAddr() + elemNum_; | 150 | __ubuf__ float* tmpUbAddr = (__ubuf__ float*)tmpTensor.GetPhyAddr() + elemNum_; |
| 148 | 151 | ||
| @@ -156,7 +159,11 @@ private: | |||
| 156 | CalculateMeanVarRCommon<NUM_TWO>(xInUbAddr, meanOutUbAddr, rstdOutUbAddr, xSubMeanUbAddr, tmpUbAddr, | 159 | CalculateMeanVarRCommon<NUM_TWO>(xInUbAddr, meanOutUbAddr, rstdOutUbAddr, xSubMeanUbAddr, tmpUbAddr, |
| 157 | currentANum); | 160 | currentANum); |
| 158 | } | 161 | } |
| 159 | - CalculateRstdVF(rstdOutUbAddr, currentANum); | 162 | + __ubuf__ float* varianceUbAddr = rstdOutUbAddr; |
| 163 | + if constexpr (!IsOutRstd) { | ||
| 164 | + rstdOutUbAddr = (__ubuf__ float*)rstdBuf_.Get<float>().GetPhyAddr(); | ||
| 165 | + } | ||
| 166 | + CalculateRstdVF(rstdOutUbAddr, varianceUbAddr, currentANum); | ||
| 160 | 167 | ||
| 161 | LocalTensor<T> yOutUb = yQueue_.AllocTensor<T>(); | 168 | LocalTensor<T> yOutUb = yQueue_.AllocTensor<T>(); |
| 162 | __ubuf__ U* gammaInUbAddr = (__ubuf__ U*)gammaBetaInUb_.GetPhyAddr(); | 169 | __ubuf__ U* gammaInUbAddr = (__ubuf__ U*)gammaBetaInUb_.GetPhyAddr(); |
| @@ -505,10 +512,10 @@ private: | |||
| 505 | } | 512 | } |
| 506 | } | 513 | } |
| 507 | 514 | ||
| 508 | - __aicore__ inline void CalculateRstdVF(__ubuf__ float* rstdOutUb, uint16_t currentANum) | 515 | + __aicore__ inline void CalculateRstdVF(__ubuf__ float* rstdOutUb, __ubuf__ float* varianceUb, uint16_t currentANum) |
| 509 | { | 516 | { |
| 510 | float epsilonLocal = tl_->epsilon; | 517 | float epsilonLocal = tl_->epsilon; |
| 511 | - NormCommon::ComputeRstdNewtonRaphson<false>(rstdOutUb, rstdOutUb, static_cast<uint32_t>(currentANum), | 518 | + NormCommon::ComputeRstdNewtonRaphson<false>(varianceUb, rstdOutUb, static_cast<uint32_t>(currentANum), |
| 512 | epsilonLocal, 1.0f, VL_B32); | 519 | epsilonLocal, 1.0f, VL_B32); |
| 513 | } | 520 | } |
| 514 | 521 | ||
| @@ -603,12 +610,12 @@ private: | |||
| 603 | } | 610 | } |
| 604 | } | 611 | } |
| 605 | 612 | ||
| 606 | - __aicore__ inline void CastMeanRstd(int64_t currentANum) | 613 | + __aicore__ inline void CastMeanLastout(int64_t currentANum) |
| 607 | { | 614 | { |
| 608 | __ubuf__ float* meanInAddr = (__ubuf__ float*)meanOutUb_.GetPhyAddr(); | 615 | __ubuf__ float* meanInAddr = (__ubuf__ float*)meanOutUb_.GetPhyAddr(); |
| 609 | - __ubuf__ float* rstdInAddr = (__ubuf__ float*)rstdOutUb_.GetPhyAddr(); | 616 | + __ubuf__ float* rstdInAddr = (__ubuf__ float*)lastoutOutUb_.GetPhyAddr(); |
| 610 | __ubuf__ M* meanOutAddr = (__ubuf__ M*)meanOutUb_.GetPhyAddr(); | 617 | __ubuf__ M* meanOutAddr = (__ubuf__ M*)meanOutUb_.GetPhyAddr(); |
| 611 | - __ubuf__ M* rstdOutAddr = (__ubuf__ M*)rstdOutUb_.GetPhyAddr(); | 618 | + __ubuf__ M* rstdOutAddr = (__ubuf__ M*)lastoutOutUb_.GetPhyAddr(); |
| 612 | 619 | ||
| 613 | uint32_t castCount = static_cast<uint32_t>(currentANum); | 620 | uint32_t castCount = static_cast<uint32_t>(currentANum); |
| 614 | uint16_t castLoops = static_cast<uint32_t>((castCount + VL_B32 - 1) / VL_B32); | 621 | uint16_t castLoops = static_cast<uint32_t>((castCount + VL_B32 - 1) / VL_B32); |
| @@ -631,15 +638,15 @@ private: | |||
| 631 | } | 638 | } |
| 632 | } | 639 | } |
| 633 | 640 | ||
| 634 | - __aicore__ inline void CopyOutMeanRstd(int64_t offset, int64_t currentANum) | 641 | + __aicore__ inline void CopyOutMeanLastout(int64_t offset, int64_t currentANum) |
| 635 | { | 642 | { |
| 636 | if constexpr (!IsSameType<M, float>::value) { | 643 | if constexpr (!IsSameType<M, float>::value) { |
| 637 | // float to bfloat16 or float16, input continue and output each repeat have only half value | 644 | // float to bfloat16 or float16, input continue and output each repeat have only half value |
| 638 | - CastMeanRstd(currentANum); | 645 | + CastMeanLastout(currentANum); |
| 639 | meanQueue_.EnQue(meanOutUb_); | 646 | meanQueue_.EnQue(meanOutUb_); |
| 640 | - rstdQueue_.EnQue(rstdOutUb_); | 647 | + lastoutQueue_.EnQue(lastoutOutUb_); |
| 641 | LocalTensor<M> meanInUb = meanQueue_.template DeQue<M>(); | 648 | LocalTensor<M> meanInUb = meanQueue_.template DeQue<M>(); |
| 642 | - LocalTensor<M> rstdInUb = rstdQueue_.template DeQue<M>(); | 649 | + LocalTensor<M> rstdInUb = lastoutQueue_.template DeQue<M>(); |
| 643 | 650 | ||
| 644 | uint32_t castDmaCount = static_cast<uint32_t>(currentANum); | 651 | uint32_t castDmaCount = static_cast<uint32_t>(currentANum); |
| 645 | uint32_t castDmaLoops = static_cast<uint32_t>(castDmaCount / VL_B32); | 652 | uint32_t castDmaLoops = static_cast<uint32_t>(castDmaCount / VL_B32); |
| @@ -650,7 +657,7 @@ private: | |||
| 650 | copyInParams.srcStride = (VECTOR_REG_WIDTH - VL_B32 * sizeof(M)) / BLOCK_SIZE; | 657 | copyInParams.srcStride = (VECTOR_REG_WIDTH - VL_B32 * sizeof(M)) / BLOCK_SIZE; |
| 651 | copyInParams.dstStride = 0; | 658 | copyInParams.dstStride = 0; |
| 652 | DataCopyPad(meanGm_[offset], meanInUb, copyInParams); | 659 | DataCopyPad(meanGm_[offset], meanInUb, copyInParams); |
| 653 | - DataCopyPad(rstdGm_[offset], rstdInUb, copyInParams); | 660 | + DataCopyPad(lastoutGm_[offset], rstdInUb, copyInParams); |
| 654 | } | 661 | } |
| 655 | // tail | 662 | // tail |
| 656 | uint32_t tailSize = static_cast<uint32_t>(castDmaCount % VL_B32); | 663 | uint32_t tailSize = static_cast<uint32_t>(castDmaCount % VL_B32); |
| @@ -661,24 +668,25 @@ private: | |||
| 661 | copyInParamsTail.srcStride = 0; | 668 | copyInParamsTail.srcStride = 0; |
| 662 | copyInParamsTail.dstStride = 0; | 669 | copyInParamsTail.dstStride = 0; |
| 663 | DataCopyPad(meanGm_[offset + castDmaLoops * VL_B32], meanInUb[castDmaLoops * VL_B16], copyInParamsTail); | 670 | DataCopyPad(meanGm_[offset + castDmaLoops * VL_B32], meanInUb[castDmaLoops * VL_B16], copyInParamsTail); |
| 664 | - DataCopyPad(rstdGm_[offset + castDmaLoops * VL_B32], rstdInUb[castDmaLoops * VL_B16], copyInParamsTail); | 671 | + DataCopyPad(lastoutGm_[offset + castDmaLoops * VL_B32], rstdInUb[castDmaLoops * VL_B16], |
| 672 | + copyInParamsTail); | ||
| 665 | } | 673 | } |
| 666 | meanQueue_.FreeTensor(meanInUb); | 674 | meanQueue_.FreeTensor(meanInUb); |
| 667 | - rstdQueue_.FreeTensor(rstdInUb); | 675 | + lastoutQueue_.FreeTensor(rstdInUb); |
| 668 | } else { | 676 | } else { |
| 669 | meanQueue_.EnQue(meanOutUb_); | 677 | meanQueue_.EnQue(meanOutUb_); |
| 670 | - rstdQueue_.EnQue(rstdOutUb_); | 678 | + lastoutQueue_.EnQue(lastoutOutUb_); |
| 671 | LocalTensor<float> meanInUb = meanQueue_.template DeQue<float>(); | 679 | LocalTensor<float> meanInUb = meanQueue_.template DeQue<float>(); |
| 672 | - LocalTensor<float> rstdInUb = rstdQueue_.template DeQue<float>(); | 680 | + LocalTensor<float> rstdInUb = lastoutQueue_.template DeQue<float>(); |
| 673 | DataCopyExtParams copyInParams; | 681 | DataCopyExtParams copyInParams; |
| 674 | copyInParams.blockCount = 1; | 682 | copyInParams.blockCount = 1; |
| 675 | copyInParams.blockLen = currentANum * sizeof(float); | 683 | copyInParams.blockLen = currentANum * sizeof(float); |
| 676 | copyInParams.srcStride = 0; | 684 | copyInParams.srcStride = 0; |
| 677 | copyInParams.dstStride = 0; | 685 | copyInParams.dstStride = 0; |
| 678 | DataCopyPad(meanGm_[offset], meanInUb, copyInParams); | 686 | DataCopyPad(meanGm_[offset], meanInUb, copyInParams); |
| 679 | - DataCopyPad(rstdGm_[offset], rstdInUb, copyInParams); | 687 | + DataCopyPad(lastoutGm_[offset], rstdInUb, copyInParams); |
| 680 | meanQueue_.FreeTensor(meanInUb); | 688 | meanQueue_.FreeTensor(meanInUb); |
| 681 | - rstdQueue_.FreeTensor(rstdInUb); | 689 | + lastoutQueue_.FreeTensor(rstdInUb); |
| 682 | } | 690 | } |
| 683 | } | 691 | } |
| 684 | 692 | ||
| @@ -701,7 +709,7 @@ private: | |||
| 701 | 709 | ||
| 702 | GlobalTensor<T> yGm_; | 710 | GlobalTensor<T> yGm_; |
| 703 | GlobalTensor<M> meanGm_; | 711 | GlobalTensor<M> meanGm_; |
| 704 | - GlobalTensor<M> rstdGm_; | 712 | + GlobalTensor<M> lastoutGm_; |
| 705 | 713 | ||
| 706 | /* variable */ | 714 | /* variable */ |
| 707 | bool hasGamma_ = true; | 715 | bool hasGamma_ = true; |
| @@ -717,13 +725,14 @@ private: | |||
| 717 | 725 | ||
| 718 | TQue<QuePosition::VECOUT, 1> yQueue_; | 726 | TQue<QuePosition::VECOUT, 1> yQueue_; |
| 719 | TQue<QuePosition::VECOUT, 1> meanQueue_; | 727 | TQue<QuePosition::VECOUT, 1> meanQueue_; |
| 720 | - TQue<QuePosition::VECOUT, 1> rstdQueue_; | 728 | + TQue<QuePosition::VECOUT, 1> lastoutQueue_; |
| 721 | 729 | ||
| 722 | TBuf<TPosition::VECCALC> tmpBuf; | 730 | TBuf<TPosition::VECCALC> tmpBuf; |
| 731 | + TBuf<TPosition::VECCALC> rstdBuf_; | ||
| 723 | 732 | ||
| 724 | LocalTensor<U> gammaBetaInUb_; | 733 | LocalTensor<U> gammaBetaInUb_; |
| 725 | LocalTensor<float> meanOutUb_; | 734 | LocalTensor<float> meanOutUb_; |
| 726 | - LocalTensor<float> rstdOutUb_; | 735 | + LocalTensor<float> lastoutOutUb_; |
| 727 | }; | 736 | }; |
| 728 | } // namespace LayerNormV3 | 737 | } // namespace LayerNormV3 |
| 729 | 738 | ||
| @@ -105,15 +105,15 @@ __aicore__ inline void RegbaseWelfordImpl(GM_ADDR x, GM_ADDR gamma, GM_ADDR beta | |||
| 105 | op.Process(); | 105 | op.Process(); |
| 106 | } | 106 | } |
| 107 | 107 | ||
| 108 | -template <typename Tfm, typename Tweight, typename Tmean> | 108 | +template <typename Tfm, typename Tweight, typename Tmean, bool IsOutRstd> |
| 109 | __aicore__ inline void RegbaseTwoPassPerfImpl(GM_ADDR x, GM_ADDR gamma, GM_ADDR beta, GM_ADDR y, GM_ADDR mean, | 109 | __aicore__ inline void RegbaseTwoPassPerfImpl(GM_ADDR x, GM_ADDR gamma, GM_ADDR beta, GM_ADDR y, GM_ADDR mean, |
| 110 | - GM_ADDR rstd, GM_ADDR tiling) | 110 | + GM_ADDR lastout, GM_ADDR tiling) |
| 111 | { | 111 | { |
| 112 | TPipe pipeIn; | 112 | TPipe pipeIn; |
| 113 | GET_TILING_DATA_WITH_STRUCT(LayerNormV3TilingDataRegBaseTwoPassPerf, tiling_data_in, tiling); | 113 | GET_TILING_DATA_WITH_STRUCT(LayerNormV3TilingDataRegBaseTwoPassPerf, tiling_data_in, tiling); |
| 114 | const LayerNormV3TilingDataRegBaseTwoPassPerf* __restrict tilingData = &tiling_data_in; | 114 | const LayerNormV3TilingDataRegBaseTwoPassPerf* __restrict tilingData = &tiling_data_in; |
| 115 | - LayerNormV3RegbaseTwoPassPerf<Tfm, Tweight, Tmean> op(tilingData, &pipeIn); | 115 | + LayerNormV3RegbaseTwoPassPerf<Tfm, Tweight, Tmean, IsOutRstd> op(tilingData, &pipeIn); |
| 116 | - op.Init(x, gamma, beta, y, mean, rstd); | 116 | + op.Init(x, gamma, beta, y, mean, lastout); |
| 117 | op.Process(); | 117 | op.Process(); |
| 118 | } | 118 | } |
| 119 | template <typename Tfm, typename Tweight, typename Tmean, bool IsOutRstd> | 119 | template <typename Tfm, typename Tweight, typename Tmean, bool IsOutRstd> |
| @@ -190,19 +190,19 @@ static __aicore__ inline void layer_norm_impl(GM_ADDR x, GM_ADDR gamma, GM_ADDR | |||
| 190 | RegbaseWelfordImpl<bfloat16_t, bfloat16_t, DTYPE_MEAN, IsOutRstd>(x, gamma, beta, y, mean, lastout, tiling); | 190 | RegbaseWelfordImpl<bfloat16_t, bfloat16_t, DTYPE_MEAN, IsOutRstd>(x, gamma, beta, y, mean, lastout, tiling); |
| 191 | return; | 191 | return; |
| 192 | } else if (TILING_KEY_IS(LNV3_REGBASE_TWO_PASS_PERF_FLOAT_FLOAT)) { | 192 | } else if (TILING_KEY_IS(LNV3_REGBASE_TWO_PASS_PERF_FLOAT_FLOAT)) { |
| 193 | - RegbaseTwoPassPerfImpl<float, float, DTYPE_MEAN>(x, gamma, beta, y, mean, lastout, tiling); | 193 | + RegbaseTwoPassPerfImpl<float, float, DTYPE_MEAN, IsOutRstd>(x, gamma, beta, y, mean, lastout, tiling); |
| 194 | return; | 194 | return; |
| 195 | } else if (TILING_KEY_IS(LNV3_REGBASE_TWO_PASS_PERF_HALF_FLOAT)) { | 195 | } else if (TILING_KEY_IS(LNV3_REGBASE_TWO_PASS_PERF_HALF_FLOAT)) { |
| 196 | - RegbaseTwoPassPerfImpl<half, float, DTYPE_MEAN>(x, gamma, beta, y, mean, lastout, tiling); | 196 | + RegbaseTwoPassPerfImpl<half, float, DTYPE_MEAN, IsOutRstd>(x, gamma, beta, y, mean, lastout, tiling); |
| 197 | return; | 197 | return; |
| 198 | } else if (TILING_KEY_IS(LNV3_REGBASE_TWO_PASS_PERF_HALF_HALF)) { | 198 | } else if (TILING_KEY_IS(LNV3_REGBASE_TWO_PASS_PERF_HALF_HALF)) { |
| 199 | - RegbaseTwoPassPerfImpl<half, half, DTYPE_MEAN>(x, gamma, beta, y, mean, lastout, tiling); | 199 | + RegbaseTwoPassPerfImpl<half, half, DTYPE_MEAN, IsOutRstd>(x, gamma, beta, y, mean, lastout, tiling); |
| 200 | return; | 200 | return; |
| 201 | } else if (TILING_KEY_IS(LNV3_REGBASE_TWO_PASS_PERF_BF16_FLOAT)) { | 201 | } else if (TILING_KEY_IS(LNV3_REGBASE_TWO_PASS_PERF_BF16_FLOAT)) { |
| 202 | - RegbaseTwoPassPerfImpl<bfloat16_t, float, DTYPE_MEAN>(x, gamma, beta, y, mean, lastout, tiling); | 202 | + RegbaseTwoPassPerfImpl<bfloat16_t, float, DTYPE_MEAN, IsOutRstd>(x, gamma, beta, y, mean, lastout, tiling); |
| 203 | return; | 203 | return; |
| 204 | } else if (TILING_KEY_IS(LNV3_REGBASE_TWO_PASS_PERF_BF16_BF16)) { | 204 | } else if (TILING_KEY_IS(LNV3_REGBASE_TWO_PASS_PERF_BF16_BF16)) { |
| 205 | - RegbaseTwoPassPerfImpl<bfloat16_t, bfloat16_t, DTYPE_MEAN>(x, gamma, beta, y, mean, lastout, tiling); | 205 | + RegbaseTwoPassPerfImpl<bfloat16_t, bfloat16_t, DTYPE_MEAN, IsOutRstd>(x, gamma, beta, y, mean, lastout, tiling); |
| 206 | return; | 206 | return; |
| 207 | } else if (TILING_KEY_IS(LNV3_REGBASE_NO_REDUCE_FLOAT_FLOAT)) { | 207 | } else if (TILING_KEY_IS(LNV3_REGBASE_NO_REDUCE_FLOAT_FLOAT)) { |
| 208 | RegbaseNoReduceImpl<float, float, DTYPE_MEAN, IsOutRstd>(x, gamma, beta, y, mean, lastout, tiling); | 208 | RegbaseNoReduceImpl<float, float, DTYPE_MEAN, IsOutRstd>(x, gamma, beta, y, mean, lastout, tiling); |
[Major] 这个 UB 预算修正同时改了 LayerNormV3 的 aUbFactor,建议补 V3 的回归数据:kernel 里
meanQueue_/lastoutQueue_都是DOUBLE_BUFFER * aUbFactorAlignB32 * sizeof(float),实际 16B/元素,而原式LNV3_NUM_TWO * LNV3_DOUBLE_BUFFER * curAAlign只扣了 4B/元素(少算 12B)。乘上sizeof(float)后是对的,但 V3 走的是同一份CanFitInBuffer,它的 aUbFactor 也会跟着变小、UB loop 变多。PR 描述写"未提供性能数据",建议补 V3(大 A / 大 rowAlign 场景)的精度与性能对比,确认这次修正没有把 V3 拖慢。