已合并
QuantConv3d后融transdata #973
wym_666创建于 1月23日
QuantConv3d后融transdata #973
已合并
共 19 个文件变更+365-198
| @@ -758,7 +758,7 @@ | |||
| 758 | ] | 758 | ] |
| 759 | }, | 759 | }, |
| 760 | { | 760 | { |
| 761 | - "bin_filename": "Conv3DV2_NDC1HWC0_INT8_FRACTAL_Z_3D_INT8_FP32_NDC1HWC0_BF16", | 761 | + "bin_filename": "Conv3DV2_NDC1HWC0_INT8_FRACTAL_Z_3D_INT8_FP32_NCDHW_BF16", |
 | |||
| 762 | "inputs": [ | 762 | "inputs": [ |
| 763 | { | 763 | { |
| 764 | "name": "x", | 764 | "name": "x", |
| @@ -832,12 +832,12 @@ | |||
| 832 | "name": "y", | 832 | "name": "y", |
| 833 | "index": 0, | 833 | "index": 0, |
| 834 | "dtype": "bfloat16", | 834 | "dtype": "bfloat16", |
| 835 | - "format": "NDC1HWC0", | 835 | + "format": "NCDHW", |
| 836 | "ori_format": "NCDHW", | 836 | "ori_format": "NCDHW", |
| 837 | "paramType": "required", | 837 | "paramType": "required", |
| 838 | - "shape": [-1, -1, -1, -1, -1, -1], | 838 | + "shape": [-1, -1, -1, -1, -1], |
| 839 | "ori_shape": [-1, -1, -1, -1, -1], | 839 | "ori_shape": [-1, -1, -1, -1, -1], |
| 840 | - "range": [[1, -1], [1, -1], [1, -1], [1, -1], [1, -1], [1, -1]] | 840 | + "range": [[1, -1], [1, -1], [1, -1], [1, -1], [1, -1]] |
| 841 | } | 841 | } |
| 842 | ], | 842 | ], |
| 843 | "attrs": [ | 843 | "attrs": [ |
| @@ -884,7 +884,7 @@ | |||
| 884 | ] | 884 | ] |
| 885 | }, | 885 | }, |
| 886 | { | 886 | { |
| 887 | - "bin_filename": "Conv3DV2_NDC1HWC0_INT8_FRACTAL_Z_3D_INT8_FP32_NDC1HWC0_FP16", | 887 | + "bin_filename": "Conv3DV2_NDC1HWC0_INT8_FRACTAL_Z_3D_INT8_FP32_NCDHW_FP16", |
| 888 | "inputs": [ | 888 | "inputs": [ |
| 889 | { | 889 | { |
| 890 | "name": "x", | 890 | "name": "x", |
| @@ -958,12 +958,12 @@ | |||
| 958 | "name": "y", | 958 | "name": "y", |
| 959 | "index": 0, | 959 | "index": 0, |
| 960 | "dtype": "float16", | 960 | "dtype": "float16", |
| 961 | - "format": "NDC1HWC0", | 961 | + "format": "NCDHW", |
| 962 | "ori_format": "NCDHW", | 962 | "ori_format": "NCDHW", |
| 963 | "paramType": "required", | 963 | "paramType": "required", |
| 964 | - "shape": [-1, -1, -1, -1, -1, -1], | 964 | + "shape": [-1, -1, -1, -1, -1], |
| 965 | "ori_shape": [-1, -1, -1, -1, -1], | 965 | "ori_shape": [-1, -1, -1, -1, -1], |
| 966 | - "range": [[1, -1], [1, -1], [1, -1], [1, -1], [1, -1], [1, -1]] | 966 | + "range": [[1, -1], [1, -1], [1, -1], [1, -1], [1, -1]] |
| 967 | } | 967 | } |
| 968 | ], | 968 | ], |
| 969 | "attrs": [ | 969 | "attrs": [ |
| @@ -758,7 +758,7 @@ | |||
| 758 | ] | 758 | ] |
| 759 | }, | 759 | }, |
| 760 | { | 760 | { |
| 761 | - "bin_filename": "Conv3DV2_NDC1HWC0_INT8_FRACTAL_Z_3D_INT8_FP32_NDC1HWC0_BF16", | 761 | + "bin_filename": "Conv3DV2_NDC1HWC0_INT8_FRACTAL_Z_3D_INT8_FP32_NCDHW_BF16", |
| 762 | "inputs": [ | 762 | "inputs": [ |
| 763 | { | 763 | { |
| 764 | "name": "x", | 764 | "name": "x", |
| @@ -832,12 +832,12 @@ | |||
| 832 | "name": "y", | 832 | "name": "y", |
| 833 | "index": 0, | 833 | "index": 0, |
| 834 | "dtype": "bfloat16", | 834 | "dtype": "bfloat16", |
| 835 | - "format": "NDC1HWC0", | 835 | + "format": "NCDHW", |
| 836 | "ori_format": "NCDHW", | 836 | "ori_format": "NCDHW", |
| 837 | "paramType": "required", | 837 | "paramType": "required", |
| 838 | - "shape": [-1, -1, -1, -1, -1, -1], | 838 | + "shape": [-1, -1, -1, -1, -1], |
| 839 | "ori_shape": [-1, -1, -1, -1, -1], | 839 | "ori_shape": [-1, -1, -1, -1, -1], |
| 840 | - "range": [[1, -1], [1, -1], [1, -1], [1, -1], [1, -1], [1, -1]] | 840 | + "range": [[1, -1], [1, -1], [1, -1], [1, -1], [1, -1]] |
| 841 | } | 841 | } |
| 842 | ], | 842 | ], |
| 843 | "attrs": [ | 843 | "attrs": [ |
| @@ -884,7 +884,7 @@ | |||
| 884 | ] | 884 | ] |
| 885 | }, | 885 | }, |
| 886 | { | 886 | { |
| 887 | - "bin_filename": "Conv3DV2_NDC1HWC0_INT8_FRACTAL_Z_3D_INT8_FP32_NDC1HWC0_FP16", | 887 | + "bin_filename": "Conv3DV2_NDC1HWC0_INT8_FRACTAL_Z_3D_INT8_FP32_NCDHW_FP16", |
| 888 | "inputs": [ | 888 | "inputs": [ |
| 889 | { | 889 | { |
| 890 | "name": "x", | 890 | "name": "x", |
| @@ -958,12 +958,12 @@ | |||
| 958 | "name": "y", | 958 | "name": "y", |
| 959 | "index": 0, | 959 | "index": 0, |
| 960 | "dtype": "float16", | 960 | "dtype": "float16", |
| 961 | - "format": "NDC1HWC0", | 961 | + "format": "NCDHW", |
| 962 | "ori_format": "NCDHW", | 962 | "ori_format": "NCDHW", |
| 963 | "paramType": "required", | 963 | "paramType": "required", |
| 964 | - "shape": [-1, -1, -1, -1, -1, -1], | 964 | + "shape": [-1, -1, -1, -1, -1], |
| 965 | "ori_shape": [-1, -1, -1, -1, -1], | 965 | "ori_shape": [-1, -1, -1, -1, -1], |
| 966 | - "range": [[1, -1], [1, -1], [1, -1], [1, -1], [1, -1], [1, -1]] | 966 | + "range": [[1, -1], [1, -1], [1, -1], [1, -1], [1, -1]] |
| 967 | } | 967 | } |
| 968 | ], | 968 | ], |
| 969 | "attrs": [ | 969 | "attrs": [ |
| @@ -98,8 +98,8 @@ static const std::map<std::string, std::vector<ge::Format>> conv3dv2OffsetWForma | |||
| 98 | }; | 98 | }; |
| 99 | static const std::map<std::string, std::vector<ge::Format>> conv3dv2OutputFormat = { | 99 | static const std::map<std::string, std::vector<ge::Format>> conv3dv2OutputFormat = { |
| 100 | {"ascend910b", {ge::FORMAT_NDC1HWC0, ge::FORMAT_NDC1HWC0, ge::FORMAT_NDC1HWC0, ge::FORMAT_NCDHW, | 100 | {"ascend910b", {ge::FORMAT_NDC1HWC0, ge::FORMAT_NDC1HWC0, ge::FORMAT_NDC1HWC0, ge::FORMAT_NCDHW, |
| 101 | - ge::FORMAT_NCDHW, ge::FORMAT_NCDHW, ge::FORMAT_NDC1HWC0, ge::FORMAT_NDC1HWC0, | 101 | + ge::FORMAT_NCDHW, ge::FORMAT_NCDHW, ge::FORMAT_NCDHW, ge::FORMAT_NCDHW, |
| 102 | - ge::FORMAT_NDC1HWC0, ge::FORMAT_NDC1HWC0}}, | 102 | + ge::FORMAT_NCDHW, ge::FORMAT_NCDHW}}, |
| 103 | {"ascend950", {ge::FORMAT_NCDHW, ge::FORMAT_NCDHW, ge::FORMAT_NCDHW, ge::FORMAT_NCDHW, | 103 | {"ascend950", {ge::FORMAT_NCDHW, ge::FORMAT_NCDHW, ge::FORMAT_NCDHW, ge::FORMAT_NCDHW, |
| 104 | ge::FORMAT_NDHWC, ge::FORMAT_NDHWC, ge::FORMAT_NDHWC, ge::FORMAT_NDHWC, | 104 | ge::FORMAT_NDHWC, ge::FORMAT_NDHWC, ge::FORMAT_NDHWC, ge::FORMAT_NDHWC, |
| 105 | ge::FORMAT_NDHWC, ge::FORMAT_NDHWC, ge::FORMAT_NDHWC}} | 105 | ge::FORMAT_NDHWC, ge::FORMAT_NDHWC, ge::FORMAT_NDHWC}} |
| @@ -1016,10 +1016,11 @@ void Conv3dTilingAlgorithm::GetVecTiling() const | |||
| 1016 | tilingIns_->ubTilingInfo.scaleAndBiasLoadType = NUB_LOAD; | 1016 | tilingIns_->ubTilingInfo.scaleAndBiasLoadType = NUB_LOAD; |
| 1017 | } | 1017 | } |
| 1018 | 1018 | ||
| 1019 | + uint64_t multiN0 = tilingIns_->l0TilingInfo.nL0 / tilingIns_->cubeInfo.n0; | ||
| 1019 | for (uint64_t m = 1; m <= tilingIns_->l0TilingInfo.mL0; m++) { //优先全载m方向，切分n方向，再考虑是否可以开doubleBuffer | 1020 | for (uint64_t m = 1; m <= tilingIns_->l0TilingInfo.mL0; m++) { //优先全载m方向，切分n方向，再考虑是否可以开doubleBuffer |
| 1020 | - for (uint64_t n = 1; n <= tilingIns_->l0TilingInfo.nL0 / tilingIns_->cubeInfo.n0; n++) { | 1021 | + for (uint64_t n = 1; n <= multiN0; n++) { |
| 1021 | uint64_t mUB = CeilDiv(tilingIns_->l0TilingInfo.mL0, m); | 1022 | uint64_t mUB = CeilDiv(tilingIns_->l0TilingInfo.mL0, m); |
| 1022 | - uint64_t nUB = CeilDiv(tilingIns_->l0TilingInfo.nL0, n); | 1023 | + uint64_t nUB = CeilDiv(multiN0, n) * tilingIns_->cubeInfo.n0; |
| 1023 | uint64_t usedInSize = this->ubInDTypeSize * mUB * nUB; | 1024 | uint64_t usedInSize = this->ubInDTypeSize * mUB * nUB; |
| 1024 | uint64_t usedOutSize = this->outputDTypeSize * mUB * nUB; | 1025 | uint64_t usedOutSize = this->outputDTypeSize * mUB * nUB; |
| 1025 | if (tilingIns_->ubTilingInfo.scaleAndBiasLoadType == NUB_LOAD) { | 1026 | if (tilingIns_->ubTilingInfo.scaleAndBiasLoadType == NUB_LOAD) { |
| @@ -807,38 +807,56 @@ bool Conv3dTilingEngine::CheckParamsDtype() | |||
| 807 | return CheckParamsDtypeWithoutBias(logTag, descInfo_); | 807 | return CheckParamsDtypeWithoutBias(logTag, descInfo_); |
| 808 | } | 808 | } |
| 809 | 809 | ||
| 810 | +bool Conv3dTilingEngine::CheckValidFormatCombo(Conv3dApiTiling::ConvFormat expectFmap, | ||
| 811 | + Conv3dApiTiling::ConvFormat expectWeight, | ||
| 812 | + Conv3dApiTiling::ConvFormat expectOut, | ||
| 813 | + const char *errMsg) | ||
| 814 | +{ | ||
| 815 | + if (descInfo_.fMapFormat != expectFmap || | ||
| 816 | + descInfo_.weightFormat != expectWeight || | ||
| 817 | + descInfo_.outFormat != expectOut) { | ||
| 818 | + OP_LOGE(logTag_.c_str(), | ||
| 819 | + "%s Current formats: [fmap=%s, weight=%s, output=%s]", | ||
| 820 | + errMsg, | ||
| 821 | + Conv3dApiTiling::g_formatToStr.at(descInfo_.fMapFormat).c_str(), | ||
| 822 | + Conv3dApiTiling::g_formatToStr.at(descInfo_.weightFormat).c_str(), | ||
| 823 | + Conv3dApiTiling::g_formatToStr.at(descInfo_.outFormat).c_str()); | ||
| 824 | + return false; | ||
| 825 | + } | ||
| 826 | + return true; | ||
| 827 | +} | ||
| 828 | + | ||
| 810 | bool Conv3dTilingEngine::CheckInputFormat() | 829 | bool Conv3dTilingEngine::CheckInputFormat() |
| 811 | { | 830 | { |
| 812 | OP_LOGD(logTag_.c_str(), "Checking input format compatibility - mode: %s", | 831 | OP_LOGD(logTag_.c_str(), "Checking input format compatibility - mode: %s", |
| 813 | - isPointWise ? "Pointwise" : "Regular"); | 832 | + isPointWise ? "Pointwise" : (flagInfo_.hasScale ? "Quant" : "Regular")); |
| 814 | 833 | ||
| 815 | // Validate based on pointwise mode | 834 | // Validate based on pointwise mode |
| 816 | - if (isPointWise) { | 835 | + if (isPointWise) { |
| 817 | // Pointwise mode: all tensors must be NCDHW | 836 | // Pointwise mode: all tensors must be NCDHW |
| 818 | - if (descInfo_.fMapFormat != Conv3dApiTiling::ConvFormat::NCDHW || | 837 | + if(!CheckValidFormatCombo( |
| 819 | - descInfo_.weightFormat != Conv3dApiTiling::ConvFormat::NCDHW || | 838 | + Conv3dApiTiling::ConvFormat::NCDHW, |
| 820 | - descInfo_.outFormat != Conv3dApiTiling::ConvFormat::NCDHW) { | 839 | + Conv3dApiTiling::ConvFormat::NCDHW, |
| 821 | - OP_LOGE(logTag_.c_str(), | 840 | + Conv3dApiTiling::ConvFormat::NCDHW, |
| 822 | - "Pointwise convolution (1x1x1 kernel) requires NCDHW format for all tensors. " | 841 | + "Pointwise convolution (1x1x1 kernel) requires NCDHW format for all tensors.")) { |
| 823 | - "Current formats: [fmap=%s, weight=%s, output=%s]", | ||
| 824 | - Conv3dApiTiling::g_formatToStr.at(descInfo_.fMapFormat).c_str(), | ||
| 825 | - Conv3dApiTiling::g_formatToStr.at(descInfo_.weightFormat).c_str(), | ||
| 826 | - Conv3dApiTiling::g_formatToStr.at(descInfo_.outFormat).c_str()); | ||
| 827 | return false; | 842 | return false; |
| 828 | } | 843 | } |
| 829 | - } else { | 844 | + } else if (flagInfo_.hasScale) { |
| 845 | + // Quant mode: NDC1HWC0 for fmap, FRACTAL_Z_3D for weight, NCDHW for output | ||
| 846 | + if(!CheckValidFormatCombo( | ||
| 847 | + Conv3dApiTiling::ConvFormat::NDC1HWC0, | ||
| 848 | + Conv3dApiTiling::ConvFormat::FRACTAL_Z_3D, | ||
| 849 | + Conv3dApiTiling::ConvFormat::NCDHW, | ||
| 850 | + "Quant convolution requires [NDC1HWC0, FRACTAL_Z_3D, NCDHW] formats.")) { | ||
| 851 | + return false; | ||
| 852 | + } | ||
| 853 | + } else if (!CheckValidFormatCombo( | ||
| 854 | + Conv3dApiTiling::ConvFormat::NDC1HWC0, | ||
| 855 | + Conv3dApiTiling::ConvFormat::FRACTAL_Z_3D, | ||
| 856 | + Conv3dApiTiling::ConvFormat::NDC1HWC0, | ||
| 857 | + "Regular convolution requires [NDC1HWC0, FRACTAL_Z_3D, NDC1HWC0] formats.")) { | ||
| 830 | // Regular mode: NDC1HWC0 for fmap/output, FRACTAL_Z_3D for weight | 858 | // Regular mode: NDC1HWC0 for fmap/output, FRACTAL_Z_3D for weight |
| 831 | - if (descInfo_.fMapFormat != Conv3dApiTiling::ConvFormat::NDC1HWC0 || | 859 | + return false; |
| 832 | - descInfo_.weightFormat != Conv3dApiTiling::ConvFormat::FRACTAL_Z_3D || | ||
| 833 | - descInfo_.outFormat != Conv3dApiTiling::ConvFormat::NDC1HWC0) { | ||
| 834 | - OP_LOGE(logTag_.c_str(), | ||
| 835 | - "Regular convolution requires [NDC1HWC0, FRACTAL_Z_3D, NDC1HWC0] formats. " | ||
| 836 | - "Current formats: [fmap=%s, weight=%s, output=%s]", | ||
| 837 | - Conv3dApiTiling::g_formatToStr.at(descInfo_.fMapFormat).c_str(), | ||
| 838 | - Conv3dApiTiling::g_formatToStr.at(descInfo_.weightFormat).c_str(), | ||
| 839 | - Conv3dApiTiling::g_formatToStr.at(descInfo_.outFormat).c_str()); | ||
| 840 | - return false; | ||
| 841 | - } | ||
| 842 | } | 860 | } |
| 843 | 861 | ||
| 844 | // Validate bias format if present | 862 | // Validate bias format if present |
| @@ -150,6 +150,10 @@ public: | |||
| 150 | uint64_t CalcMinL1LoadSize(uint8_t outputOrder); | 150 | uint64_t CalcMinL1LoadSize(uint8_t outputOrder); |
| 151 | bool CheckInputLimitsHwMode(); | 151 | bool CheckInputLimitsHwMode(); |
| 152 | bool CheckDims(const std::vector<int64_t>& shape); | 152 | bool CheckDims(const std::vector<int64_t>& shape); |
| 153 | + bool CheckValidFormatCombo(Conv3dApiTiling::ConvFormat expectFmap, | ||
| 154 | + Conv3dApiTiling::ConvFormat expectWeight, | ||
| 155 | + Conv3dApiTiling::ConvFormat expectOut, | ||
| 156 | + const char *errMsg); | ||
| 153 | 157 | ||
| 154 | private: | 158 | private: |
| 155 | std::string logTag_ {"Conv3DV2"}; | 159 | std::string logTag_ {"Conv3DV2"}; |
| @@ -693,6 +693,9 @@ struct Iterate { | |||
| 693 | } else { | 693 | } else { |
| 694 | self->ctx.madIns.SetMN(AlignB(m, BLOCK_L0_M), AlignB(n, BLOCK_L0_N)); | 694 | self->ctx.madIns.SetMN(AlignB(m, BLOCK_L0_M), AlignB(n, BLOCK_L0_N)); |
| 695 | self->ctx.copyOutIns.SetMN(m, AlignB(n, self->ctx.cout0)); | 695 | self->ctx.copyOutIns.SetMN(m, AlignB(n, self->ctx.cout0)); |
| 696 | + if constexpr (Intf::quantType == static_cast<int8_t>(QuantType::PER_CHANNEL_NO_OFFSET)) { | ||
| 697 | + self->ctx.copyOutIns.SetVecN(n); | ||
| 698 | + } | ||
| 696 | if ASCEND_IS_AIC { | 699 | if ASCEND_IS_AIC { |
| 697 | InitBiasWithNormal(self, m, n); | 700 | InitBiasWithNormal(self, m, n); |
| 698 | } | 701 | } |
| @@ -34,20 +34,24 @@ struct VecCompute { | |||
| 34 | static __aicore__ inline bool call( | 34 | static __aicore__ inline bool call( |
| 35 | Intf *self, const GlobalTensor<typename Intf::OutputT> &output) | 35 | Intf *self, const GlobalTensor<typename Intf::OutputT> &output) |
| 36 | { | 36 | { |
| 37 | - uint64_t mSize, nSize; | 37 | + uint64_t mSize, nSize, curNSize; |
| 38 | - uint64_t ws_startoffset = self->ctx.workspaceDbFlag * self->ctx.conv3dTiling->mL0 * self->ctx.conv3dTiling->nL0; | 38 | + uint64_t ws_startoffset = self->ctx.workspaceDbFlag * self->ctx.conv3dTiling->mL0 * |
| 39 | + self->ctx.conv3dTiling->nL0; | ||
| 39 | self->ctx.copyOutIns.GetL0CSize(mSize, nSize); | 40 | self->ctx.copyOutIns.GetL0CSize(mSize, nSize); |
| 41 | + self->ctx.copyOutIns.GetCurNSize(curNSize); | ||
| 40 | uint64_t n16num = nSize / BLOCK_L0_N; | 42 | uint64_t n16num = nSize / BLOCK_L0_N; |
| 41 | if (n16num > 1) { | 43 | if (n16num > 1) { |
| 42 | self->ctx.outMoffset = 0; | 44 | self->ctx.outMoffset = 0; |
| 43 | if (self->ctx.subblockIdx) { | 45 | if (self->ctx.subblockIdx) { |
| 44 | uint64_t halfnum = CeilDIV(n16num, 2) * BLOCK_L0_N; | 46 | uint64_t halfnum = CeilDIV(n16num, 2) * BLOCK_L0_N; |
| 45 | nSize = nSize - halfnum; | 47 | nSize = nSize - halfnum; |
| 48 | + curNSize = curNSize - halfnum; | ||
| 46 | self->ctx.channelOffset = halfnum; | 49 | self->ctx.channelOffset = halfnum; |
| 47 | ws_startoffset += halfnum * mSize; | 50 | ws_startoffset += halfnum * mSize; |
| 48 | self->ctx.outNoffset = halfnum; | 51 | self->ctx.outNoffset = halfnum; |
| 49 | } else { | 52 | } else { |
| 50 | nSize = CeilDIV(n16num, 2) * BLOCK_L0_N; | 53 | nSize = CeilDIV(n16num, 2) * BLOCK_L0_N; |
| 54 | + curNSize = nSize; | ||
| 51 | } | 55 | } |
| 52 | } else { | 56 | } else { |
| 53 | self->ctx.channelOffset = 0; | 57 | self->ctx.channelOffset = 0; |
| @@ -71,18 +75,19 @@ struct VecCompute { | |||
| 71 | CopyScaleAndBias(self, 0, nSize); | 75 | CopyScaleAndBias(self, 0, nSize); |
| 72 | } | 76 | } |
| 73 | CrossCoreWaitFlag(self->ctx.C2VEvent + self->ctx.workspaceDbFlag); | 77 | CrossCoreWaitFlag(self->ctx.C2VEvent + self->ctx.workspaceDbFlag); |
| 74 | - DequantCompute(self, mSize, nSize, ws_startoffset, output); | 78 | + DequantCompute(self, mSize, nSize, curNSize, ws_startoffset, output); |
| 75 | CrossCoreSetFlag<0x2, PIPE_MTE2>(self->ctx.V2CEvent + self->ctx.workspaceDbFlag); | 79 | CrossCoreSetFlag<0x2, PIPE_MTE2>(self->ctx.V2CEvent + self->ctx.workspaceDbFlag); |
| 76 | 80 | ||
| 77 | return false; | 81 | return false; |
| 78 | } | 82 | } |
| 79 | 83 | ||
| 80 | - static __aicore__ inline void DequantCompute(Intf *self, uint64_t mSize, uint64_t nSize, uint64_t ws_startoffset, | 84 | + static __aicore__ inline void DequantCompute(Intf *self, uint64_t mSize, uint64_t nSize, |
| 85 | + uint64_t nCurSize, uint64_t ws_startoffset, | ||
| 81 | const GlobalTensor<typename Intf::OutputT> &output) | 86 | const GlobalTensor<typename Intf::OutputT> &output) |
| 82 | { | 87 | { |
| 83 | uint32_t maxnUBIter = CeilDIV(nSize, self->ctx.conv3dTiling->nUB); | 88 | uint32_t maxnUBIter = CeilDIV(nSize, self->ctx.conv3dTiling->nUB); |
| 84 | uint32_t maxmUBIter = CeilDIV(mSize, self->ctx.conv3dTiling->mUB); | 89 | uint32_t maxmUBIter = CeilDIV(mSize, self->ctx.conv3dTiling->mUB); |
| 85 | - uint16_t totalSrcStride = (mSize - self->ctx.conv3dTiling->mUB) * BLOCK_L0_N * self->ctx.sizeOfL0c / 32; | 90 | + uint16_t totalSrcStride = (mSize - self->ctx.conv3dTiling->mUB) * BLOCK_L0_N * self->ctx.sizeOfL0c / C0_SIZE; |
| 86 | for (uint32_t nIter = 0; nIter < maxnUBIter - 1; nIter++) { | 91 | for (uint32_t nIter = 0; nIter < maxnUBIter - 1; nIter++) { |
| 87 | if (self->ctx.conv3dTiling->scaleAndBiasLoadType == static_cast<int8_t>(LoadChannelType::NORMAL)) { | 92 | if (self->ctx.conv3dTiling->scaleAndBiasLoadType == static_cast<int8_t>(LoadChannelType::NORMAL)) { |
| 88 | CopyScaleAndBias(self, nIter, self->ctx.conv3dTiling->nUB); | 93 | CopyScaleAndBias(self, nIter, self->ctx.conv3dTiling->nUB); |
| @@ -91,14 +96,20 @@ struct VecCompute { | |||
| 91 | uint64_t srcOffset = ws_startoffset + (nIter * self->ctx.conv3dTiling->nUB * mSize + | 96 | uint64_t srcOffset = ws_startoffset + (nIter * self->ctx.conv3dTiling->nUB * mSize + |
| 92 | mIter * self->ctx.conv3dTiling->mUB * BLOCK_L0_N); | 97 | mIter * self->ctx.conv3dTiling->mUB * BLOCK_L0_N); |
| 93 | CopyIn(self, self->ctx.totalBlockCount, self->ctx.totalBlockLen, totalSrcStride, srcOffset); | 98 | CopyIn(self, self->ctx.totalBlockCount, self->ctx.totalBlockLen, totalSrcStride, srcOffset); |
| 94 | - oneVecCompute(self, self->ctx.conv3dTiling->mUB, self->ctx.conv3dTiling->nUB, mIter, nIter, output); | 99 | + oneVecCompute(self, self->ctx.conv3dTiling->mUB, self->ctx.conv3dTiling->nUB, mIter, nIter); |
| 100 | + CopyOut(self, self->ctx.conv3dTiling->mUB, self->ctx.conv3dTiling->nUB, mIter, nIter, output); | ||
| 95 | } | 101 | } |
| 96 | - uint16_t blockLen = (mSize - (maxmUBIter - 1) * self->ctx.conv3dTiling->mUB) * BLOCK_L0_N * self->ctx.sizeOfL0c / 32; | 102 | + uint16_t blockLen = (mSize - (maxmUBIter - 1) * self->ctx.conv3dTiling->mUB) * BLOCK_L0_N * |
| 97 | - uint16_t srcStride = (maxmUBIter - 1) * self->ctx.conv3dTiling->mUB * BLOCK_L0_N * self->ctx.sizeOfL0c / 32 ; | 103 | + self->ctx.sizeOfL0c / C0_SIZE; |
| 104 | + uint16_t srcStride = (maxmUBIter - 1) * self->ctx.conv3dTiling->mUB * BLOCK_L0_N * | ||
| 105 | + self->ctx.sizeOfL0c / C0_SIZE ; | ||
| 98 | uint64_t srcOffset = ws_startoffset + (nIter * self->ctx.conv3dTiling->nUB * mSize + | 106 | uint64_t srcOffset = ws_startoffset + (nIter * self->ctx.conv3dTiling->nUB * mSize + |
| 99 | (maxmUBIter - 1) * self->ctx.conv3dTiling->mUB * BLOCK_L0_N); | 107 | (maxmUBIter - 1) * self->ctx.conv3dTiling->mUB * BLOCK_L0_N); |
| 100 | CopyIn(self, self->ctx.totalBlockCount, blockLen, srcStride, srcOffset); | 108 | CopyIn(self, self->ctx.totalBlockCount, blockLen, srcStride, srcOffset); |
| 101 | - oneVecCompute(self, mSize - (maxmUBIter - 1) * self->ctx.conv3dTiling->mUB, self->ctx.conv3dTiling->nUB, maxmUBIter - 1, nIter, output); | 109 | + oneVecCompute(self, mSize - (maxmUBIter - 1) * self->ctx.conv3dTiling->mUB, self->ctx.conv3dTiling->nUB, |
| 110 | + maxmUBIter - 1, nIter); | ||
| 111 | + CopyOut(self, mSize - (maxmUBIter - 1) * self->ctx.conv3dTiling->mUB, self->ctx.conv3dTiling->nUB, | ||
| 112 | + maxmUBIter - 1, nIter, output); | ||
| 102 | if (self->ctx.conv3dTiling->scaleAndBiasLoadType == static_cast<int8_t>(LoadChannelType::NORMAL)) { | 113 | if (self->ctx.conv3dTiling->scaleAndBiasLoadType == static_cast<int8_t>(LoadChannelType::NORMAL)) { |
| 103 | FreeScaleAndBias(self); | 114 | FreeScaleAndBias(self); |
| 104 | } | 115 | } |
| @@ -111,22 +122,30 @@ struct VecCompute { | |||
| 111 | uint64_t srcOffset = ws_startoffset + (maxnUBIter-1) * self->ctx.conv3dTiling->nUB * mSize + | 122 | uint64_t srcOffset = ws_startoffset + (maxnUBIter-1) * self->ctx.conv3dTiling->nUB * mSize + |
| 112 | mIter * self->ctx.conv3dTiling->mUB * BLOCK_L0_N ; | 123 | mIter * self->ctx.conv3dTiling->mUB * BLOCK_L0_N ; |
| 113 | CopyIn(self, blockCount, self->ctx.totalBlockLen, totalSrcStride, srcOffset); | 124 | CopyIn(self, blockCount, self->ctx.totalBlockLen, totalSrcStride, srcOffset); |
| 114 | - oneVecCompute(self, self->ctx.conv3dTiling->mUB, nSize - (maxnUBIter - 1) * self->ctx.conv3dTiling->nUB, mIter, maxnUBIter - 1, output); | 125 | + oneVecCompute(self, self->ctx.conv3dTiling->mUB, nSize - (maxnUBIter - 1) * self->ctx.conv3dTiling->nUB, |
| 126 | + mIter, maxnUBIter - 1); | ||
| 127 | + CopyOut(self, self->ctx.conv3dTiling->mUB, nCurSize - (maxnUBIter - 1) * self->ctx.conv3dTiling->nUB, | ||
| 128 | + mIter, maxnUBIter - 1, output); | ||
| 115 | } | 129 | } |
| 116 | - uint16_t blockLen = (mSize - (maxmUBIter - 1) * self->ctx.conv3dTiling->mUB) * BLOCK_L0_N * self->ctx.sizeOfL0c / 32; | 130 | + uint16_t blockLen = (mSize - (maxmUBIter - 1) * self->ctx.conv3dTiling->mUB) * BLOCK_L0_N * |
| 117 | - uint16_t srcStride = (maxmUBIter - 1) * self->ctx.conv3dTiling->mUB * BLOCK_L0_N * self->ctx.sizeOfL0c / 32 ; | 131 | + self->ctx.sizeOfL0c / C0_SIZE; |
| 132 | + uint16_t srcStride = (maxmUBIter - 1) * self->ctx.conv3dTiling->mUB * BLOCK_L0_N * | ||
| 133 | + self->ctx.sizeOfL0c / C0_SIZE; | ||
| 118 | uint64_t srcOffset = ws_startoffset + ((maxnUBIter-1) * self->ctx.conv3dTiling->nUB * mSize + | 134 | uint64_t srcOffset = ws_startoffset + ((maxnUBIter-1) * self->ctx.conv3dTiling->nUB * mSize + |
| 119 | (maxmUBIter - 1) * self->ctx.conv3dTiling->mUB * BLOCK_L0_N); | 135 | (maxmUBIter - 1) * self->ctx.conv3dTiling->mUB * BLOCK_L0_N); |
| 120 | CopyIn(self, blockCount, blockLen, srcStride, srcOffset); | 136 | CopyIn(self, blockCount, blockLen, srcStride, srcOffset); |
| 121 | - oneVecCompute(self, mSize - (maxmUBIter - 1) * self->ctx.conv3dTiling->mUB, nSize - (maxnUBIter - 1) * self->ctx.conv3dTiling->nUB, | 137 | + oneVecCompute(self, mSize - (maxmUBIter - 1) * self->ctx.conv3dTiling->mUB, |
| 122 | - maxmUBIter - 1, maxnUBIter - 1, output); | 138 | + nSize - (maxnUBIter - 1) * self->ctx.conv3dTiling->nUB, maxmUBIter - 1, maxnUBIter - 1); |
| 139 | + CopyOut(self, mSize - (maxmUBIter - 1) * self->ctx.conv3dTiling->mUB, | ||
| 140 | + nCurSize - (maxnUBIter - 1) * self->ctx.conv3dTiling->nUB, maxmUBIter - 1, maxnUBIter - 1, output); | ||
| 123 | if (self->ctx.conv3dTiling->scaleAndBiasLoadType == static_cast<int8_t>(LoadChannelType::LOAD_TOTAL_LC0) || | 141 | if (self->ctx.conv3dTiling->scaleAndBiasLoadType == static_cast<int8_t>(LoadChannelType::LOAD_TOTAL_LC0) || |
| 124 | self->ctx.conv3dTiling->scaleAndBiasLoadType == static_cast<int8_t>(LoadChannelType::NORMAL)) { | 142 | self->ctx.conv3dTiling->scaleAndBiasLoadType == static_cast<int8_t>(LoadChannelType::NORMAL)) { |
| 125 | FreeScaleAndBias(self); | 143 | FreeScaleAndBias(self); |
| 126 | } | 144 | } |
| 127 | } | 145 | } |
| 128 | 146 | ||
| 129 | - static __aicore__ inline void CopyIn(Intf *self, uint16_t blockCount, uint16_t blockLen, uint16_t srcStride, uint64_t srcOffset) | 147 | + static __aicore__ inline void CopyIn(Intf *self, uint16_t blockCount, uint16_t blockLen, |
| 148 | + uint16_t srcStride, uint64_t srcOffset) | ||
| 130 | { | 149 | { |
| 131 | self->ctx.ubin = self->ctx.queueUBin.template AllocTensor<typename Intf::L0cT>(); | 150 | self->ctx.ubin = self->ctx.queueUBin.template AllocTensor<typename Intf::L0cT>(); |
| 132 | DataCopyParams copyinParams; | 151 | DataCopyParams copyinParams; |
| @@ -192,38 +211,43 @@ struct VecCompute { | |||
| 192 | } | 211 | } |
| 193 | 212 | ||
| 194 | template <typename DataTypeT> | 213 | template <typename DataTypeT> |
| 195 | - static __aicore__ inline void CopyInChannel(Intf *self, const LocalTensor<DataTypeT>& dst, const GlobalTensor<DataTypeT> &src, uint16_t nIter, uint16_t num) | 214 | + static __aicore__ inline void CopyInChannel(Intf *self, const LocalTensor<DataTypeT>& dst, |
| 215 | + const GlobalTensor<DataTypeT> &src, | ||
| 216 | + uint16_t nIter, uint16_t num) | ||
| 196 | { | 217 | { |
| 197 | DataCopyParams copyinParams; | 218 | DataCopyParams copyinParams; |
| 198 | copyinParams.blockCount = 1; | 219 | copyinParams.blockCount = 1; |
| 199 | - copyinParams.blockLen = num * sizeof(DataTypeT) / 32; | 220 | + copyinParams.blockLen = num * sizeof(DataTypeT) / C0_SIZE; |
| 200 | uint64_t Offset = self->ctx.copyOutIns.GetChannelOffset(nIter * self->ctx.conv3dTiling->nUB); | 221 | uint64_t Offset = self->ctx.copyOutIns.GetChannelOffset(nIter * self->ctx.conv3dTiling->nUB); |
| 201 | DataCopy(dst, src[Offset], copyinParams); | 222 | DataCopy(dst, src[Offset], copyinParams); |
| 202 | } | 223 | } |
| 203 | 224 | ||
| 204 | - static __aicore__ inline void MulScaleAddBias(uint16_t m, uint16_t n, const LocalTensor<typename Intf::FP32T>& src, | 225 | + static __aicore__ inline void MulScaleAddBias(uint16_t m, uint16_t n, |
| 205 | - const LocalTensor<typename Intf::FP32T>& bias, const LocalTensor<typename Intf::FP32T>& scale) | 226 | + const LocalTensor<typename Intf::FP32T>& src, |
| 227 | + const LocalTensor<typename Intf::FP32T>& bias, | ||
| 228 | + const LocalTensor<typename Intf::FP32T>& scale) | ||
| 206 | { | 229 | { |
| 207 | uint64_t mask = BLOCK_L0_N; | 230 | uint64_t mask = BLOCK_L0_N; |
| 208 | BinaryRepeatParams repeatParams; | 231 | BinaryRepeatParams repeatParams; |
| 209 | repeatParams.dstBlkStride = 1; | 232 | repeatParams.dstBlkStride = 1; |
| 210 | repeatParams.src0BlkStride = 1; | 233 | repeatParams.src0BlkStride = 1; |
| 211 | repeatParams.src1BlkStride = 1; | 234 | repeatParams.src1BlkStride = 1; |
| 212 | - repeatParams.dstRepStride = BLOCK_L0_N * sizeof(typename Intf::FP32T) / 32; | 235 | + repeatParams.dstRepStride = BLOCK_L0_N * sizeof(typename Intf::FP32T) / C0_SIZE; |
| 213 | - repeatParams.src0RepStride = BLOCK_L0_N * sizeof(typename Intf::FP32T) / 32; | 236 | + repeatParams.src0RepStride = BLOCK_L0_N * sizeof(typename Intf::FP32T) / C0_SIZE; |
| 214 | repeatParams.src1RepStride = 0; | 237 | repeatParams.src1RepStride = 0; |
| 215 | 238 | ||
| 216 | uint16_t maxNiter = n / BLOCK_L0_N; | 239 | uint16_t maxNiter = n / BLOCK_L0_N; |
| 217 | - uint16_t maxMiter = CeilDIV(m, 255); | 240 | + uint16_t maxMiter = CeilDIV(m, MAX_VEC_LEN); |
| 218 | for(uint16_t niter = 0; niter < maxNiter; niter++) { | 241 | for(uint16_t niter = 0; niter < maxNiter; niter++) { |
| 219 | for (uint16_t miter = 0; miter < maxMiter - 1; miter++) | 242 | for (uint16_t miter = 0; miter < maxMiter - 1; miter++) |
| 220 | { | 243 | { |
| 221 | - uint64_t offset = m * niter * BLOCK_L0_N + 255 * miter * BLOCK_L0_N; | 244 | + uint64_t offset = m * niter * BLOCK_L0_N + MAX_VEC_LEN * miter * BLOCK_L0_N; |
| 222 | - Mul(src[offset], src[offset], scale[niter * BLOCK_L0_N], mask, 255, repeatParams); | 245 | + Mul(src[offset], src[offset], scale[niter * BLOCK_L0_N], mask, MAX_VEC_LEN, repeatParams); |
| 223 | } | 246 | } |
| 224 | 247 | ||
| 225 | - uint64_t offset = m * niter * BLOCK_L0_N + 255 * (maxMiter - 1) * BLOCK_L0_N; | 248 | + uint64_t offset = m * niter * BLOCK_L0_N + MAX_VEC_LEN * (maxMiter - 1) * BLOCK_L0_N; |
| 226 | - Mul(src[offset], src[offset], scale[niter * BLOCK_L0_N], mask, m - (maxMiter - 1) * 255, repeatParams); | 249 | + Mul(src[offset], src[offset], scale[niter * BLOCK_L0_N], mask, |
| 250 | + m - (maxMiter - 1) * MAX_VEC_LEN, repeatParams); | ||
| 227 | } | 251 | } |
| 228 | 252 | ||
| 229 | PipeBarrier<PIPE_V>(); | 253 | PipeBarrier<PIPE_V>(); |
| @@ -231,18 +255,20 @@ struct VecCompute { | |||
| 231 | for(uint16_t niter = 0; niter < maxNiter; niter++) { | 255 | for(uint16_t niter = 0; niter < maxNiter; niter++) { |
| 232 | for (uint16_t miter = 0; miter < maxMiter - 1; miter++) | 256 | for (uint16_t miter = 0; miter < maxMiter - 1; miter++) |
| 233 | { | 257 | { |
| 234 | - uint64_t offset = m * niter * BLOCK_L0_N + 255 * miter * BLOCK_L0_N; | 258 | + uint64_t offset = m * niter * BLOCK_L0_N + MAX_VEC_LEN * miter * BLOCK_L0_N; |
| 235 | - Add(src[offset], src[offset], bias[niter * BLOCK_L0_N], mask, 255, repeatParams); | 259 | + Add(src[offset], src[offset], bias[niter * BLOCK_L0_N], mask, MAX_VEC_LEN, repeatParams); |
| 236 | } | 260 | } |
| 237 | - uint64_t offset = m * niter * BLOCK_L0_N + 255 * (maxMiter - 1) * BLOCK_L0_N; | 261 | + uint64_t offset = m * niter * BLOCK_L0_N + MAX_VEC_LEN * (maxMiter - 1) * BLOCK_L0_N; |
| 238 | - Add(src[offset], src[offset], bias[niter * BLOCK_L0_N], mask, m - (maxMiter - 1) * 255, repeatParams); | 262 | + Add(src[offset], src[offset], bias[niter * BLOCK_L0_N], mask, |
| 263 | + m - (maxMiter - 1) * MAX_VEC_LEN, repeatParams); | ||
| 239 | } | 264 | } |
| 240 | } | 265 | } |
| 241 | 266 | ||
| 242 | - static __aicore__ inline void oneVecCompute(Intf *self, uint16_t m, uint16_t n, uint16_t mIter, uint16_t nIter, const GlobalTensor<typename Intf::OutputT> &output) | 267 | + static __aicore__ inline void oneVecCompute(Intf *self, uint16_t m, uint16_t n, uint16_t mIter, uint16_t nIter) |
| 243 | { | 268 | { |
| 244 | LocalTensor<typename Intf::L0cT> localUBin = self->ctx.queueUBin.template DeQue<typename Intf::L0cT>(); | 269 | LocalTensor<typename Intf::L0cT> localUBin = self->ctx.queueUBin.template DeQue<typename Intf::L0cT>(); |
| 245 | LocalTensor<typename Intf::FP32T> dstLocal = localUBin.template ReinterpretCast<typename Intf::FP32T>(); | 270 | LocalTensor<typename Intf::FP32T> dstLocal = localUBin.template ReinterpretCast<typename Intf::FP32T>(); |
| 271 | + LocalTensor<typename Intf::OutputT> transLocal = localUBin.template ReinterpretCast<typename Intf::OutputT>(); | ||
| 246 | 272 | ||
| 247 | Cast(dstLocal, localUBin, AscendC::RoundMode::CAST_RINT, m * n); | 273 | Cast(dstLocal, localUBin, AscendC::RoundMode::CAST_RINT, m * n); |
| 248 | 274 | ||
| @@ -259,7 +285,8 @@ struct VecCompute { | |||
| 259 | } | 285 | } |
| 260 | 286 | ||
| 261 | if (self->ctx.conv3dTiling->scaleAndBiasLoadType == static_cast<int8_t>(LoadChannelType::LOAD_TOTAL_CORE)) { | 287 | if (self->ctx.conv3dTiling->scaleAndBiasLoadType == static_cast<int8_t>(LoadChannelType::LOAD_TOTAL_CORE)) { |
| 262 | - uint64_t Offset = self->ctx.copyOutIns.GetChannelOffset(nIter * self->ctx.conv3dTiling->nUB) + self->ctx.channelOffset; | 288 | + uint64_t Offset = self->ctx.copyOutIns.GetChannelOffset(nIter * self->ctx.conv3dTiling->nUB) + |
| 289 | + self->ctx.channelOffset; | ||
| 263 | bias = self->ctx.ubbias[Offset]; | 290 | bias = self->ctx.ubbias[Offset]; |
| 264 | scale = self->ctx.ubscale[Offset]; | 291 | scale = self->ctx.ubscale[Offset]; |
| 265 | } | 292 | } |
| @@ -267,13 +294,65 @@ struct VecCompute { | |||
| 267 | MulScaleAddBias(m, n, dstLocal, bias, scale); | 294 | MulScaleAddBias(m, n, dstLocal, bias, scale); |
| 268 | 295 | ||
| 269 | //cast to bf16 or fp16 | 296 | //cast to bf16 or fp16 |
| 270 | - self->ctx.ubout = self->ctx.queueUBout.template AllocTensor<typename Intf::OutputT>(); | ||
| 271 | PipeBarrier<PIPE_V>(); | 297 | PipeBarrier<PIPE_V>(); |
| 272 | - Cast(self->ctx.ubout, dstLocal, AscendC::RoundMode::CAST_RINT, m * n); | 298 | + Cast(transLocal, dstLocal, AscendC::RoundMode::CAST_RINT, m * n); |
| 273 | - self->ctx.queueUBin.FreeTensor(localUBin); | 299 | + TransFormat(self, transLocal, m, n); |
| 274 | - self->ctx.queueUBout.EnQue(self->ctx.ubout); | 300 | + } |
| 275 | - LocalTensor<typename Intf::OutputT> copyubout = self->ctx.queueUBout.template DeQue<typename Intf::OutputT>(); | ||
| 276 | 301 | ||
| 302 | + static __aicore__ inline void TransFormat(Intf *self, const LocalTensor<typename Intf::OutputT>& transLocal, | ||
| 303 | + uint16_t m, uint16_t n) | ||
| 304 | + { | ||
| 305 | + using transT = std::conditional_t<IsSameType<typename Intf::OutputT, bfloat16_t>::value, | ||
| 306 | + uint16_t, | ||
| 307 | + typename Intf::OutputT>; | ||
| 308 | + LocalTensor<transT> vecInBuf_ = transLocal.template ReinterpretCast<transT>(); | ||
| 309 | + LocalTensor<transT> vecOutBuf_ = self->ctx.queueUBout.template AllocTensor<transT>(); | ||
| 310 | + | ||
| 311 | + TransDataTo5HDParams transDataParams; | ||
| 312 | + transDataParams.dstHighHalf = false; | ||
| 313 | + transDataParams.srcHighHalf = false; | ||
| 314 | + transDataParams.repeatTimes = static_cast<uint16_t>(CeilDIV(m, NCHW_CONV_ADDR_LIST_SIZE)); | ||
| 315 | + transDataParams.dstRepStride = 1; | ||
| 316 | + transDataParams.srcRepStride = NCHW_CONV_ADDR_LIST_SIZE; | ||
| 317 | + // 参考AscendC API的使用说明，当repeatTimes为1时，repStride需要设置为0 | ||
| 318 | + if (transDataParams.repeatTimes == 1) { | ||
| 319 | + transDataParams.dstRepStride = 0; | ||
| 320 | + transDataParams.srcRepStride = 0; | ||
| 321 | + } | ||
| 322 | + uint64_t dstLocalList[NCHW_CONV_ADDR_LIST_SIZE]; | ||
| 323 | + uint64_t srcLocalList[NCHW_CONV_ADDR_LIST_SIZE]; | ||
| 324 | + int64_t dstCount = 0; | ||
| 325 | + int64_t srcCount = 0; | ||
| 326 | + int loopCountN = CeilDIV(n, BLOCK_L0_N); | ||
| 327 | + int loopCountM = CeilDIV(m, BLOCK_L0_M); | ||
| 328 | + uint64_t dstIncrement = loopCountM * BLOCK_L0_M * BLOCK_L0_N; | ||
| 329 | + uint64_t srcIncrement = m * BLOCK_L0_N; | ||
| 330 | + for (int nIdx = 0; nIdx < loopCountN; nIdx++) { | ||
| 331 | + dstCount = nIdx * dstIncrement; | ||
| 332 | + for (int i = 0; i < NCHW_CONV_ADDR_LIST_SIZE; i++) { | ||
| 333 | + dstLocalList[i] = reinterpret_cast<uint64_t>(vecOutBuf_[dstCount].GetPhyAddr()); | ||
| 334 | + dstCount += loopCountM * BLOCK_L0_M; | ||
| 335 | + } | ||
| 336 | + srcCount = nIdx * srcIncrement; | ||
| 337 | + for (int i = 0; i < NCHW_CONV_ADDR_LIST_SIZE; i++) { | ||
| 338 | + srcLocalList[i] = reinterpret_cast<uint64_t>(vecInBuf_[srcCount].GetPhyAddr()); | ||
| 339 | + srcCount += BLOCK_L0_N; | ||
| 340 | + } | ||
| 341 | + TransDataTo5HD<transT>(dstLocalList, srcLocalList, transDataParams); | ||
| 342 | + } | ||
| 343 | + if constexpr (IsSameType<typename Intf::OutputT, bfloat16_t>::value) { | ||
| 344 | + self->ctx.ubout = vecOutBuf_.template ReinterpretCast<typename Intf::OutputT>(); | ||
| 345 | + } else { | ||
| 346 | + self->ctx.ubout = vecOutBuf_; | ||
| 347 | + } | ||
| 348 | + self->ctx.queueUBout.EnQue(self->ctx.ubout); | ||
| 349 | + self->ctx.queueUBin.FreeTensor(vecInBuf_); | ||
| 350 | + } | ||
| 351 | + | ||
| 352 | + static __aicore__ inline void CopyOut(Intf *self, uint16_t m, uint16_t n, uint16_t mIter, uint16_t nIter, | ||
| 353 | + const GlobalTensor<typename Intf::OutputT> &output) | ||
| 354 | + { | ||
| 355 | + LocalTensor<typename Intf::OutputT> copyubout = self->ctx.queueUBout.template DeQue<typename Intf::OutputT>(); | ||
| 277 | self->ctx.copyOutIns.CopyUBOut(output, mIter, nIter, m, n, copyubout); | 356 | self->ctx.copyOutIns.CopyUBOut(output, mIter, nIter, m, n, copyubout); |
| 278 | self->ctx.queueUBout.FreeTensor(copyubout); | 357 | self->ctx.queueUBout.FreeTensor(copyubout); |
| 279 | } | 358 | } |
| @@ -58,11 +58,19 @@ constexpr ConvFormat aFormat = ConvFormat::NCDHW; | |||
| 58 | constexpr ConvFormat bFormat = ConvFormat::NCDHW; | 58 | constexpr ConvFormat bFormat = ConvFormat::NCDHW; |
| 59 | constexpr ConvFormat cFormat = ConvFormat::NCDHW; | 59 | constexpr ConvFormat cFormat = ConvFormat::NCDHW; |
| 60 | constexpr ConvBL1ByPass bL1ByPassFlag = ConvBL1ByPass::BYPASS_OFF; | 60 | constexpr ConvBL1ByPass bL1ByPassFlag = ConvBL1ByPass::BYPASS_OFF; |
| 61 | -#else | 61 | +constexpr QuantType quantType = QuantType::NO_QUANT; |
| 62 | + | ||
| 62 | constexpr ConvFormat aFormat = ConvFormat::NDC1HWC0; | 63 | constexpr ConvFormat aFormat = ConvFormat::NDC1HWC0; |
| 63 | constexpr ConvFormat bFormat = ConvFormat::FRACTAL_Z_3D; | 64 | constexpr ConvFormat bFormat = ConvFormat::FRACTAL_Z_3D; |
| 64 | constexpr ConvFormat cFormat = ConvFormat::NDC1HWC0; | 65 | constexpr ConvFormat cFormat = ConvFormat::NDC1HWC0; |
| 65 | constexpr ConvBL1ByPass bL1ByPassFlag = ConvBL1ByPass::BYPASS_ON; | 66 | constexpr ConvBL1ByPass bL1ByPassFlag = ConvBL1ByPass::BYPASS_ON; |
| 67 | +constexpr QuantType quantType = QuantType::NO_QUANT; | ||
| 68 | + | ||
| 69 | +constexpr ConvFormat aFormat = ConvFormat::NDC1HWC0; | ||
| 70 | +constexpr ConvFormat bFormat = ConvFormat::FRACTAL_Z_3D; | ||
| 71 | +constexpr ConvFormat cFormat = ConvFormat::NCDHW; | ||
| 72 | +constexpr ConvBL1ByPass bL1ByPassFlag = ConvBL1ByPass::BYPASS_ON; | ||
| 73 | +constexpr QuantType quantType = QuantType::PER_CHANNEL_NO_OFFSET; | ||
| 66 | 74 | ||
| 67 | 75 | ||
| 68 | 76 | ||
| @@ -70,12 +78,6 @@ constexpr ConvBL1ByPass bL1ByPassFlag = ConvBL1ByPass::BYPASS_ON; | |||
| 70 | 78 | ||
| 71 | constexpr ConvFormat biasFormat = ConvFormat::ND; | 79 | constexpr ConvFormat biasFormat = ConvFormat::ND; |
| 72 | constexpr ConvFormat scaleFormat = ConvFormat::ND; | 80 | constexpr ConvFormat scaleFormat = ConvFormat::ND; |
| 73 | - | ||
| 74 | - | ||
| 75 | -constexpr QuantType quantType = QuantType::NO_QUANT; | ||
| 76 | - | ||
| 77 | -constexpr QuantType quantType = QuantType::PER_CHANNEL_NO_OFFSET; | ||
| 78 | - | ||
| 79 | 81 | ||
| 80 | 82 | ||
| 81 | 83 | ||
| @@ -169,7 +169,11 @@ protected: | |||
| 169 | __aicore__ inline void InitBuffer(GM_ADDR x, GM_ADDR filter, GM_ADDR bias, GM_ADDR y, GM_ADDR scale, GM_ADDR workspace) | 169 | __aicore__ inline void InitBuffer(GM_ADDR x, GM_ADDR filter, GM_ADDR bias, GM_ADDR y, GM_ADDR scale, GM_ADDR workspace) |
| 170 | { | 170 | { |
| 171 | fmapOneBatchSize = conv3dRunInfo->din * c1In * conv3dRunInfo->hin * conv3dRunInfo->win * c0In; | 171 | fmapOneBatchSize = conv3dRunInfo->din * c1In * conv3dRunInfo->hin * conv3dRunInfo->win * c0In; |
| 172 | - outputOneBatchSize = conv3dRunInfo->dout * c1Out * conv3dRunInfo->hout * conv3dRunInfo->wout * c0Out; | 172 | + if constexpr (CONV_CFG::quantType == static_cast<int8_t>(QuantType::PER_CHANNEL_NO_OFFSET)) { |
| 173 | + outputOneBatchSize = conv3dRunInfo->cout * conv3dRunInfo->dout * conv3dRunInfo->hout * conv3dRunInfo->wout; | ||
| 174 | + } else { | ||
| 175 | + outputOneBatchSize = conv3dRunInfo->dout * c1Out * conv3dRunInfo->hout * conv3dRunInfo->wout * c0Out; | ||
| 176 | + } | ||
| 173 | 177 | ||
| 174 | int64_t diIdxStart = doIdxStart * conv3dRunInfo->strideD - conv3dRunInfo->padHead; | 178 | int64_t diIdxStart = doIdxStart * conv3dRunInfo->strideD - conv3dRunInfo->padHead; |
| 175 | int64_t hiIdxStart = (mIdxStart / conv3dRunInfo->wout) * conv3dRunInfo->strideH - conv3dRunInfo->padTop; | 179 | int64_t hiIdxStart = (mIdxStart / conv3dRunInfo->wout) * conv3dRunInfo->strideH - conv3dRunInfo->padTop; |
| @@ -181,6 +185,12 @@ protected: | |||
| 181 | uint64_t outputStartAddr = batchIdxStart * outputOneBatchSize + | 185 | uint64_t outputStartAddr = batchIdxStart * outputOneBatchSize + |
| 182 | doIdxStart * c1Out * conv3dRunInfo->hout * conv3dRunInfo->wout * c0Out + | 186 | doIdxStart * c1Out * conv3dRunInfo->hout * conv3dRunInfo->wout * c0Out + |
| 183 | nIdxStart * conv3dRunInfo->hout * conv3dRunInfo->wout + mIdxStart * c0Out; | 187 | nIdxStart * conv3dRunInfo->hout * conv3dRunInfo->wout + mIdxStart * c0Out; |
| 188 | + if constexpr (CONV_CFG::quantType == static_cast<int8_t>(QuantType::PER_CHANNEL_NO_OFFSET)) { | ||
| 189 | + outputStartAddr = batchIdxStart * outputOneBatchSize + | ||
| 190 | + nIdxStart * conv3dRunInfo->dout * conv3dRunInfo->hout * conv3dRunInfo->wout + | ||
| 191 | + doIdxStart * conv3dRunInfo->hout * conv3dRunInfo->wout + | ||
| 192 | + mIdxStart; | ||
| 193 | + } | ||
| 184 | ASC_OP_LOGD("[InitBuffer] fmStartAddr %d weightStartAddr %d outputStartAddr %d.\n", | 194 | ASC_OP_LOGD("[InitBuffer] fmStartAddr %d weightStartAddr %d outputStartAddr %d.\n", |
| 185 | fmStartAddr, | 195 | fmStartAddr, |
| 186 | weightStartAddr, | 196 | weightStartAddr, |
| @@ -208,7 +218,6 @@ protected: | |||
| 208 | if (isDoDimTail || isNDimTail || isMDimTail) [[unlikely]] { | 218 | if (isDoDimTail || isNDimTail || isMDimTail) [[unlikely]] { |
| 209 | conv.SetSingleOutputShape(1, singleCoreN, singleCoreDout, singleCoreM, 0); | 219 | conv.SetSingleOutputShape(1, singleCoreN, singleCoreDout, singleCoreM, 0); |
| 210 | } | 220 | } |
| 211 | - | ||
| 212 | int64_t diIdxStart = doIdxStart * conv3dRunInfo->strideD; | 221 | int64_t diIdxStart = doIdxStart * conv3dRunInfo->strideD; |
| 213 | int64_t hiIdxStart = (mIdxStart / conv3dRunInfo->wout) * conv3dRunInfo->strideH; | 222 | int64_t hiIdxStart = (mIdxStart / conv3dRunInfo->wout) * conv3dRunInfo->strideH; |
| 214 | ASC_OP_LOGD("[Conv3DV2KernelImpl] doIdxStart %d mIdxStart %d diIdxStart %d hiIdxStart %d.\n", | 223 | ASC_OP_LOGD("[Conv3DV2KernelImpl] doIdxStart %d mIdxStart %d diIdxStart %d hiIdxStart %d.\n", |
| @@ -76,8 +76,13 @@ protected: | |||
| 76 | { | 76 | { |
| 77 | this->fmapOneBatchSize = | 77 | this->fmapOneBatchSize = |
| 78 | this->conv3dRunInfo->din * this->c1In * this->conv3dRunInfo->hin * this->conv3dRunInfo->win * this->c0In; | 78 | this->conv3dRunInfo->din * this->c1In * this->conv3dRunInfo->hin * this->conv3dRunInfo->win * this->c0In; |
| 79 | - this->outputOneBatchSize = | 79 | + if constexpr (CONV_CFG::quantType == static_cast<int8_t>(QuantType::PER_CHANNEL_NO_OFFSET)) { |
| 80 | - this->conv3dRunInfo->dout * this->c1Out * this->conv3dRunInfo->hout * this->conv3dRunInfo->wout * this->c0Out; | 80 | + this->outputOneBatchSize = |
| 81 | + this->conv3dRunInfo->cout * this->conv3dRunInfo->dout * this->conv3dRunInfo->hout * this->conv3dRunInfo->wout; | ||
| 82 | + } else { | ||
| 83 | + this->outputOneBatchSize = | ||
| 84 | + this->conv3dRunInfo->dout * this->c1Out * this->conv3dRunInfo->hout * this->conv3dRunInfo->wout * this->c0Out; | ||
| 85 | + } | ||
| 81 | 86 | ||
| 82 | int64_t diIdxStart = this->doIdxStart * this->conv3dRunInfo->strideD - this->conv3dRunInfo->padHead; | 87 | int64_t diIdxStart = this->doIdxStart * this->conv3dRunInfo->strideD - this->conv3dRunInfo->padHead; |
| 83 | int64_t hiIdxStart = this->mIdxStart * this->conv3dRunInfo->strideH - this->conv3dRunInfo->padTop; | 88 | int64_t hiIdxStart = this->mIdxStart * this->conv3dRunInfo->strideH - this->conv3dRunInfo->padTop; |
| @@ -92,6 +97,12 @@ protected: | |||
| 92 | this->doIdxStart * this->c1Out * this->conv3dRunInfo->hout * this->conv3dRunInfo->wout * this->c0Out + | 97 | this->doIdxStart * this->c1Out * this->conv3dRunInfo->hout * this->conv3dRunInfo->wout * this->c0Out + |
| 93 | this->nIdxStart * this->conv3dRunInfo->hout * this->conv3dRunInfo->wout + | 98 | this->nIdxStart * this->conv3dRunInfo->hout * this->conv3dRunInfo->wout + |
| 94 | this->mIdxStart * this->conv3dRunInfo->wout * this->c0Out; | 99 | this->mIdxStart * this->conv3dRunInfo->wout * this->c0Out; |
| 100 | + if constexpr (CONV_CFG::quantType == static_cast<int8_t>(QuantType::PER_CHANNEL_NO_OFFSET)) { | ||
| 101 | + outputStartAddr = this->batchIdxStart * this->outputOneBatchSize + | ||
| 102 | + this->nIdxStart * this->conv3dRunInfo->dout * this->conv3dRunInfo->hout * this->conv3dRunInfo->wout + | ||
| 103 | + this->doIdxStart * this->conv3dRunInfo->hout * this->conv3dRunInfo->wout + | ||
| 104 | + this->mIdxStart * this->conv3dRunInfo->wout; | ||
| 105 | + } | ||
| 95 | ASC_OP_LOGD("[Conv3DV2HwMode] fmStartAddr %d weightStartAddr %d outputStartAddr %d.\n", | 106 | ASC_OP_LOGD("[Conv3DV2HwMode] fmStartAddr %d weightStartAddr %d outputStartAddr %d.\n", |
| 96 | fmStartAddr, | 107 | fmStartAddr, |
| 97 | weightStartAddr, | 108 | weightStartAddr, |
| @@ -35,6 +35,7 @@ constexpr uint64_t PAD_IDX_B = 3; | |||
| 35 | constexpr uint64_t PAD_IDX_L = 0; | 35 | constexpr uint64_t PAD_IDX_L = 0; |
| 36 | constexpr uint64_t PAD_IDX_R = 1; | 36 | constexpr uint64_t PAD_IDX_R = 1; |
| 37 | constexpr uint64_t MAX_PAD_R = 255; | 37 | constexpr uint64_t MAX_PAD_R = 255; |
| 38 | +constexpr uint64_t MAX_VEC_LEN = 255; | ||
| 38 | constexpr uint64_t FMAP_BATCH_DIM = 0; | 39 | constexpr uint64_t FMAP_BATCH_DIM = 0; |
| 39 | constexpr uint64_t FMAP_CIN_DIM = 1; | 40 | constexpr uint64_t FMAP_CIN_DIM = 1; |
| 40 | constexpr uint64_t FMAP_H_DIM = 2; | 41 | constexpr uint64_t FMAP_H_DIM = 2; |
| @@ -43,6 +43,11 @@ public: | |||
| 43 | currentNL0_ = n; | 43 | currentNL0_ = n; |
| 44 | } | 44 | } |
| 45 | 45 | ||
| 46 | + __aicore__ inline void SetVecN(uint64_t n) | ||
| 47 | + { | ||
| 48 | + currentVecNL0_ = n; | ||
| 49 | + } | ||
| 50 | + | ||
| 46 | __aicore__ inline void SetFixpipeIntriParams(FixpipeParamsV220 &intriParams) | 51 | __aicore__ inline void SetFixpipeIntriParams(FixpipeParamsV220 &intriParams) |
| 47 | { | 52 | { |
| 48 | if (self_->ctx.nBL1Iter == self_->ctx.maxNBL1Iter && self_->ctx.nBL0Iter == self_->ctx.maxNL0Iter) { | 53 | if (self_->ctx.nBL1Iter == self_->ctx.maxNBL1Iter && self_->ctx.nBL0Iter == self_->ctx.maxNL0Iter) { |
| @@ -87,7 +92,7 @@ public: | |||
| 87 | { | 92 | { |
| 88 | FixpipeParamsV220 intriParams; | 93 | FixpipeParamsV220 intriParams; |
| 89 | SetFixpipeIntriParams(intriParams); | 94 | SetFixpipeIntriParams(intriParams); |
| 90 | - uint64_t offset = CalcFixpipeOffset(0, 0); | 95 | + uint64_t offset = CalcFixpipeOffset(); |
| 91 | ASC_OP_LOGD("[CopyOut] offset %d.\n", offset); | 96 | ASC_OP_LOGD("[CopyOut] offset %d.\n", offset); |
| 92 | if constexpr (!(AscendC::IsSameType<typename Intf::L0cT, int32_t>::value && | 97 | if constexpr (!(AscendC::IsSameType<typename Intf::L0cT, int32_t>::value && |
| 93 | AscendC::IsSameType<typename Intf::OutputT, bfloat16_t>::value)) { | 98 | AscendC::IsSameType<typename Intf::OutputT, bfloat16_t>::value)) { |
| @@ -105,35 +110,26 @@ public: | |||
| 105 | Fixpipe<typename Intf::L0cT, typename Intf::L0cT, CFG_NZ>(output[offset], self_->ctx.cl0, intriParams); | 110 | Fixpipe<typename Intf::L0cT, typename Intf::L0cT, CFG_NZ>(output[offset], self_->ctx.cl0, intriParams); |
| 106 | } | 111 | } |
| 107 | 112 | ||
| 108 | - __aicore__ inline void CopyUBOut(const GlobalTensor<typename Intf::OutputT> &output, uint32_t mIter, uint32_t nIter, uint32_t mLen, uint32_t nLen, | 113 | + __aicore__ inline void CopyUBOut(const GlobalTensor<typename Intf::OutputT> &output, uint32_t mIter, uint32_t nIter, |
| 109 | - const LocalTensor<typename Intf::OutputT> &ubOut) | 114 | + uint32_t mLen, uint32_t nLen, const LocalTensor<typename Intf::OutputT> &ubOut) |
| 110 | { | 115 | { |
| 111 | - uint64_t offset = CalcFixpipeOffset(mIter * self_->ctx.conv3dTiling->mUB + self_->ctx.outMoffset, | 116 | + uint64_t offset = CalcQuantFixpipeOffset(mIter * self_->ctx.conv3dTiling->mUB + self_->ctx.outMoffset, |
| 112 | - nIter * self_->ctx.conv3dTiling->nUB + self_->ctx.outNoffset); | 117 | + nIter * self_->ctx.conv3dTiling->nUB + self_->ctx.outNoffset); |
| 113 | 118 | ||
| 114 | - DataCopyParams copyParams; | 119 | + DataCopyExtParams copyParams(nLen, |
| 115 | - | 120 | + mLen * sizeof(typename Intf::OutputT), |
| 116 | - int32_t blockCount = nLen / BLOCK_L0_N; | 121 | + 0, |
| 117 | - uint64_t dstStride = valueHoWo_ - mLen; | 122 | + (self_->ctx.orgDo * valueHoWo_ - mLen) * sizeof(typename Intf::OutputT), |
| 118 | - | 123 | + 0); |
| 119 | - copyParams.blockLen = mLen; | 124 | + if (UINT32_MAX >= copyParams.dstStride) { |
| 120 | - copyParams.srcStride = 0; | 125 | + DataCopyPad(output[offset], ubOut, copyParams); |
| 121 | - if (dstStride <= MAX_UINT16) { | ||
| 122 | - copyParams.dstStride = dstStride; | ||
| 123 | - copyParams.blockCount = blockCount; | ||
| 124 | - DataCopy(output[offset], ubOut, copyParams); | ||
| 125 | } else { | 126 | } else { |
| 126 | - copyParams.dstStride = 0; | ||
| 127 | copyParams.blockCount = 1; | 127 | copyParams.blockCount = 1; |
| 128 | - uint64_t gmoffset = offset; | 128 | + copyParams.dstStride = 0; |
| 129 | - uint64_t localoffset = 0; | 129 | + uint32_t mLenAlign = AlignB(mLen, BLOCK_L0_M); |
| 130 | - uint64_t blockElm = mLen * BLOCK_L0_N; | 130 | + uint64_t orgDoHoWo = self_->ctx.orgDo * valueHoWo_; |
| 131 | - int iter = 0; | 131 | + for (int i = 0; i < nLen; i++) { |
| 132 | - while (iter < blockCount) { | 132 | + DataCopyPad(output[offset + i * orgDoHoWo], ubOut[i * mLenAlign], copyParams); |
| 133 | - DataCopy(output[gmoffset], ubOut[localoffset], copyParams); | ||
| 134 | - iter++; | ||
| 135 | - localoffset += blockElm; | ||
| 136 | - gmoffset += valueHoWo_ * BLOCK_L0_N; | ||
| 137 | } | 133 | } |
| 138 | } | 134 | } |
| 139 | } | 135 | } |
| @@ -146,17 +142,37 @@ public: | |||
| 146 | nSize = intriParams.nSize; | 142 | nSize = intriParams.nSize; |
| 147 | } | 143 | } |
| 148 | 144 | ||
| 149 | - __aicore__ inline uint64_t GetChannelOffset(uint64_t noffset) | 145 | + __aicore__ inline void GetCurNSize(uint64_t &nCurSize) |
| 150 | { | 146 | { |
| 151 | - uint64_t offsetCout = tilingNBL1_ * self_->ctx.nBL1Iter + self_->ctx.conv3dTiling->nL0 * self_->ctx.nBL0Iter + noffset; | 147 | + nCurSize = currentVecNL0_; |
| 148 | + } | ||
| 149 | + | ||
| 150 | + __aicore__ inline uint64_t GetChannelOffset(uint64_t nOffset) | ||
| 151 | + { | ||
| 152 | + uint64_t offsetCout = tilingNBL1_ * self_->ctx.nBL1Iter + | ||
| 153 | + self_->ctx.conv3dTiling->nL0 * self_->ctx.nBL0Iter + | ||
| 154 | + nOffset; | ||
| 152 | return offsetCout; | 155 | return offsetCout; |
| 153 | } | 156 | } |
| 154 | 157 | ||
| 155 | private: | 158 | private: |
| 156 | - __aicore__ inline uint64_t CalcFixpipeOffset(uint64_t mOffset, uint64_t nOffset) | 159 | + __aicore__ inline uint64_t CalcQuantFixpipeOffset(uint64_t mOffset, uint64_t nOffset) |
| 157 | { | 160 | { |
| 158 | - uint64_t offsetCout = tilingNBL1_ * self_->ctx.nBL1Iter + self_->ctx.conv3dTiling->nL0 * self_->ctx.nBL0Iter + nOffset; | 161 | + uint64_t offsetCout = tilingNBL1_ * self_->ctx.nBL1Iter + |
| 159 | - uint64_t offsetM = tilingMAL1_ * self_->ctx.mAL1Iter + self_->ctx.conv3dTiling->mL0 * self_->ctx.mAL0Iter + mOffset; | 162 | + self_->ctx.conv3dTiling->nL0 * self_->ctx.nBL0Iter + |
| 163 | + nOffset; | ||
| 164 | + uint64_t offsetM = tilingMAL1_ * self_->ctx.mAL1Iter + | ||
| 165 | + self_->ctx.conv3dTiling->mL0 * self_->ctx.mAL0Iter + | ||
| 166 | + mOffset; | ||
| 167 | + // 当前每次只出一个dout | ||
| 168 | + uint64_t offsetDout = self_->ctx.dOutIter; | ||
| 169 | + return offsetCout * self_->ctx.orgDo * valueHoWo_ + offsetDout * valueHoWo_ + | ||
| 170 | + self_->ctx.hoL1Iter * self_->ctx.orgWo + offsetM; | ||
| 171 | + } | ||
| 172 | + __aicore__ inline uint64_t CalcFixpipeOffset() | ||
| 173 | + { | ||
| 174 | + uint64_t offsetCout = tilingNBL1_ * self_->ctx.nBL1Iter + self_->ctx.conv3dTiling->nL0 * self_->ctx.nBL0Iter; | ||
| 175 | + uint64_t offsetM = tilingMAL1_ * self_->ctx.mAL1Iter + self_->ctx.conv3dTiling->mL0 * self_->ctx.mAL0Iter; | ||
| 160 | // 当前每次只出一个dout | 176 | // 当前每次只出一个dout |
| 161 | uint64_t offsetDout = self_->ctx.dOutIter; | 177 | uint64_t offsetDout = self_->ctx.dOutIter; |
| 162 | return offsetDout * self_->ctx.orgCoAlignK0 * valueHoWo_ + offsetCout * valueHoWo_ + | 178 | return offsetDout * self_->ctx.orgCoAlignK0 * valueHoWo_ + offsetCout * valueHoWo_ + |
| @@ -195,6 +211,7 @@ private: | |||
| 195 | uint64_t valueHoWo_ = 0; | 211 | uint64_t valueHoWo_ = 0; |
| 196 | uint64_t currentML0_ = 0; | 212 | uint64_t currentML0_ = 0; |
| 197 | uint64_t currentNL0_ = 0; | 213 | uint64_t currentNL0_ = 0; |
| 214 | + uint64_t currentVecNL0_ = 0; | ||
| 198 | }; | 215 | }; |
| 199 | 216 | ||
| 200 | }; // namespace Conv3dFunc | 217 | }; // namespace Conv3dFunc |
| @@ -317,6 +317,11 @@ public: | |||
| 317 | // impl for pointwise | 317 | // impl for pointwise |
| 318 | } | 318 | } |
| 319 | 319 | ||
| 320 | + __aicore__ inline void GetCurNSize(uint64_t &nCurSize) | ||
| 321 | + { | ||
| 322 | + // impl for pointwise | ||
| 323 | + } | ||
| 324 | + | ||
| 320 | __aicore__ inline uint64_t GetChannelOffset(uint64_t noffset) | 325 | __aicore__ inline uint64_t GetChannelOffset(uint64_t noffset) |
| 321 | { | 326 | { |
| 322 | return 0; | 327 | return 0; |
| @@ -202,8 +202,8 @@ public: | |||
| 202 | } | 202 | } |
| 203 | } | 203 | } |
| 204 | ASC_OP_LOGD( | 204 | ASC_OP_LOGD( |
| 205 | - "[Mad] mmadParams.cmatrixInitVal %d, mmadParams.cmatrixSource %d, mmadParams.isBias %d, mmadParams.k %d, " | 205 | + "[Mad] mmadParams.cmatrixInitVal %d, mmadParams.cmatrixSource %d, mmadParams.isBias %d," |
| 206 | - "mmadParams.n %d, mmadParams.m %d.\n", | 206 | + "mmadParams.k %d, mmadParams.n %d, mmadParams.m %d.\n", |
| 207 | mmadParams.cmatrixInitVal, | 207 | mmadParams.cmatrixInitVal, |
| 208 | mmadParams.cmatrixSource, | 208 | mmadParams.cmatrixSource, |
| 209 | mmadParams.isBias, | 209 | mmadParams.isBias, |
| @@ -240,6 +240,11 @@ public: | |||
| 240 | currentNL0_ = n; | 240 | currentNL0_ = n; |
| 241 | } | 241 | } |
| 242 | 242 | ||
| 243 | + __aicore__ inline void SetVecN(uint64_t n) | ||
| 244 | + { | ||
| 245 | + currentVecNL0_ = n; | ||
| 246 | + } | ||
| 247 | + | ||
| 243 | __aicore__ inline void SetFixpipeIntriParams(FixpipeParamsV220 &intriParams) | 248 | __aicore__ inline void SetFixpipeIntriParams(FixpipeParamsV220 &intriParams) |
| 244 | { | 249 | { |
| 245 | if (self_->ctx.nBL1Iter == self_->ctx.maxNBL1Iter && self_->ctx.nBL0Iter == self_->ctx.maxNL0Iter) { | 250 | if (self_->ctx.nBL1Iter == self_->ctx.maxNBL1Iter && self_->ctx.nBL0Iter == self_->ctx.maxNL0Iter) { |
| @@ -285,7 +290,7 @@ public: | |||
| 285 | { | 290 | { |
| 286 | FixpipeParamsV220 intriParams; | 291 | FixpipeParamsV220 intriParams; |
| 287 | SetFixpipeIntriParams(intriParams); | 292 | SetFixpipeIntriParams(intriParams); |
| 288 | - uint64_t offset = CalcFixpipeOffset(0, 0); | 293 | + uint64_t offset = CalcFixpipeOffset(); |
| 289 | ASC_OP_LOGD("[CopyOut] offset %d.\n", offset); | 294 | ASC_OP_LOGD("[CopyOut] offset %d.\n", offset); |
| 290 | if constexpr (!(AscendC::IsSameType<typename Intf::L0cT, int32_t>::value && | 295 | if constexpr (!(AscendC::IsSameType<typename Intf::L0cT, int32_t>::value && |
| 291 | AscendC::IsSameType<typename Intf::OutputT, bfloat16_t>::value)) { | 296 | AscendC::IsSameType<typename Intf::OutputT, bfloat16_t>::value)) { |
| @@ -304,39 +309,25 @@ public: | |||
| 304 | Fixpipe<typename Intf::L0cT, typename Intf::L0cT, CFG_NZ>(output[offset], self_->ctx.cl0, intriParams); | 309 | Fixpipe<typename Intf::L0cT, typename Intf::L0cT, CFG_NZ>(output[offset], self_->ctx.cl0, intriParams); |
| 305 | } | 310 | } |
| 306 | 311 | ||
| 307 | - __aicore__ inline void CopyUBOut(const GlobalTensor<typename Intf::OutputT> &output, uint32_t mIter, uint32_t nIter, uint32_t m, uint32_t n, | 312 | + __aicore__ inline void CopyUBOut(const GlobalTensor<typename Intf::OutputT> &output, uint32_t mIter, uint32_t nIter, |
| 308 | - const LocalTensor<typename Intf::OutputT> &ubout) | 313 | + uint32_t mLen, uint32_t nLen, const LocalTensor<typename Intf::OutputT> &ubOut) |
| 309 | { | 314 | { |
| 310 | - uint64_t offset = CalcFixpipeOffset(mIter * self_->ctx.conv3dTiling->mUB + self_->ctx.outMoffset, nIter * self_->ctx.conv3dTiling->nUB + self_->ctx.outNoffset); | 315 | + uint64_t offset = CalcQuantFixpipeOffset(mIter * self_->ctx.conv3dTiling->mUB + self_->ctx.outMoffset, |
| 311 | - | 316 | + nIter * self_->ctx.conv3dTiling->nUB + self_->ctx.outNoffset); |
| 312 | - DataCopyParams copyParams; | 317 | + DataCopyExtParams copyParams(nLen, |
| 313 | - | 318 | + mLen * sizeof(typename Intf::OutputT), |
| 314 | - int32_t blockCount = n / BLOCK_L0_N; | 319 | + 0, |
| 315 | - uint64_t dstStride = valueHoWo_ - m; | 320 | + (self_->ctx.orgDo * valueHoWo_ - mLen) * sizeof(typename Intf::OutputT), |
| 316 | - | 321 | + 0); |
| 317 | - copyParams.blockLen = m; | 322 | + if (UINT32_MAX >= copyParams.dstStride) { |
| 318 | - copyParams.srcStride = 0; | 323 | + DataCopyPad(output[offset], ubOut, copyParams); |
| 319 | - if (dstStride <= MAX_UINT16) { | ||
| 320 | - copyParams.dstStride = dstStride; | ||
| 321 | - copyParams.blockCount = blockCount; | ||
| 322 | - DataCopy(output[offset], ubout, copyParams); | ||
| 323 | } else { | 324 | } else { |
| 324 | - copyParams.dstStride = 0; | ||
| 325 | copyParams.blockCount = 1; | 325 | copyParams.blockCount = 1; |
| 326 | - uint64_t gmoffset = offset; | 326 | + copyParams.dstStride = 0; |
| 327 | - uint64_t localoffset = 0; | 327 | + uint32_t mLenAlign = AlignB(mLen, BLOCK_L0_M); |
| 328 | - uint64_t blockElm = m * BLOCK_L0_N; | 328 | + uint64_t orgDoHoWo = self_->ctx.orgDo * valueHoWo_; |
| 329 | - int iter = 0; | 329 | + for (int i = 0; i < nLen; i++) { |
| 330 | - | 330 | + DataCopyPad(output[offset + i * orgDoHoWo], ubOut[i * mLenAlign], copyParams); |
| 331 | - while (true) { | ||
| 332 | - DataCopy(output[gmoffset], ubout[localoffset], copyParams); | ||
| 333 | - iter++; | ||
| 334 | - if (iter < blockCount) { | ||
| 335 | - localoffset += blockElm; | ||
| 336 | - gmoffset += valueHoWo_ * BLOCK_L0_N; | ||
| 337 | - } else { | ||
| 338 | - break; | ||
| 339 | - } | ||
| 340 | } | 331 | } |
| 341 | } | 332 | } |
| 342 | } | 333 | } |
| @@ -349,17 +340,36 @@ public: | |||
| 349 | nSize = intriParams.nSize; | 340 | nSize = intriParams.nSize; |
| 350 | } | 341 | } |
| 351 | 342 | ||
| 352 | - __aicore__ inline uint64_t GetChannelOffset(uint64_t noffset) | 343 | + __aicore__ inline void GetCurNSize(uint64_t &nCurSize) |
| 353 | { | 344 | { |
| 354 | - uint64_t offsetCout = tilingNBL1_ * self_->ctx.nBL1Iter + self_->ctx.conv3dTiling->nL0 * self_->ctx.nBL0Iter + noffset; | 345 | + nCurSize = currentVecNL0_; |
| 346 | + } | ||
| 347 | + | ||
| 348 | + __aicore__ inline uint64_t GetChannelOffset(uint64_t nOffset) | ||
| 349 | + { | ||
| 350 | + uint64_t offsetCout = tilingNBL1_ * self_->ctx.nBL1Iter + | ||
| 351 | + self_->ctx.conv3dTiling->nL0 * self_->ctx.nBL0Iter + | ||
| 352 | + nOffset; | ||
| 355 | return offsetCout; | 353 | return offsetCout; |
| 356 | } | 354 | } |
| 357 | 355 | ||
| 358 | private: | 356 | private: |
| 359 | - __aicore__ inline uint64_t CalcFixpipeOffset(uint64_t moffset, uint64_t noffset) | 357 | + __aicore__ inline uint64_t CalcQuantFixpipeOffset(uint64_t mOffset, uint64_t nOffset) |
| 360 | { | 358 | { |
| 361 | - uint64_t offsetCout = tilingNBL1_ * self_->ctx.nBL1Iter + self_->ctx.conv3dTiling->nL0 * self_->ctx.nBL0Iter + noffset; | 359 | + uint64_t offsetCout = tilingNBL1_ * self_->ctx.nBL1Iter + |
| 362 | - uint64_t offsetM = tilingMAL1_ * self_->ctx.mAL1Iter + self_->ctx.conv3dTiling->mL0 * self_->ctx.mAL0Iter + moffset; | 360 | + self_->ctx.conv3dTiling->nL0 * self_->ctx.nBL0Iter + |
| 361 | + nOffset; | ||
| 362 | + uint64_t offsetM = tilingMAL1_ * self_->ctx.mAL1Iter + | ||
| 363 | + self_->ctx.conv3dTiling->mL0 * self_->ctx.mAL0Iter + | ||
| 364 | + mOffset; | ||
| 365 | + // 当前每次只出一个dout | ||
| 366 | + uint64_t offsetDout = self_->ctx.dOutIter; | ||
| 367 | + return offsetCout * self_->ctx.orgDo * valueHoWo_ + offsetDout * valueHoWo_ + offsetM; | ||
| 368 | + } | ||
| 369 | + __aicore__ inline uint64_t CalcFixpipeOffset() | ||
| 370 | + { | ||
| 371 | + uint64_t offsetCout = tilingNBL1_ * self_->ctx.nBL1Iter + self_->ctx.conv3dTiling->nL0 * self_->ctx.nBL0Iter; | ||
| 372 | + uint64_t offsetM = tilingMAL1_ * self_->ctx.mAL1Iter + self_->ctx.conv3dTiling->mL0 * self_->ctx.mAL0Iter; | ||
| 363 | // 当前每次只出一个dout | 373 | // 当前每次只出一个dout |
| 364 | uint64_t offsetDout = self_->ctx.dOutIter; | 374 | uint64_t offsetDout = self_->ctx.dOutIter; |
| 365 | if constexpr (Intf::groupConvType) { | 375 | if constexpr (Intf::groupConvType) { |
| @@ -404,6 +414,7 @@ private: | |||
| 404 | uint64_t valueHoWo_ = 0; | 414 | uint64_t valueHoWo_ = 0; |
| 405 | uint64_t currentML0_ = 0; | 415 | uint64_t currentML0_ = 0; |
| 406 | uint64_t currentNL0_ = 0; | 416 | uint64_t currentNL0_ = 0; |
| 417 | + uint64_t currentVecNL0_ = 0; | ||
| 407 | }; | 418 | }; |
| 408 | 419 | ||
| 409 | }; // namespace Conv3dFunc | 420 | }; // namespace Conv3dFunc |
| @@ -1421,11 +1421,30 @@ TEST(TestConv3dTilingEngine, CheckInputFormat_ScaleFormat_Valid) | |||
| 1421 | // Regular mode with valid formats | 1421 | // Regular mode with valid formats |
| 1422 | std::vector<int64_t> weightShape = {16, 1, 3, 3, 3}; | 1422 | std::vector<int64_t> weightShape = {16, 1, 3, 3, 3}; |
| 1423 | engine.SetOrgWeightShape(weightShape); | 1423 | engine.SetOrgWeightShape(weightShape); |
| 1424 | - engine.SetFormat(ConvFormat::NDC1HWC0, ConvFormat::FRACTAL_Z_3D, ConvFormat::NDC1HWC0); | 1424 | + engine.SetFormat(ConvFormat::NDC1HWC0, ConvFormat::FRACTAL_Z_3D, ConvFormat::NCDHW); |
| 1425 | 1425 | ||
| 1426 | EXPECT_TRUE(engine.CheckInputFormat()); | 1426 | EXPECT_TRUE(engine.CheckInputFormat()); |
| 1427 | } | 1427 | } |
| 1428 | 1428 | ||
| 1429 | +TEST(TestConv3dTilingEngine, CheckInputFormat_Quant_OutFormat_Invalid) | ||
| 1430 | +{ | ||
| 1431 | + using Conv3dApiTiling::ConvFormat; | ||
| 1432 | + using Conv3dApiTiling::ConvDtype; | ||
| 1433 | + | ||
| 1434 | + Conv3dTilingEngine engine; | ||
| 1435 | + InitSimpleConv3dEngine(engine); | ||
| 1436 | + | ||
| 1437 | + // Setup with scale | ||
| 1438 | + engine.SetScale(true, ConvDtype::FLOAT32); | ||
| 1439 | + | ||
| 1440 | + // Regular mode with valid main formats | ||
| 1441 | + std::vector<int64_t> weightShape = {16, 1, 3, 3, 3}; | ||
| 1442 | + engine.SetOrgWeightShape(weightShape); | ||
| 1443 | + engine.SetFormat(ConvFormat::NDC1HWC0, ConvFormat::FRACTAL_Z_3D, ConvFormat::NDC1HWC0); | ||
| 1444 | + | ||
| 1445 | + EXPECT_FALSE(engine.CheckInputFormat()); | ||
| 1446 | +} | ||
| 1447 | + | ||
| 1429 | TEST(TestConv3dTilingEngine, CheckInputFormat_ScaleFormat_Invalid) | 1448 | TEST(TestConv3dTilingEngine, CheckInputFormat_ScaleFormat_Invalid) |
| 1430 | { | 1449 | { |
| 1431 | using Conv3dApiTiling::ConvFormat; | 1450 | using Conv3dApiTiling::ConvFormat; |
| @@ -1443,7 +1462,7 @@ TEST(TestConv3dTilingEngine, CheckInputFormat_ScaleFormat_Invalid) | |||
| 1443 | // Regular mode with valid main formats | 1462 | // Regular mode with valid main formats |
| 1444 | std::vector<int64_t> weightShape = {16, 1, 3, 3, 3}; | 1463 | std::vector<int64_t> weightShape = {16, 1, 3, 3, 3}; |
| 1445 | engine.SetOrgWeightShape(weightShape); | 1464 | engine.SetOrgWeightShape(weightShape); |
| 1446 | - engine.SetFormat(ConvFormat::NDC1HWC0, ConvFormat::FRACTAL_Z_3D, ConvFormat::NDC1HWC0); | 1465 | + engine.SetFormat(ConvFormat::NDC1HWC0, ConvFormat::FRACTAL_Z_3D, ConvFormat::NCDHW); |
| 1447 | 1466 | ||
| 1448 | EXPECT_FALSE(engine.CheckInputFormat()); | 1467 | EXPECT_FALSE(engine.CheckInputFormat()); |
| 1449 | } | 1468 | } |
| @@ -1116,8 +1116,8 @@ public: | |||
| 1116 | return PreProcessV2(); | 1116 | return PreProcessV2(); |
| 1117 | } | 1117 | } |
| 1118 | 1118 | ||
| 1119 | - REG_L0_FUNCTION_BY_OPTYPE(l0Functions, QuantConv3d6HdInt8To6HdBf16, "QuantConv3d6HdInt8To6HdBf16"); | 1119 | + REG_L0_FUNCTION_BY_OPTYPE(l0Functions, QuantConv3d6HdInt8ToNCDHWBf16, "QuantConv3d6HdInt8ToNCDHWBf16"); |
| 1120 | - REG_L0_FUNCTION_BY_OPTYPE(l0Functions, QuantConv3d6HdInt8To6HdFp16, "QuantConv3d6HdInt8To6HdFp16"); | 1120 | + REG_L0_FUNCTION_BY_OPTYPE(l0Functions, QuantConv3d6HdInt8ToNCDHWFp16, "QuantConv3d6HdInt8ToNCDHWFp16"); |
| 1121 | outputDtype = output->GetDataType(); | 1121 | outputDtype = output->GetDataType(); |
| 1122 | auto retContiguous = ContiguousPreProcess(input, weight, scale, bias, executor); | 1122 | auto retContiguous = ContiguousPreProcess(input, weight, scale, bias, executor); |
| 1123 | if (retContiguous != ACLNN_SUCCESS) { | 1123 | if (retContiguous != ACLNN_SUCCESS) { |
| @@ -1153,11 +1153,11 @@ public: | |||
| 1153 | } | 1153 | } |
| 1154 | } else { | 1154 | } else { |
| 1155 | if (outputDtype == DataType::DT_FLOAT16) { | 1155 | if (outputDtype == DataType::DT_FLOAT16) { |
| 1156 | - quantConvOut = FUNCTION_CALL_BY_OPTYPE(l0Functions, "QuantConv3d6HdInt8To6HdFp16", input, weight, bias, | 1156 | + quantConvOut = FUNCTION_CALL_BY_OPTYPE(l0Functions, "QuantConv3d6HdInt8ToNCDHWFp16", input, weight, bias, |
| 1157 | scale, offset, stride, padding, dilation, groups, offsetx, roundMode, outputDtype, | 1157 | scale, offset, stride, padding, dilation, groups, offsetx, roundMode, outputDtype, |
| 1158 | outputFormat, executor); | 1158 | outputFormat, executor); |
| 1159 | } else if (outputDtype == DataType::DT_BF16) { | 1159 | } else if (outputDtype == DataType::DT_BF16) { |
| 1160 | - quantConvOut = FUNCTION_CALL_BY_OPTYPE(l0Functions, "QuantConv3d6HdInt8To6HdBf16", input, weight, bias, | 1160 | + quantConvOut = FUNCTION_CALL_BY_OPTYPE(l0Functions, "QuantConv3d6HdInt8ToNCDHWBf16", input, weight, bias, |
| 1161 | scale, offset, stride, padding, dilation, groups, offsetx, roundMode, outputDtype, | 1161 | scale, offset, stride, padding, dilation, groups, offsetx, roundMode, outputDtype, |
| 1162 | outputFormat, executor); | 1162 | outputFormat, executor); |
| 1163 | } | 1163 | } |
| @@ -1173,10 +1173,7 @@ public: | |||
| 1173 | aclnnStatus PostProcess() override | 1173 | aclnnStatus PostProcess() override |
| 1174 | { | 1174 | { |
| 1175 | const aclTensor* resConvOut = quantConvOut; | 1175 | const aclTensor* resConvOut = quantConvOut; |
| 1176 | - if (!IsSocSupportND()) { | 1176 | + if (IsSocSupportND()) { |
| 1177 | - resConvOut = l0op::TransData(quantConvOut, output->GetStorageFormat(), groups, executor); | ||
| 1178 | - CHECK_RET(resConvOut != nullptr, ACLNN_ERR_INNER_NULLPTR); | ||
| 1179 | - } else { | ||
| 1180 | bool isConv3DQuant = input->GetViewShape().GetDimNum() == CONV_3D_INPUT_DIM && | 1177 | bool isConv3DQuant = input->GetViewShape().GetDimNum() == CONV_3D_INPUT_DIM && |
| 1181 | input->GetDataType() == DataType::DT_INT8 && | 1178 | input->GetDataType() == DataType::DT_INT8 && |
| 1182 | scale->GetDataType() == DataType::DT_FLOAT; | 1179 | scale->GetDataType() == DataType::DT_FLOAT; |
| @@ -75,15 +75,15 @@ const aclTensor *Conv3dv2L0Func(const aclTensor *input, const aclTensor *weight, | |||
| 75 | const aclIntArray *stride, const aclIntArray *padding, const aclIntArray *dilation, | 75 | const aclIntArray *stride, const aclIntArray *padding, const aclIntArray *dilation, |
| 76 | int groups, bool useHf32, aclOpExecutor *executor); | 76 | int groups, bool useHf32, aclOpExecutor *executor); |
| 77 | 77 | ||
| 78 | -const aclTensor *QuantConv3d6HdInt8To6HdBf16(const aclTensor *input, const aclTensor *weight, const aclTensor *bias, | 78 | +const aclTensor *QuantConv3d6HdInt8ToNCDHWBf16(const aclTensor *input, const aclTensor *weight, const aclTensor *bias, |
| 79 | - const aclTensor *scale, const aclTensor *offset, const aclIntArray *stride, | 79 | + const aclTensor *scale, const aclTensor *offset, const aclIntArray *stride, |
| 80 | - const aclIntArray *padding, const aclIntArray *dilation, int groups, | 80 | + const aclIntArray *padding, const aclIntArray *dilation, int groups, |
| 81 | - aclOpExecutor *executor); | 81 | + aclOpExecutor *executor); |
| 82 | 82 | ||
| 83 | -const aclTensor *QuantConv3d6HdInt8To6HdFp16(const aclTensor *input, const aclTensor *weight, const aclTensor *bias, | 83 | +const aclTensor *QuantConv3d6HdInt8ToNCDHWFp16(const aclTensor *input, const aclTensor *weight, const aclTensor *bias, |
| 84 | - const aclTensor *scale, const aclTensor *offset, const aclIntArray *stride, | 84 | + const aclTensor *scale, const aclTensor *offset, const aclIntArray *stride, |
| 85 | - const aclIntArray *padding, const aclIntArray *dilation, int groups, | 85 | + const aclIntArray *padding, const aclIntArray *dilation, int groups, |
| 86 | - aclOpExecutor *executor); | 86 | + aclOpExecutor *executor); |
| 87 | 87 | ||
| 88 | const aclTensor *Conv3d6HdBf16(const aclTensor *input, const aclTensor *weight, const aclTensor *bias, | 88 | const aclTensor *Conv3d6HdBf16(const aclTensor *input, const aclTensor *weight, const aclTensor *bias, |
| 89 | const aclIntArray *stride, const aclIntArray *padding, const aclIntArray *dilation, | 89 | const aclIntArray *stride, const aclIntArray *padding, const aclIntArray *dilation, |
需要确认是否直接修改原有json而不是新增json项