已合并
【feature】: Transpose融合ub模板改为Compact模式搬运优化性能 #1967
liyuewei创建于 9月4日
【feature】: Transpose融合ub模板改为Compact模式搬运优化性能 #1967
已合并
共 8 个文件变更+79-40
| @@ -11,6 +11,7 @@ | |||
| 11 | #include "reg_api_call_utils.h" | 11 | #include "reg_api_call_utils.h" |
| 12 | #include "graph/symbolizer/symbolic_utils.h" | 12 | #include "graph/symbolizer/symbolic_utils.h" |
| 13 | #include "tensor_layout_utils.h" | 13 | #include "tensor_layout_utils.h" |
| 14 | +#include "ascir_ops.h" | ||
| 14 | 15 | ||
| 15 | namespace { | 16 | namespace { |
| 16 | constexpr size_t kDmaMaxLen = 2U; | 17 | constexpr size_t kDmaMaxLen = 2U; |
| @@ -99,7 +100,7 @@ void SetLoopModeParamsExpr(const DataCopyParams &data_copy_param, LoopModeParams | |||
| 99 | } | 100 | } |
| 100 | } | 101 | } |
| 101 | 102 | ||
| 102 | -std::string GetPaddingMode(const Tensor &ub_tensor, const DataCopyParams &data_copy_param) { | 103 | +std::string GetPaddingMode(const Tensor &ub_tensor, const DataCopyParams &data_copy_param, bool has_transpose) { |
| 103 | af::AscTensorAttr attr; | 104 | af::AscTensorAttr attr; |
| 104 | attr.axis = ub_tensor.axis; | 105 | attr.axis = ub_tensor.axis; |
| 105 | attr.repeats = ub_tensor.axis_size; | 106 | attr.repeats = ub_tensor.axis_size; |
| @@ -118,7 +119,7 @@ std::string GetPaddingMode(const Tensor &ub_tensor, const DataCopyParams &data_c | |||
| 118 | auto no_multiple_discontinuities = !info.has_multiple_discontinuities; | 119 | auto no_multiple_discontinuities = !info.has_multiple_discontinuities; |
| 119 | GELOGD("GetPaddingMode conditions: analyze_success=%d, no_multiple_discontinuities=%d", analyze_success, | 120 | GELOGD("GetPaddingMode conditions: analyze_success=%d, no_multiple_discontinuities=%d", analyze_success, |
| 120 | no_multiple_discontinuities); | 121 | no_multiple_discontinuities); |
| 121 | - if (analyze_success && no_multiple_discontinuities && status) { | 122 | + if (((analyze_success && no_multiple_discontinuities) || has_transpose) && status) { |
| 122 | return kCompactPddingMode; | 123 | return kCompactPddingMode; |
| 123 | } | 124 | } |
| 124 | return kNormalPddingMode; | 125 | return kNormalPddingMode; |
| @@ -169,7 +170,7 @@ void CreateBaseEnhanceDmaCall(const Tensor &input, const Tensor &output, const D | |||
| 169 | 170 | ||
| 170 | void CreateEnhanceDmaCall(const TPipe &tpipe, const Tensor &input, const Tensor &output, const string &gm_offset, | 171 | void CreateEnhanceDmaCall(const TPipe &tpipe, const Tensor &input, const Tensor &output, const string &gm_offset, |
| 171 | const DataCopyParams &data_copy_param, const ascir::SizeExpr &offset, std::stringstream &ss, | 172 | const DataCopyParams &data_copy_param, const ascir::SizeExpr &offset, std::stringstream &ss, |
| 172 | - bool copy_in) { | 173 | + bool copy_in, bool has_transpose) { |
| 173 | size_t total_len = data_copy_param.repeats.size(); | 174 | size_t total_len = data_copy_param.repeats.size(); |
| 174 | DmaParams dma_param; | 175 | DmaParams dma_param; |
| 175 | SetDmaParams(tpipe, data_copy_param, dma_param, copy_in); | 176 | SetDmaParams(tpipe, data_copy_param, dma_param, copy_in); |
| @@ -177,7 +178,7 @@ void CreateEnhanceDmaCall(const TPipe &tpipe, const Tensor &input, const Tensor | |||
| 177 | LoopModeParams loop_mode_param; | 178 | LoopModeParams loop_mode_param; |
| 178 | SetLoopModeParams(tpipe, data_copy_param, loop_mode_param, copy_in); | 179 | SetLoopModeParams(tpipe, data_copy_param, loop_mode_param, copy_in); |
| 179 | const Tensor &ub_tensor = copy_in ? output : input; | 180 | const Tensor &ub_tensor = copy_in ? output : input; |
| 180 | - std::string padding_mode = GetPaddingMode(ub_tensor, data_copy_param); | 181 | + std::string padding_mode = GetPaddingMode(ub_tensor, data_copy_param, has_transpose); |
| 181 | if (total_len <= kDmaMaxLen) { | 182 | if (total_len <= kDmaMaxLen) { |
| 182 | CreateBaseDmaCall(input, output, dma_param, padding_mode, ss, copy_in); | 183 | CreateBaseDmaCall(input, output, dma_param, padding_mode, ss, copy_in); |
| 183 | return; | 184 | return; |
| @@ -204,7 +205,8 @@ void CreateEnhanceDmaCall(const TPipe &tpipe, const Tensor &input, const Tensor | |||
| 204 | } | 205 | } |
| 205 | 206 | ||
| 206 | void SetNddmaParams(const TPipe &tpipe, const DataCopyParams &data_copy_param, NddmaParams &nddma_param, | 207 | void SetNddmaParams(const TPipe &tpipe, const DataCopyParams &data_copy_param, NddmaParams &nddma_param, |
| 207 | - const int64_t &tensor_id, std::stringstream &ss) { | 208 | + const int64_t &tensor_id, std::string padding_mode, std::stringstream &ss) { |
| 209 | + bool is_compact_mode = (padding_mode == kCompactPddingMode); | ||
| 208 | nddma_param.ss_output_dims << "const int64_t output_dims_" << tensor_id << "[5] = {"; | 210 | nddma_param.ss_output_dims << "const int64_t output_dims_" << tensor_id << "[5] = {"; |
| 209 | nddma_param.ss_output_stride << "const int64_t output_stride_" << tensor_id << "[5] = {"; | 211 | nddma_param.ss_output_stride << "const int64_t output_stride_" << tensor_id << "[5] = {"; |
| 210 | nddma_param.ss_input_stride << "const int64_t input_stride_" << tensor_id << "[5] = {"; | 212 | nddma_param.ss_input_stride << "const int64_t input_stride_" << tensor_id << "[5] = {"; |
| @@ -225,12 +227,15 @@ void SetNddmaParams(const TPipe &tpipe, const DataCopyParams &data_copy_param, N | |||
| 225 | while (j > 0UL) { | 227 | while (j > 0UL) { |
| 226 | if (j == 1UL) { | 228 | if (j == 1UL) { |
| 227 | nddma_param.ss_output_dims << tpipe.tiler.ActualSize(data_copy_param.repeats[i]); | 229 | nddma_param.ss_output_dims << tpipe.tiler.ActualSize(data_copy_param.repeats[i]); |
| 228 | - nddma_param.ss_output_stride << tpipe.tiler.Size(data_copy_param.ub_strides[i]); | 230 | + nddma_param.ss_output_stride << (is_compact_mode ? tpipe.tiler.ActualSize(data_copy_param.ub_strides[i]) |
| 231 | + : tpipe.tiler.Size(data_copy_param.ub_strides[i])); | ||
| 229 | nddma_param.ss_input_stride << tpipe.tiler.Size(data_copy_param.gm_strides[i]); | 232 | nddma_param.ss_input_stride << tpipe.tiler.Size(data_copy_param.gm_strides[i]); |
| 230 | break; | 233 | break; |
| 231 | } | 234 | } |
| 232 | nddma_param.ss_output_dims << tpipe.tiler.ActualSize(data_copy_param.repeats[i]) << ", "; | 235 | nddma_param.ss_output_dims << tpipe.tiler.ActualSize(data_copy_param.repeats[i]) << ", "; |
| 233 | - nddma_param.ss_output_stride << tpipe.tiler.Size(data_copy_param.ub_strides[i]) << ", "; | 236 | + nddma_param.ss_output_stride << (is_compact_mode ? tpipe.tiler.ActualSize(data_copy_param.ub_strides[i]) |
| 237 | + : tpipe.tiler.Size(data_copy_param.ub_strides[i])) | ||
| 238 | + << ", "; | ||
| 234 | nddma_param.ss_input_stride << tpipe.tiler.Size(data_copy_param.gm_strides[i]) << ", "; | 239 | nddma_param.ss_input_stride << tpipe.tiler.Size(data_copy_param.gm_strides[i]) << ", "; |
| 235 | i++; | 240 | i++; |
| 236 | j--; | 241 | j--; |
| @@ -253,7 +258,8 @@ void CreateNddmaCall(const TPipe &tpipe, const Tensor &input, const Tensor &outp | |||
| 253 | const std::string ub_inner_offset = CalcInnerOffset(tpipe, ub_stride); | 258 | const std::string ub_inner_offset = CalcInnerOffset(tpipe, ub_stride); |
| 254 | std::stringstream ss1; | 259 | std::stringstream ss1; |
| 255 | NddmaParams nddma_param; | 260 | NddmaParams nddma_param; |
| 256 | - SetNddmaParams(tpipe, data_copy_param, nddma_param, output.id, ss); | 261 | + std::string padding_mode = GetPaddingMode(output, data_copy_param, false); |
| 262 | + SetNddmaParams(tpipe, data_copy_param, nddma_param, output.id, padding_mode, ss); | ||
| 257 | ss1 << "DataCopyNddma(" << output << "[" << ub_inner_offset << "], " << input << "[" << gm_offset << " + " | 263 | ss1 << "DataCopyNddma(" << output << "[" << ub_inner_offset << "], " << input << "[" << gm_offset << " + " |
| 258 | << tpipe.tiler.Size(offset) << " + " << gm_inner_offset << "], " | 264 | << tpipe.tiler.Size(offset) << " + " << gm_inner_offset << "], " |
| 259 | << "output_dims_" << output.id << ", " << "output_stride_" << output.id << ", " << "input_stride_" << output.id | 265 | << "output_dims_" << output.id << ", " << "output_stride_" << output.id << ", " << "input_stride_" << output.id |
| @@ -351,12 +357,12 @@ void BuildDataCopyLoopModeParams(DataCopyParams &data_copy_param, DmaSpecificPar | |||
| 351 | 357 | ||
| 352 | Status BuildDataCopyApiParamInNormal(const TPipe &tpipe, CodegenApiParam &api_param, | 358 | Status BuildDataCopyApiParamInNormal(const TPipe &tpipe, CodegenApiParam &api_param, |
| 353 | DmaSpecificParams &dma_specific_params, const Tensor &src, const Tensor &dst, | 359 | DmaSpecificParams &dma_specific_params, const Tensor &src, const Tensor &dst, |
| 354 | - std::string &gm_offset, bool copy_in) { | 360 | + std::string &gm_offset, bool copy_in, bool has_transpose) { |
| 355 | DataCopyParams data_copy_param; | 361 | DataCopyParams data_copy_param; |
| 356 | GE_ASSERT_TRUE(CalculateDmaParams(tpipe, dst, dst, data_copy_param), "CalculateDmaParams failed"); | 362 | GE_ASSERT_TRUE(CalculateDmaParams(tpipe, dst, dst, data_copy_param), "CalculateDmaParams failed"); |
| 357 | size_t total_len = data_copy_param.repeats.size(); | 363 | size_t total_len = data_copy_param.repeats.size(); |
| 358 | const Tensor &ub_tensor = copy_in ? dst : src; | 364 | const Tensor &ub_tensor = copy_in ? dst : src; |
| 359 | - std::string padding_mode = GetPaddingMode(ub_tensor, data_copy_param); | 365 | + std::string padding_mode = GetPaddingMode(ub_tensor, data_copy_param, has_transpose); |
| 360 | api_param.template_params.emplace_back(padding_mode); | 366 | api_param.template_params.emplace_back(padding_mode); |
| 361 | 367 | ||
| 362 | BuildDataCopyBaseParams(tpipe, data_copy_param, dma_specific_params, copy_in); | 368 | BuildDataCopyBaseParams(tpipe, data_copy_param, dma_specific_params, copy_in); |
| @@ -432,4 +438,14 @@ Status GenDataCopyDimParam(const CodegenApiParam &api_param, const Tiler &tiler, | |||
| 432 | return af::SUCCESS; | 438 | return af::SUCCESS; |
| 433 | } | 439 | } |
| 434 | 440 | ||
| 441 | +bool IsGraphHasTransposeNode(const af::AscNodePtr &node) { | ||
| 442 | + const auto owner_graph = node->GetOwnerComputeGraph(); | ||
| 443 | + for (const auto &graph_node : owner_graph->GetAllNodes()) { | ||
| 444 | + if (af::ops::IsOps<af::ascir_op::Transpose>(graph_node)) { | ||
| 445 | + return true; | ||
| 446 | + } | ||
| 447 | + } | ||
| 448 | + return false; | ||
| 449 | +} | ||
| 450 | + | ||
| 435 | } // namespace codegen | 451 | } // namespace codegen |
| @@ -80,19 +80,20 @@ void CreateEnhanceDmaCall(const TPipe &tpipe, const Tensor &input, const Tensor | |||
| 80 | void CreateNddmaCall(const TPipe &tpipe, const Tensor &input, const Tensor &output, const string &gm_offset, | 80 | void CreateNddmaCall(const TPipe &tpipe, const Tensor &input, const Tensor &output, const string &gm_offset, |
| 81 | const DataCopyParams &data_copy_param, const ascir::SizeExpr &offset, std::stringstream &ss); | 81 | const DataCopyParams &data_copy_param, const ascir::SizeExpr &offset, std::stringstream &ss); |
| 82 | void SetNddmaParams(const TPipe &tpipe, const DataCopyParams &data_copy_param, NddmaParams &nddma_param, | 82 | void SetNddmaParams(const TPipe &tpipe, const DataCopyParams &data_copy_param, NddmaParams &nddma_param, |
| 83 | - const int64_t &tensor_id, std::stringstream &ss); | 83 | + const int64_t &tensor_id, std::string padding_mode, std::stringstream &ss); |
| 84 | void SetLoopModeParams(const TPipe &tpipe, const DataCopyParams &data_copy_param, LoopModeParams &loop_mode_param, | 84 | void SetLoopModeParams(const TPipe &tpipe, const DataCopyParams &data_copy_param, LoopModeParams &loop_mode_param, |
| 85 | bool copy_in); | 85 | bool copy_in); |
| 86 | void SetLoopModeParamsExpr(const DataCopyParams &data_copy_param, LoopModeParamsExpr &loop_mode_param, bool copy_in); | 86 | void SetLoopModeParamsExpr(const DataCopyParams &data_copy_param, LoopModeParamsExpr &loop_mode_param, bool copy_in); |
| 87 | -std::string GetPaddingMode(const Tensor &ub_tensor, const DataCopyParams &data_copy_param); | 87 | +std::string GetPaddingMode(const Tensor &ub_tensor, const DataCopyParams &data_copy_param, bool has_transpose); |
| 88 | void BuildDataCopyApiParamInCVFusion(const TPipe &tpipe, CodegenApiParam &api_param, | 88 | void BuildDataCopyApiParamInCVFusion(const TPipe &tpipe, CodegenApiParam &api_param, |
| 89 | DmaSpecificParams &dma_specific_params, const Tensor &gm, const Tensor &ub, | 89 | DmaSpecificParams &dma_specific_params, const Tensor &gm, const Tensor &ub, |
| 90 | std::string &dtype_name, bool copy_in); | 90 | std::string &dtype_name, bool copy_in); |
| 91 | Status BuildDataCopyApiParamInNormal(const TPipe &tpipe, CodegenApiParam &api_param, | 91 | Status BuildDataCopyApiParamInNormal(const TPipe &tpipe, CodegenApiParam &api_param, |
| 92 | DmaSpecificParams &dma_specific_params, const Tensor &src, const Tensor &dst, | 92 | DmaSpecificParams &dma_specific_params, const Tensor &src, const Tensor &dst, |
| 93 | - std::string &gm_offset, bool copy_in); | 93 | + std::string &gm_offset, bool copy_in, bool has_transpose); |
| 94 | Status GenDataCopyDimParam(const CodegenApiParam &api_param, const Tiler &tiler, std::string graph_name, | 94 | Status GenDataCopyDimParam(const CodegenApiParam &api_param, const Tiler &tiler, std::string graph_name, |
| 95 | std::string node_name, std::stringstream &ss); | 95 | std::string node_name, std::stringstream &ss); |
| 96 | +bool IsGraphHasTransposeNode(const af::AscNodePtr &node); | ||
| 96 | 97 | ||
| 97 | } // namespace codegen | 98 | } // namespace codegen |
| 98 | 99 | ||
| @@ -22,15 +22,15 @@ namespace codegen { | |||
| 22 | using namespace std; | 22 | using namespace std; |
| 23 | using namespace ascgen_utils; | 23 | using namespace ascgen_utils; |
| 24 | 24 | ||
| 25 | -static void GenParams(const TPipe &tpipe, const Tensor &input, const Tensor &output, std::stringstream &ss, | 25 | +static void GenParams(const TPipe &tpipe, const Tensor &input, const Tensor &output, std::stringstream &ss, bool is_src, |
| 26 | - bool is_src) { | 26 | + bool has_transpose) { |
| 27 | // 只保证在仅对张量尾轴做32B对齐的场景下有效，若对中间轴做了对齐，则还需要增加处理逻辑 | 27 | // 只保证在仅对张量尾轴做32B对齐的场景下有效，若对中间轴做了对齐，则还需要增加处理逻辑 |
| 28 | auto vectorized_axis_size = input.vectorized_axis.size(); | 28 | auto vectorized_axis_size = input.vectorized_axis.size(); |
| 29 | const char *shape_prefix = is_src ? "src_shape_" : "dst_shape_"; | 29 | const char *shape_prefix = is_src ? "src_shape_" : "dst_shape_"; |
| 30 | DataCopyParams data_copy_param; | 30 | DataCopyParams data_copy_param; |
| 31 | (void)CalculateDmaParams(tpipe, output, output, data_copy_param); | 31 | (void)CalculateDmaParams(tpipe, output, output, data_copy_param); |
| 32 | constexpr char kCompactPddingMode[] = "AscendC::PaddingMode::Compact"; | 32 | constexpr char kCompactPddingMode[] = "AscendC::PaddingMode::Compact"; |
| 33 | - std::string padding_mode = GetPaddingMode(output, data_copy_param); | 33 | + std::string padding_mode = GetPaddingMode(output, data_copy_param, has_transpose); |
| 34 | ss << "const uint32_t " << shape_prefix << input.id << "_brc_to_" << output.id << "[" << vectorized_axis_size | 34 | ss << "const uint32_t " << shape_prefix << input.id << "_brc_to_" << output.id << "[" << vectorized_axis_size |
| 35 | << "] = {"; | 35 | << "] = {"; |
| 36 | const char *sep = ""; | 36 | const char *sep = ""; |
| @@ -115,9 +115,10 @@ Status BroadcastRegApiCall::Generate(const TPipe &tpipe, const std::vector<ascir | |||
| 115 | std::stringstream ss; | 115 | std::stringstream ss; |
| 116 | // 生成参数 const uint32_t *dst_shape; | 116 | // 生成参数 const uint32_t *dst_shape; |
| 117 | std::stringstream params_name; | 117 | std::stringstream params_name; |
| 118 | - GenParams(tpipe, x, y, ss, false); | 118 | + auto has_transpose = IsGraphHasTransposeNode(this->node); |
| 119 | + GenParams(tpipe, x, y, ss, false, has_transpose); | ||
| 119 | // 生成参数 const uint32_t *src_shape; | 120 | // 生成参数 const uint32_t *src_shape; |
| 120 | - GenParams(tpipe, x, y, ss, true); | 121 | + GenParams(tpipe, x, y, ss, true, has_transpose); |
| 121 | 122 | ||
| 122 | std::string dtype_name; | 123 | std::string dtype_name; |
| 123 | Tensor::DtypeName(x.dtype, dtype_name); | 124 | Tensor::DtypeName(x.dtype, dtype_name); |
| @@ -46,7 +46,8 @@ Status LoadRegApiCall::BuildApiParam(const TPipe &tpipe, const std::vector<ascir | |||
| 46 | } else { | 46 | } else { |
| 47 | std::string gm_offset = ub.is_ub_scalar ? "0" : tpipe.tiler.Offset(current_axis, ub.axis, ub.axis_strides); | 47 | std::string gm_offset = ub.is_ub_scalar ? "0" : tpipe.tiler.Offset(current_axis, ub.axis, ub.axis_strides); |
| 48 | gm_offset = gm_offset + " + " + tpipe.tiler.Size(offset_); | 48 | gm_offset = gm_offset + " + " + tpipe.tiler.Size(offset_); |
| 49 | - BuildDataCopyApiParamInNormal(tpipe, *api_param, dma_specific_params, gm, ub, gm_offset, true); | 49 | + auto has_transpose = IsGraphHasTransposeNode(this->node); |
| 50 | + BuildDataCopyApiParamInNormal(tpipe, *api_param, dma_specific_params, gm, ub, gm_offset, true, has_transpose); | ||
| 50 | } | 51 | } |
| 51 | api_param->specific_params = dma_specific_params; | 52 | api_param->specific_params = dma_specific_params; |
| 52 | 53 | ||
| @@ -76,7 +76,8 @@ Status GenerateDefaultNddma(const TPipe &tpipe, const std::vector<ascir::AxisId> | |||
| 76 | ub.is_ub_scalar ? "0" : tpipe.tiler.Offset(current_axis, ub.axis, ub.axis_strides); // 每次从gm搬到ub的偏移量 | 76 | ub.is_ub_scalar ? "0" : tpipe.tiler.Offset(current_axis, ub.axis, ub.axis_strides); // 每次从gm搬到ub的偏移量 |
| 77 | if (param.repeats.size() <= kNddmaMaxLen) { | 77 | if (param.repeats.size() <= kNddmaMaxLen) { |
| 78 | NddmaParams nddma_param; | 78 | NddmaParams nddma_param; |
| 79 | - SetNddmaParams(tpipe, param, nddma_param, ub.id, ss); | 79 | + std::string padding_mode = GetPaddingMode(ub, param, false); |
| 80 | + SetNddmaParams(tpipe, param, nddma_param, ub.id, padding_mode, ss); | ||
| 80 | ss << api_name << "(" << ub << ", " << gm << "[" << gm_offset << " + " << tpipe.tiler.Size(offset) << "], " | 81 | ss << api_name << "(" << ub << ", " << gm << "[" << gm_offset << " + " << tpipe.tiler.Size(offset) << "], " |
| 81 | << "output_dims_" << ub.id << ", " << "output_stride_" << ub.id << ", " << "input_stride_" << ub.id << ");" | 82 | << "output_dims_" << ub.id << ", " << "output_stride_" << ub.id << ", " << "input_stride_" << ub.id << ");" |
| 82 | << std::endl; | 83 | << std::endl; |
| @@ -58,7 +58,8 @@ Status StoreRegApiCall::BuildApiParam(const TPipe &tpipe, const std::vector<asci | |||
| 58 | } else { | 58 | } else { |
| 59 | std::string gm_offset = tpipe.tiler.Offset(current_axis, gm.axis, gm.axis_strides); | 59 | std::string gm_offset = tpipe.tiler.Offset(current_axis, gm.axis, gm.axis_strides); |
| 60 | gm_offset = gm_offset + " + " + tpipe.tiler.Size(offset_); | 60 | gm_offset = gm_offset + " + " + tpipe.tiler.Size(offset_); |
| 61 | - BuildDataCopyApiParamInNormal(tpipe, *api_param, dma_specific_params, ub, gm, gm_offset, false); | 61 | + auto has_transpose = IsGraphHasTransposeNode(this->node); |
| 62 | + BuildDataCopyApiParamInNormal(tpipe, *api_param, dma_specific_params, ub, gm, gm_offset, false, has_transpose); | ||
| 62 | } | 63 | } |
| 63 | api_param->specific_params = dma_specific_params; | 64 | api_param->specific_params = dma_specific_params; |
| 64 | if (IsUnitLastRead(*(this->inputs[0])) && ub.is_load_link_store_and_vec) { | 65 | if (IsUnitLastRead(*(this->inputs[0])) && ub.is_load_link_store_and_vec) { |
| @@ -21,6 +21,10 @@ | |||
| 21 | #include "api_call/utils/api_call_utils.h" | 21 | #include "api_call/utils/api_call_utils.h" |
| 22 | #include "ascir_node_param/ascir_node_param.h" | 22 | #include "ascir_node_param/ascir_node_param.h" |
| 23 | 23 | ||
| 24 | +namespace { | ||
| 25 | +constexpr uint64_t kDmaMaxLen = 2U; | ||
| 26 | +} | ||
| 27 | + | ||
| 24 | namespace codegen { | 28 | namespace codegen { |
| 25 | using namespace std; | 29 | using namespace std; |
| 26 | using namespace af::ops; | 30 | using namespace af::ops; |
| @@ -51,12 +55,13 @@ uint32_t GetContinuousInnerAxisNum(const Tensor &y, std::vector<ascir::SizeExpr> | |||
| 51 | return transpose_inner_axis_num; | 55 | return transpose_inner_axis_num; |
| 52 | } | 56 | } |
| 53 | 57 | ||
| 54 | -Status ReorderInputStrideByOutputAxisOrder(const Tensor &x, const Tensor &y, | 58 | +Status ReorderInputStrideByOutputAxisOrder(const Tensor &x, const Tensor &y, std::vector<uint64_t> &origin_axis_pos, |
| 55 | std::vector<ascir::SizeExpr> &reordered_input_stride) { | 59 | std::vector<ascir::SizeExpr> &reordered_input_stride) { |
| 56 | for (size_t i = 0; i < y.vectorized_axis.size(); i++) { | 60 | for (size_t i = 0; i < y.vectorized_axis.size(); i++) { |
| 57 | auto it = std::find(x.vectorized_axis.begin(), x.vectorized_axis.end(), y.vectorized_axis[i]); | 61 | auto it = std::find(x.vectorized_axis.begin(), x.vectorized_axis.end(), y.vectorized_axis[i]); |
| 58 | GE_ASSERT_TRUE(it != x.vectorized_axis.end(), "InValid axis ID in input vectorized_axis: %zu", i); | 62 | GE_ASSERT_TRUE(it != x.vectorized_axis.end(), "InValid axis ID in input vectorized_axis: %zu", i); |
| 59 | auto axis_pos = static_cast<uint64_t>(std::distance(x.vectorized_axis.begin(), it)); | 63 | auto axis_pos = static_cast<uint64_t>(std::distance(x.vectorized_axis.begin(), it)); |
| 64 | + origin_axis_pos.emplace_back(axis_pos); | ||
| 60 | reordered_input_stride.emplace_back(x.vectorized_strides[axis_pos]); | 65 | reordered_input_stride.emplace_back(x.vectorized_strides[axis_pos]); |
| 61 | } | 66 | } |
| 62 | return af::SUCCESS; | 67 | return af::SUCCESS; |
| @@ -65,14 +70,29 @@ Status ReorderInputStrideByOutputAxisOrder(const Tensor &x, const Tensor &y, | |||
| 65 | void BuildTransposeLoopParams(TransposeSpecificParams &transpose_specific_params, | 70 | void BuildTransposeLoopParams(TransposeSpecificParams &transpose_specific_params, |
| 66 | std::vector<ascir::SizeExpr> &out_vectorized_repeats, | 71 | std::vector<ascir::SizeExpr> &out_vectorized_repeats, |
| 67 | std::vector<ascir::SizeExpr> &reordered_in_vectorized_strides, | 72 | std::vector<ascir::SizeExpr> &reordered_in_vectorized_strides, |
| 68 | - std::vector<ascir::SizeExpr> &out_vectorized_strides, uint32_t transpose_total_axis_num) { | 73 | + std::vector<ascir::SizeExpr> &out_vectorized_strides, uint32_t transpose_total_axis_num, |
| 74 | + std::vector<uint64_t> &origin_axis_pos) { | ||
| 69 | for (size_t i = out_vectorized_repeats.size() - transpose_total_axis_num; i < out_vectorized_repeats.size(); i++) { | 75 | for (size_t i = out_vectorized_repeats.size() - transpose_total_axis_num; i < out_vectorized_repeats.size(); i++) { |
| 70 | transpose_specific_params.output_dims.emplace_back( | 76 | transpose_specific_params.output_dims.emplace_back( |
| 71 | CombinedExpression(ExprItemFactory::ActualSize(out_vectorized_repeats[i]))); | 77 | CombinedExpression(ExprItemFactory::ActualSize(out_vectorized_repeats[i]))); |
| 72 | - transpose_specific_params.input_strides.emplace_back( | 78 | + // Transpose融合的ub-Transpose模板，如果最后两根向量化轴不连续，统一走Compact模式， |
| 73 | - CombinedExpression(ExprItemFactory::Size(reordered_in_vectorized_strides[i]))); | 79 | + // 此时后两根轴的内存排布是紧凑的，因此对应的stride应该使用ActualSize。 |
| 74 | - transpose_specific_params.output_strides.emplace_back( | 80 | + // 如果最后两根向量化轴连续，此时最后一根轴一定不是切分内轴，最后两根轴使用ActualSize或Size没有区别。 |
| 75 | - CombinedExpression(ExprItemFactory::Size(out_vectorized_strides[i]))); | 81 | + // 除此之外，其他的轴在Load/Store使用的stride默认是Size，因此Transpose的时候也使用Size即可。 |
| 82 | + if (origin_axis_pos[i] + kDmaMaxLen >= out_vectorized_repeats.size()) { | ||
| 83 | + transpose_specific_params.input_strides.emplace_back( | ||
| 84 | + CombinedExpression(ExprItemFactory::ActualSize(reordered_in_vectorized_strides[i]))); | ||
| 85 | + } else { | ||
| 86 | + transpose_specific_params.input_strides.emplace_back( | ||
| 87 | + CombinedExpression(ExprItemFactory::Size(reordered_in_vectorized_strides[i]))); | ||
| 88 | + } | ||
| 89 | + if (i + kDmaMaxLen >= out_vectorized_repeats.size()) { | ||
| 90 | + transpose_specific_params.output_strides.emplace_back( | ||
| 91 | + CombinedExpression(ExprItemFactory::ActualSize(out_vectorized_strides[i]))); | ||
| 92 | + } else { | ||
| 93 | + transpose_specific_params.output_strides.emplace_back( | ||
| 94 | + CombinedExpression(ExprItemFactory::Size(out_vectorized_strides[i]))); | ||
| 95 | + } | ||
| 76 | } | 96 | } |
| 77 | } | 97 | } |
| 78 | 98 | ||
| @@ -109,15 +129,6 @@ af::Status FillTransposeNodeParams(const af::AscNodePtr &node, | |||
| 109 | return af::SUCCESS; | 129 | return af::SUCCESS; |
| 110 | } | 130 | } |
| 111 | 131 | ||
| 112 | -// 构建带循环偏移的 inner_offset 表达式（简化表达式合并操作） | ||
| 113 | -CombinedExpression BuildInnerOffsetWithLoopOffset(const std::string &base_offset, | ||
| 114 | - const std::vector<ascir::SizeExpr> &loop_strides) { | ||
| 115 | - CombinedExpression inner_offset = CombinedExpression(ExprItemFactory::Direct(ge::Symbol(base_offset.c_str()))); | ||
| 116 | - CombinedExpression loop_offset = CalcInnerOffsetExpr(loop_strides); | ||
| 117 | - inner_offset.AddExpression(loop_offset, "+"); | ||
| 118 | - return inner_offset; | ||
| 119 | -} | ||
| 120 | - | ||
| 121 | Status TransposeRegApiCall::BuildApiParam(const TPipe &tpipe, const std::vector<ascir::AxisId> ¤t_axis, | 132 | Status TransposeRegApiCall::BuildApiParam(const TPipe &tpipe, const std::vector<ascir::AxisId> ¤t_axis, |
| 122 | const std::vector<std::reference_wrapper<const Tensor>> &inputs, | 133 | const std::vector<std::reference_wrapper<const Tensor>> &inputs, |
| 123 | const std::vector<std::reference_wrapper<const Tensor>> &outputs) const { | 134 | const std::vector<std::reference_wrapper<const Tensor>> &outputs) const { |
| @@ -133,7 +144,10 @@ Status TransposeRegApiCall::BuildApiParam(const TPipe &tpipe, const std::vector< | |||
| 133 | api_param->template_params.emplace_back(std::to_string(transpose_inner_axis_num)); | 144 | api_param->template_params.emplace_back(std::to_string(transpose_inner_axis_num)); |
| 134 | api_param->template_params.emplace_back(std::to_string(transpose_total_axis_num)); | 145 | api_param->template_params.emplace_back(std::to_string(transpose_total_axis_num)); |
| 135 | std::vector<ascir::SizeExpr> reordered_in_vectorized_strides; | 146 | std::vector<ascir::SizeExpr> reordered_in_vectorized_strides; |
| 136 | - GE_CHK_STATUS_RET(ReorderInputStrideByOutputAxisOrder(x, y, reordered_in_vectorized_strides)); | 147 | + std::vector<uint64_t> origin_axis_pos; |
| 148 | + GE_CHK_STATUS_RET(ReorderInputStrideByOutputAxisOrder(x, y, origin_axis_pos, reordered_in_vectorized_strides)); | ||
| 149 | + GE_ASSERT_TRUE(origin_axis_pos.size() == y.vectorized_axis.size(), "InValid origin_axis_pos size: %zu", | ||
| 150 | + origin_axis_pos.size()); | ||
| 137 | // 构建 inner_offset 表达式 | 151 | // 构建 inner_offset 表达式 |
| 138 | CombinedExpression input_inner_offset = CombinedExpression( | 152 | CombinedExpression input_inner_offset = CombinedExpression( |
| 139 | ExprItemFactory::Direct(ge::Symbol(tpipe.tiler.TensorVectorizedOffset(current_axis, x).c_str()))); | 153 | ExprItemFactory::Direct(ge::Symbol(tpipe.tiler.TensorVectorizedOffset(current_axis, x).c_str()))); |
| @@ -165,7 +179,7 @@ Status TransposeRegApiCall::BuildApiParam(const TPipe &tpipe, const std::vector< | |||
| 165 | 179 | ||
| 166 | TransposeSpecificParams transpose_specific_params; | 180 | TransposeSpecificParams transpose_specific_params; |
| 167 | BuildTransposeLoopParams(transpose_specific_params, out_vectorized_repeats, reordered_in_vectorized_strides, | 181 | BuildTransposeLoopParams(transpose_specific_params, out_vectorized_repeats, reordered_in_vectorized_strides, |
| 168 | - y.vectorized_strides, transpose_total_axis_num); | 182 | + y.vectorized_strides, transpose_total_axis_num, origin_axis_pos); |
| 169 | api_param->specific_params = transpose_specific_params; | 183 | api_param->specific_params = transpose_specific_params; |
| 170 | GE_ASSERT_SUCCESS(FillTransposeNodeParams(this->node, out_vectorized_repeats, reordered_in_vectorized_strides, | 184 | GE_ASSERT_SUCCESS(FillTransposeNodeParams(this->node, out_vectorized_repeats, reordered_in_vectorized_strides, |
| 171 | y.vectorized_strides, transpose_inner_axis_num, transpose_total_axis_num)); | 185 | y.vectorized_strides, transpose_inner_axis_num, transpose_total_axis_num)); |
| @@ -421,8 +421,12 @@ Status UnAlignmentStrategy::ModifyTransposeFusionVectorizedStrides(af::AscGraph | |||
| 421 | // Transpose及后续节点或无Transpose：按 Store 轴判断 | 421 | // Transpose及后续节点或无Transpose：按 Store 轴判断 |
| 422 | GE_ASSERT_SUCCESS(GetNodeContinuousTailAxisNumByStore(node, continuous_tail_axis_num)); | 422 | GE_ASSERT_SUCCESS(GetNodeContinuousTailAxisNumByStore(node, continuous_tail_axis_num)); |
| 423 | } | 423 | } |
| 424 | - if (continuous_tail_axis_num <= 1U || | 424 | + // 如果是ub-Transpose模板，则可以走Compact模式搬运，连续轴数量调整为2。 |
| 425 | - continuous_tail_axis_num == UINT32_MAX) { // 小于一个连续轴，不需要调整strides | 425 | + // 实际上nddma模板也可以走Compact模式，但是codegen目前只能判断显式Transpose节点，暂时不放开。 |
| 426 | + if (!transpose_pre_nodes.empty() && continuous_tail_axis_num <= 1U) { | ||
| 427 | + continuous_tail_axis_num = 2U; | ||
| 428 | + } | ||
| 429 | + if (continuous_tail_axis_num <= 1 || continuous_tail_axis_num == UINT32_MAX) { | ||
| 426 | continue; | 430 | continue; |
| 427 | } | 431 | } |
| 428 | for (const auto &output : node->outputs()) { | 432 | for (const auto &output : node->outputs()) { |