已合并
fix(cv-ubfuse): align UB physical layout for mixed-dtype vector tensors #2177
tianbaolin创建于 9月18日
fix(cv-ubfuse): align UB physical layout for mixed-dtype vector tensors #2177
已合并
共 22 个文件变更+251-49
| @@ -18,6 +18,7 @@ | |||
| 18 | #include "graph/symbolizer/symbolic_utils.h" | 18 | #include "graph/symbolizer/symbolic_utils.h" |
| 19 | #include "common/checker.h" | 19 | #include "common/checker.h" |
| 20 | #include "api_call/utils/api_call_factory.h" | 20 | #include "api_call/utils/api_call_factory.h" |
| 21 | +#include "api_call/utils/api_call_utils.h" | ||
| 21 | #include "codegen/expression_convert_struct.h" | 22 | #include "codegen/expression_convert_struct.h" |
| 22 | 23 | ||
| 23 | namespace codegen { | 24 | namespace codegen { |
| @@ -95,6 +96,25 @@ Status BinaryApiCall::Generate(const TPipe &tpipe, const std::vector<ascir::Axis | |||
| 95 | ss << ");" << std::endl; | 96 | ss << ");" << std::endl; |
| 96 | } | 97 | } |
| 97 | } else { // 两个输入都不是Scalar | 98 | } else { // 两个输入都不是Scalar |
| 99 | + if (IsCVFusionStage(this->api_call_context) || tpipe.cv_fusion_type == ascir::CubeTemplateType::kUBFuse) { | ||
| 100 | + // CV UBFuse 下 flat 1D 调用的 count 取自 stage 尺寸, 无法覆盖按输出 dtype 对齐的二维布局 | ||
| 101 | + // (如 bool 输入 N=4: 行距32, 总量 M*32, stage 仅 M*8), 尾部行读到未初始化数据。 | ||
| 102 | + // 按 per-dtype 物理行距逐行调用, 与 UnaryApiCall/位宽类 API 的 CV 修复方式一致。 | ||
| 103 | + const std::string output_stride = GenBlockAlignNExpr(y, "curAivN"); | ||
| 104 | + const auto gen_input_stride = [&](const Tensor &t) { | ||
| 105 | + return (tpipe.cv_fusion_type == ascir::CubeTemplateType::kUBFuse && t.id == tpipe.cube_output_tensor_id) | ||
| 106 | + ? std::string("curAlignN") | ||
| 107 | + : GenBlockAlignNExpr(t, "curAivN"); | ||
| 108 | + }; | ||
| 109 | + ss << y.actual_size << " = curAivM * " << output_stride << ";" << std::endl; | ||
| 110 | + ss << "for (uint32_t cv_row = 0; cv_row < ConvertToUint32(curAivM); cv_row++) {" << std::endl; | ||
| 111 | + ss << " " << this->api_name_ << "(" << y << "[cv_row * ConvertToUint32(" << output_stride << ")], " << x1 | ||
| 112 | + << "[cv_row * ConvertToUint32(" << gen_input_stride(x1) << ")], " << x2 << "[cv_row * ConvertToUint32(" | ||
| 113 | + << gen_input_stride(x2) << ")], ConvertToUint32(curAivN));" << std::endl; | ||
| 114 | + ss << "}" << std::endl; | ||
| 115 | + result = ss.str(); | ||
| 116 | + return af::SUCCESS; | ||
| 117 | + } | ||
| 98 | ss << this->api_name_ << "(" << y << "[" << tpipe.tiler.TensorVectorizedOffset(current_axis, y) << "], " << x1 | 118 | ss << this->api_name_ << "(" << y << "[" << tpipe.tiler.TensorVectorizedOffset(current_axis, y) << "], " << x1 |
| 99 | << "[" << tpipe.tiler.TensorVectorizedOffset(current_axis, x1) << "], " << x2 << "[" | 119 | << "[" << tpipe.tiler.TensorVectorizedOffset(current_axis, x1) << "], " << x2 << "[" |
| 100 | << tpipe.tiler.TensorVectorizedOffset(current_axis, x2) << "], " << x1.actual_size << ");" << std::endl; | 120 | << tpipe.tiler.TensorVectorizedOffset(current_axis, x2) << "], " << x1.actual_size << ");" << std::endl; |
| @@ -17,6 +17,7 @@ | |||
| 17 | #include "graph/ascendc_ir/utils/asc_tensor_utils.h" | 17 | #include "graph/ascendc_ir/utils/asc_tensor_utils.h" |
| 18 | #include "common/checker.h" | 18 | #include "common/checker.h" |
| 19 | #include "api_call/utils/api_call_factory.h" | 19 | #include "api_call/utils/api_call_factory.h" |
| 20 | +#include "api_call/utils/api_call_utils.h" | ||
| 20 | #include "codegen/expression_convert_struct.h" | 21 | #include "codegen/expression_convert_struct.h" |
| 21 | 22 | ||
| 22 | namespace codegen { | 23 | namespace codegen { |
| @@ -35,6 +36,22 @@ Status UnaryApiCall::Generate(const TPipe &tpipe, const std::vector<ascir::AxisI | |||
| 35 | (void)RegisterBasicDumpParam(this->api_name_, inputs, outputs, CombinedExprFactory::SymbolVar(x.actual_size.Str())); | 36 | (void)RegisterBasicDumpParam(this->api_name_, inputs, outputs, CombinedExprFactory::SymbolVar(x.actual_size.Str())); |
| 36 | 37 | ||
| 37 | stringstream ss; | 38 | stringstream ss; |
| 39 | + if (IsCVFusionStage(this->api_call_context) || tpipe.cv_fusion_type == ascir::CubeTemplateType::kUBFuse) { | ||
| 40 | + // CV UBFuse 下 flat 1D 调用的 count 取自 stage 尺寸, 无法覆盖按输出 dtype 对齐的二维布局 | ||
| 41 | + // (如 bool 输入 N=4: 行距32, 总量 M*32, stage 仅 M*8), 尾部行读到未初始化数据。 | ||
| 42 | + // 按 per-dtype 物理行距逐行调用, 输出按输出 dtype 行距落位, 与 stage 写出/下游消费侧行距一致。 | ||
| 43 | + const std::string output_stride = GenBlockAlignNExpr(y, "curAivN"); | ||
| 44 | + const bool input_is_cube_output = | ||
| 45 | + tpipe.cv_fusion_type == ascir::CubeTemplateType::kUBFuse && x.id == tpipe.cube_output_tensor_id; | ||
| 46 | + const std::string input_stride = input_is_cube_output ? "curAlignN" : GenBlockAlignNExpr(x, "curAivN"); | ||
| 47 | + ss << y.actual_size << " = curAivM * " << output_stride << ";" << std::endl; | ||
| 48 | + ss << "for (uint32_t cv_row = 0; cv_row < ConvertToUint32(curAivM); cv_row++) {" << std::endl; | ||
| 49 | + ss << " " << this->api_name_ << "(" << y << "[cv_row * ConvertToUint32(" << output_stride << ")], " << x | ||
| 50 | + << "[cv_row * ConvertToUint32(" << input_stride << ")], ConvertToUint32(curAivN));" << std::endl; | ||
| 51 | + ss << "}" << std::endl; | ||
| 52 | + result = ss.str(); | ||
| 53 | + return af::SUCCESS; | ||
| 54 | + } | ||
| 38 | ss << this->api_name_ << "(" << y << "[" << tpipe.tiler.TensorVectorizedOffset(current_axis, y) << "], " << x << "[" | 55 | ss << this->api_name_ << "(" << y << "[" << tpipe.tiler.TensorVectorizedOffset(current_axis, y) << "], " << x << "[" |
| 39 | << tpipe.tiler.TensorVectorizedOffset(current_axis, x) << "], " << x.actual_size << ");" << std::endl; | 56 | << tpipe.tiler.TensorVectorizedOffset(current_axis, x) << "], " << x.actual_size << ");" << std::endl; |
| 40 | result = ss.str(); | 57 | result = ss.str(); |
| @@ -195,6 +195,14 @@ std::string GetVectorGroupWorkspaceOutputName(const ascir::ScheduleGroup &schedu | |||
| 195 | if (GetSingleWorkspaceName(impl_graph) != workspace_name) { | 195 | if (GetSingleWorkspaceName(impl_graph) != workspace_name) { |
| 196 | return ""; | 196 | return ""; |
| 197 | } | 197 | } |
| 198 | + | ||
| 199 | + // vector读取外部输入时, 输出buffer可能与该输入复用, 复用会覆盖输入, 故禁用 | ||
| 200 | + for (const auto &node : impl_graph.GetAllNodes()) { | ||
| 201 | + if (IsOps<Data>(node)) { | ||
| 202 | + return ""; | ||
| 203 | + } | ||
| 204 | + } | ||
| 205 | + | ||
| 198 | const auto current_output_name = GetWorkspacePathOutputName(impl_graph, workspace_name); | 206 | const auto current_output_name = GetWorkspacePathOutputName(impl_graph, workspace_name); |
| 199 | if (current_output_name.empty() || (!output_name.empty() && output_name != current_output_name)) { | 207 | if (current_output_name.empty() || (!output_name.empty() && output_name != current_output_name)) { |
| 200 | return ""; | 208 | return ""; |
| @@ -2165,6 +2173,33 @@ Status Kernel::ParseWorkspaceTensor(const ascir::TensorAttr *tensor, | |||
| 2165 | return af::SUCCESS; | 2173 | return af::SUCCESS; |
| 2166 | } | 2174 | } |
| 2167 | 2175 | ||
| 2176 | +namespace { | ||
| 2177 | +// GetTmpBuffer 会为 tmp 公式套上 v1 repeat cap 壳: Min(size, 65312)。CV UBFuse 的 tmp | ||
| 2178 | +// 内联公式不走 tiling 求解, stage 尺寸由 cube tiling 决定, 需求无法缩到 cap 以内, 因此 | ||
| 2179 | +// 在 CV 收集点剥掉该壳取原始需求。数值须与 GetTmpBuffer 的 cap(255 * 256 + 32)保持一致。 | ||
| 2180 | +// 非 cap 形态(无壳或常量非该值)原样返回; VV 路径不经此处理, 继续使用 capped size 参与 | ||
| 2181 | +// host tiling 求解。 | ||
| 2182 | +constexpr int64_t kMaxV1RepeatTmpBytes = 255LL * 256LL + 32LL; | ||
| 2183 | + | ||
| 2184 | +// GetArgs 为非 const 接口, 此处按值传递以便调用。 | ||
| 2185 | +af::Expression StripV1RepeatCap(af::Expression size) { | ||
| 2186 | + if (!size.IsValid()) { | ||
| 2187 | + return size; | ||
| 2188 | + } | ||
| 2189 | + auto args = size.GetArgs(); | ||
| 2190 | + if (args.size() != 2UL) { | ||
| 2191 | + return size; | ||
| 2192 | + } | ||
| 2193 | + for (size_t i = 0UL; i < args.size(); i++) { | ||
| 2194 | + int64_t cap_value = 0; | ||
| 2195 | + if (args[i].IsConstExpr() && args[i].GetConstValue(cap_value) && cap_value == kMaxV1RepeatTmpBytes) { | ||
| 2196 | + return args[1UL - i]; | ||
| 2197 | + } | ||
| 2198 | + } | ||
| 2199 | + return size; | ||
| 2200 | +} | ||
| 2201 | +} // namespace | ||
| 2202 | + | ||
| 2168 | Status Kernel::ParseGraph(const ascir::ImplGraph &graph, const ascir::FusedScheduledResult &fused_schedule_result, | 2203 | Status Kernel::ParseGraph(const ascir::ImplGraph &graph, const ascir::FusedScheduledResult &fused_schedule_result, |
| 2169 | Kernel &kernel) { | 2204 | Kernel &kernel) { |
| 2170 | // Parse kernel input output | 2205 | // Parse kernel input output |
| @@ -2359,16 +2394,24 @@ Status Kernel::ParseGraph(const ascir::ImplGraph &graph, const ascir::FusedSched | |||
| 2359 | } | 2394 | } |
| 2360 | auto it = kernel.tpipe.bufs.find(tmp_buffer.id); | 2395 | auto it = kernel.tpipe.bufs.find(tmp_buffer.id); |
| 2361 | GELOGD("reuse tmp buffer id is %ld.", tmp_buffer.id); | 2396 | GELOGD("reuse tmp buffer id is %ld.", tmp_buffer.id); |
| 2397 | + // CV UBFuse 使用无 v1 repeat cap 的原始 tmp 需求; 其余路径(含 VV 的 host tiling | ||
| 2398 | + // 求解)保持 capped size 原样, 不引入额外拷贝。 | ||
| 2399 | + af::Expression stripped; | ||
| 2400 | + const af::Expression *tmp_size = &tmp_buffer.buf_desc.size; | ||
| 2401 | + if (kernel.tpipe.cv_fusion_type == ascir::CubeTemplateType::kUBFuse) { | ||
| 2402 | + stripped = StripV1RepeatCap(tmp_buffer.buf_desc.size); | ||
| 2403 | + tmp_size = &stripped; | ||
| 2404 | + } | ||
| 2362 | if (it == kernel.tpipe.bufs.end()) { | 2405 | if (it == kernel.tpipe.bufs.end()) { |
| 2363 | std::string position = "TPosition::VECCALC"; | 2406 | std::string position = "TPosition::VECCALC"; |
| 2364 | ascir::Position tensor_position = af::Position::kPositionVecCalc; | 2407 | ascir::Position tensor_position = af::Position::kPositionVecCalc; |
| 2365 | auto [new_buf, is_insert5] = | 2408 | auto [new_buf, is_insert5] = |
| 2366 | kernel.tpipe.bufs.emplace(tmp_buffer.id, TBuf{tmp_buffer.id, tensor_position, position}); | 2409 | kernel.tpipe.bufs.emplace(tmp_buffer.id, TBuf{tmp_buffer.id, tensor_position, position}); |
| 2367 | GE_CHK_BOOL_RET_STATUS(is_insert5, af::FAILED, "Codegen emplace tbuf [%ld] failed", tmp_buffer.id); | 2410 | GE_CHK_BOOL_RET_STATUS(is_insert5, af::FAILED, "Codegen emplace tbuf [%ld] failed", tmp_buffer.id); |
| 2368 | - new_buf->second.tmp_buf_size_list.emplace_back(tmp_buffer.buf_desc.size); | 2411 | + new_buf->second.tmp_buf_size_list.emplace_back(*tmp_size); |
| 2369 | new_buf->second.tmp_buf_reuse = true; | 2412 | new_buf->second.tmp_buf_reuse = true; |
| 2370 | } else { | 2413 | } else { |
| 2371 | - it->second.tmp_buf_size_list.emplace_back(tmp_buffer.buf_desc.size); | 2414 | + it->second.tmp_buf_size_list.emplace_back(*tmp_size); |
| 2372 | it->second.tmp_buf_reuse = true; | 2415 | it->second.tmp_buf_reuse = true; |
| 2373 | } | 2416 | } |
| 2374 | } | 2417 | } |
| @@ -3944,6 +3987,7 @@ std::string TPipe::TensorSizeDefine() const { | |||
| 3944 | 3987 | ||
| 3945 | Status TPipe::TensorSizeAssign(std::string dtype_name, std::string &result) const { | 3988 | Status TPipe::TensorSizeAssign(std::string dtype_name, std::string &result) const { |
| 3946 | stringstream ss; | 3989 | stringstream ss; |
| 3990 | + (void)dtype_name; | ||
| 3947 | 3991 | ||
| 3948 | for (auto &pair : this->tensors) { | 3992 | for (auto &pair : this->tensors) { |
| 3949 | auto &t = pair.second; | 3993 | auto &t = pair.second; |
| @@ -3954,7 +3998,10 @@ Status TPipe::TensorSizeAssign(std::string dtype_name, std::string &result) cons | |||
| 3954 | static_cast<int32_t>(t.dtype)); | 3998 | static_cast<int32_t>(t.dtype)); |
| 3955 | ss << t.size.Str() << " = KernelUtils::BlkAlign<" << tensor_dtype_name << ">(1);" << std::endl; | 3999 | ss << t.size.Str() << " = KernelUtils::BlkAlign<" << tensor_dtype_name << ">(1);" << std::endl; |
| 3956 | } else { | 4000 | } else { |
| 3957 | - ss << t.size.Str() << " = stage_size / sizeof(" << dtype_name << ");" << std::endl; | 4001 | + // stage_size(元素个数)已归一: 主路径 = 求解器预算a0a1t(=行数*basen_align, 任意dtype的32B行对齐 |
| 4002 | + // 行宽均<=basen_align, 按此元素数分配即覆盖compute size且不超求解器预算); | ||
| 4003 | + // 小a0a1t兜底时 = cube_ub元素数(容量, 更保守). CV与非CV统一使用stage_size. | ||
| 4004 | + ss << t.size.Str() << " = stage_size;" << std::endl; | ||
| 3958 | } | 4005 | } |
| 3959 | } | 4006 | } |
| 3960 | } | 4007 | } |
| @@ -4101,6 +4148,7 @@ class AutoFusionVector { | |||
| 4101 | if (is_inductor) { | 4148 | if (is_inductor) { |
| 4102 | result << " uint32_t stage_size_name{0};" << std::endl; | 4149 | result << " uint32_t stage_size_name{0};" << std::endl; |
| 4103 | result << " uint32_t cube_ub_stage_size{0};" << std::endl; | 4150 | result << " uint32_t cube_ub_stage_size{0};" << std::endl; |
| 4151 | + result << " uint32_t cube_m_stage_size{0};" << std::endl; | ||
| 4104 | } else { | 4152 | } else { |
| 4105 | result << " const CVAutofuseTilingData *cv_tiling_data{nullptr};" << std::endl; | 4153 | result << " const CVAutofuseTilingData *cv_tiling_data{nullptr};" << std::endl; |
| 4106 | } | 4154 | } |
| @@ -4119,6 +4167,7 @@ class AutoFusionVector { | |||
| 4119 | if (is_inductor) { | 4167 | if (is_inductor) { |
| 4120 | result << " uint32_t stage_size_name{0};" << std::endl; | 4168 | result << " uint32_t stage_size_name{0};" << std::endl; |
| 4121 | result << " uint32_t cube_ub_stage_size{0};" << std::endl; | 4169 | result << " uint32_t cube_ub_stage_size{0};" << std::endl; |
| 4170 | + result << " uint32_t cube_m_stage_size{0};" << std::endl; | ||
| 4122 | } else { | 4171 | } else { |
| 4123 | result << " const CVAutofuseTilingData *cv_tiling_data{nullptr};" << std::endl; | 4172 | result << " const CVAutofuseTilingData *cv_tiling_data{nullptr};" << std::endl; |
| 4124 | } | 4173 | } |
| @@ -4135,6 +4184,7 @@ class AutoFusionVector { | |||
| 4135 | if (is_inductor) { | 4184 | if (is_inductor) { |
| 4136 | result << " uint32_t stage_size_name{0};" << std::endl; | 4185 | result << " uint32_t stage_size_name{0};" << std::endl; |
| 4137 | result << " uint32_t cube_ub_stage_size{0};" << std::endl; | 4186 | result << " uint32_t cube_ub_stage_size{0};" << std::endl; |
| 4187 | + result << " uint32_t cube_m_stage_size{0};" << std::endl; | ||
| 4138 | } else { | 4188 | } else { |
| 4139 | result << " const CVAutofuseTilingData *cv_tiling_data{nullptr};" << std::endl; | 4189 | result << " const CVAutofuseTilingData *cv_tiling_data{nullptr};" << std::endl; |
| 4140 | } | 4190 | } |
| @@ -4219,7 +4269,7 @@ class AutoFusionVector { | |||
| 4219 | result << "const uint32_t stage_size_name = autofuse_tiling_size.STAGE_SIZE_NAME;" << std::endl; | 4269 | result << "const uint32_t stage_size_name = autofuse_tiling_size.STAGE_SIZE_NAME;" << std::endl; |
| 4220 | } | 4270 | } |
| 4221 | result << "const int32_t compute_size = stage_size_name > 144 ? stage_size_name : 144;" << std::endl; | 4271 | result << "const int32_t compute_size = stage_size_name > 144 ? stage_size_name : 144;" << std::endl; |
| 4222 | - result << "int32_t stage_size = compute_size * sizeof(" << dtype_name << ");" << std::endl; | 4272 | + result << "int32_t stage_size = compute_size;" << std::endl; |
| 4223 | result << "stage_size2 = compute_size;" << std::endl; | 4273 | result << "stage_size2 = compute_size;" << std::endl; |
| 4224 | } else { | 4274 | } else { |
| 4225 | result << "__aicore__ inline void Init(Params const& params, AscendC::LocalTensor<" << dtype_name | 4275 | result << "__aicore__ inline void Init(Params const& params, AscendC::LocalTensor<" << dtype_name |
| @@ -4231,27 +4281,42 @@ class AutoFusionVector { | |||
| 4231 | result << "const uint32_t stage_size_name = autofuse_tiling_size.STAGE_SIZE_NAME;" << std::endl; | 4281 | result << "const uint32_t stage_size_name = autofuse_tiling_size.STAGE_SIZE_NAME;" << std::endl; |
| 4232 | result << "const int32_t basen_basem_align = kConstTilingData.cube_ub_stage_size * sizeof(" << dtype_name << ");" | 4282 | result << "const int32_t basen_basem_align = kConstTilingData.cube_ub_stage_size * sizeof(" << dtype_name << ");" |
| 4233 | << std::endl; | 4283 | << std::endl; |
| 4284 | + result << "stage_size_type = static_cast<int64_t>(kConstTilingData.cube_m_stage_size);" << std::endl; | ||
| 4234 | result << "#else" << std::endl; | 4285 | result << "#else" << std::endl; |
| 4235 | result << "const uint32_t stage_size_name = params.stage_size_name;" << std::endl; | 4286 | result << "const uint32_t stage_size_name = params.stage_size_name;" << std::endl; |
| 4236 | result << "const int32_t basen_basem_align = params.cube_ub_stage_size * sizeof(" << dtype_name << ");" | 4287 | result << "const int32_t basen_basem_align = params.cube_ub_stage_size * sizeof(" << dtype_name << ");" |
| 4237 | << std::endl; | 4288 | << std::endl; |
| 4289 | + result << "stage_size_type = static_cast<int64_t>(params.cube_m_stage_size);" << std::endl; | ||
| 4238 | result << "#endif" << std::endl; | 4290 | result << "#endif" << std::endl; |
| 4239 | } else { | 4291 | } else { |
| 4240 | result << "GET_TILING_DATA_WITH_STRUCT(MatMulV3BasicTilingData, tmpTilingData, tmpTilingGM);" << std::endl; | 4292 | result << "GET_TILING_DATA_WITH_STRUCT(MatMulV3BasicTilingData, tmpTilingData, tmpTilingGM);" << std::endl; |
| 4241 | - result << "AutofuseTilingData autofuse_tiling_size;" << std::endl; | 4293 | + result << "const int32_t ub_align_value = 32;" << std::endl; |
| 4242 | - result << "const int32_t ub_align_value = 32 / sizeof(" << dtype_name << ");" << std::endl; | ||
| 4243 | result | 4294 | result |
| 4244 | << "const int32_t basen_align = (tmpTilingData.baseN + ub_align_value - 1) / ub_align_value * ub_align_value;" | 4295 | << "const int32_t basen_align = (tmpTilingData.baseN + ub_align_value - 1) / ub_align_value * ub_align_value;" |
| 4245 | << std::endl; | 4296 | << std::endl; |
| 4246 | result << "const int32_t basen_basem_align = (tmpTilingData.baseM * basen_align * sizeof(" << dtype_name | 4297 | result << "const int32_t basen_basem_align = (tmpTilingData.baseM * basen_align * sizeof(" << dtype_name |
| 4247 | << ")) / 2 + basen_align * sizeof(" << dtype_name << ");" << std::endl; | 4298 | << ")) / 2 + basen_align * sizeof(" << dtype_name << ");" << std::endl; |
| 4248 | - result << "const uint32_t stage_size_name = autofuse_tiling_size.STAGE_SIZE_NAME;" << std::endl; | 4299 | + // 非inductor: ascbc静态流程在device头烧录kConstTilingData(求解器完整结果); |
| 4300 | + // 动态shape时params携带cv_tiling_data指针读GM tiling; TF等前端在线GE编译的 | ||
| 4301 | + // 静态图两者皆无, 回退到autofuse_tiling_data.h中烧录的求解器结果 | ||
| 4302 | + // (z0z1t预算), 行数按 预算/上确界行宽 就地推导, 与conv2d分支同一模式 | ||
| 4303 | + result << "#ifdef INDUCTOR_CONST_TILING_DATA" << std::endl; | ||
| 4304 | + result << "stage_size_type = static_cast<int64_t>(kConstTilingData.cube_m_stage_size);" << std::endl; | ||
| 4305 | + result << "const uint32_t stage_size_name = kConstTilingData.stage_size_name;" << std::endl; | ||
| 4306 | + result << "#else" << std::endl; | ||
| 4307 | + if (is_dynamic) { | ||
| 4308 | + result << "stage_size_type = static_cast<int64_t>(params.cv_tiling_data->cube_m_stage_size);" << std::endl; | ||
| 4309 | + result << "const uint32_t stage_size_name = params.cv_tiling_data->stage_size_name;" << std::endl; | ||
| 4310 | + } else { | ||
| 4311 | + result << "AutofuseTilingData autofuse_tiling_size;" << std::endl; | ||
| 4312 | + result << "const uint32_t stage_size_name = autofuse_tiling_size.STAGE_SIZE_NAME;" << std::endl; | ||
| 4313 | + result << "stage_size_type = static_cast<int64_t>(stage_size_name / basen_align);" << std::endl; | ||
| 4314 | + } | ||
| 4315 | + result << "#endif" << std::endl; | ||
| 4249 | } | 4316 | } |
| 4250 | - // 下面的144为16*16/2+16，按照cube tiling最小块16*16计算得到 | 4317 | + // stage_size 统一为元素个数: 求解器预算 a0a1t, 过小时兜底到 C fragment 容量(cube_ub元素数) |
| 4251 | - result << "int32_t stage_size = stage_size_name > 144 ? " << std::endl; | 4318 | + result << "int32_t stage_size = stage_size_name > 144 ? stage_size_name : " << std::endl; |
| 4252 | - result << "stage_size_name * sizeof(" << dtype_name << ") : basen_basem_align;" << std::endl; | 4319 | + result << "static_cast<int32_t>(basen_basem_align / sizeof(" << dtype_name << "));" << std::endl; |
| 4253 | - result << "stage_size_type = static_cast<int64_t>(stage_size_name > 144 ? " << std::endl; | ||
| 4254 | - result << "stage_size_name : basen_basem_align / sizeof(" << dtype_name << "));" << std::endl; | ||
| 4255 | } | 4320 | } |
| 4256 | GE_CHK_STATUS_RET(this->root_loop.ActualSizeDefine(this->tiler, this->tpipe, dtype_name, tmp), | 4321 | GE_CHK_STATUS_RET(this->root_loop.ActualSizeDefine(this->tiler, this->tpipe, dtype_name, tmp), |
| 4257 | "actual size define failed"); | 4322 | "actual size define failed"); |
| @@ -4259,7 +4324,7 @@ class AutoFusionVector { | |||
| 4259 | if (is_conv2d) { | 4324 | if (is_conv2d) { |
| 4260 | result << ub_tensor->Str() << "_actual_size = stage_size2;" << std::endl << std::endl; | 4325 | result << ub_tensor->Str() << "_actual_size = stage_size2;" << std::endl << std::endl; |
| 4261 | } else { | 4326 | } else { |
| 4262 | - result << ub_tensor->Str() << "_actual_size = stage_size_type;" << std::endl << std::endl; | 4327 | + result << ub_tensor->Str() << "_actual_size = stage_size;" << std::endl << std::endl; |
| 4263 | } | 4328 | } |
| 4264 | GE_CHK_STATUS_RET(this->tpipe.TensorSizeAssign(dtype_name, tmp), "Tensor size assign failed"); | 4329 | GE_CHK_STATUS_RET(this->tpipe.TensorSizeAssign(dtype_name, tmp), "Tensor size assign failed"); |
| 4265 | result << tmp; | 4330 | result << tmp; |
| @@ -4275,9 +4340,9 @@ class AutoFusionVector { | |||
| 4275 | if (is_conv2d) { | 4340 | if (is_conv2d) { |
| 4276 | result << "GetTPipePtr()->InitBuffer(buf_cube, stage_size1 * sizeof(" << dtype_name << "));" << std::endl; | 4341 | result << "GetTPipePtr()->InitBuffer(buf_cube, stage_size1 * sizeof(" << dtype_name << "));" << std::endl; |
| 4277 | } else { | 4342 | } else { |
| 4278 | - result << "const uint32_t cv_ub_vector_queue_size = KernelUtils::BlkAlign<uint8_t>(stage_size);" << std::endl; | 4343 | + // 行语义下 epilogue 读取 cLocal_ 上界 stageOffset+stageSize <= inputSize <= cube_ub, |
| 4279 | - result << "tpipe.InitBuffer(buf_cube, KernelUtils::Max(static_cast<uint32_t>(basen_basem_align), " | 4344 | + // buf_cube 只需 C fragment 容量 basen_basem_align, 不再按 stage_size 撑大 |
| 4280 | - << "cv_ub_vector_queue_size));" << std::endl; | 4345 | + result << "tpipe.InitBuffer(buf_cube, static_cast<uint32_t>(basen_basem_align));" << std::endl; |
| 4281 | } | 4346 | } |
| 4282 | result << ub_tensor->name << " = buf_cube.Get<" << dtype_name << ">();" << std::endl; | 4347 | result << ub_tensor->name << " = buf_cube.Get<" << dtype_name << ">();" << std::endl; |
| 4283 | result << "cLocal = " << ub_tensor->name << ";" << std::endl << std::endl; | 4348 | result << "cLocal = " << ub_tensor->name << ";" << std::endl << std::endl; |
| @@ -4437,9 +4502,11 @@ Status Kernel::InitCVFusionAddr(std::stringstream &result, bool vector_no_db_fla | |||
| 4437 | result << "#ifdef INDUCTOR_CONST_TILING_DATA" << std::endl; | 4502 | result << "#ifdef INDUCTOR_CONST_TILING_DATA" << std::endl; |
| 4438 | result << " CV_FUSION_ADDR.stage_size_name = kConstTilingData.stage_size_name;" << std::endl; | 4503 | result << " CV_FUSION_ADDR.stage_size_name = kConstTilingData.stage_size_name;" << std::endl; |
| 4439 | result << " CV_FUSION_ADDR.cube_ub_stage_size = kConstTilingData.cube_ub_stage_size;" << std::endl; | 4504 | result << " CV_FUSION_ADDR.cube_ub_stage_size = kConstTilingData.cube_ub_stage_size;" << std::endl; |
| 4505 | + result << " CV_FUSION_ADDR.cube_m_stage_size = kConstTilingData.cube_m_stage_size;" << std::endl; | ||
| 4440 | result << "#else" << std::endl; | 4506 | result << "#else" << std::endl; |
| 4441 | result << " CV_FUSION_ADDR.stage_size_name = t.stage_size_name;" << std::endl; | 4507 | result << " CV_FUSION_ADDR.stage_size_name = t.stage_size_name;" << std::endl; |
| 4442 | result << " CV_FUSION_ADDR.cube_ub_stage_size = t.cube_ub_stage_size;" << std::endl; | 4508 | result << " CV_FUSION_ADDR.cube_ub_stage_size = t.cube_ub_stage_size;" << std::endl; |
| 4509 | + result << " CV_FUSION_ADDR.cube_m_stage_size = t.cube_m_stage_size;" << std::endl; | ||
| 4443 | result << "#endif" << std::endl; | 4510 | result << "#endif" << std::endl; |
| 4444 | result << " }" << std::endl; | 4511 | result << " }" << std::endl; |
| 4445 | } else { | 4512 | } else { |
| @@ -4464,9 +4531,11 @@ Status Kernel::InitCVFusionAddr(std::stringstream &result, bool vector_no_db_fla | |||
| 4464 | result << "#ifdef INDUCTOR_CONST_TILING_DATA" << std::endl; | 4531 | result << "#ifdef INDUCTOR_CONST_TILING_DATA" << std::endl; |
| 4465 | result << " CV_FUSION_ADDR_DB.stage_size_name = kConstTilingData.stage_size_name;" << std::endl; | 4532 | result << " CV_FUSION_ADDR_DB.stage_size_name = kConstTilingData.stage_size_name;" << std::endl; |
| 4466 | result << " CV_FUSION_ADDR_DB.cube_ub_stage_size = kConstTilingData.cube_ub_stage_size;" << std::endl; | 4533 | result << " CV_FUSION_ADDR_DB.cube_ub_stage_size = kConstTilingData.cube_ub_stage_size;" << std::endl; |
| 4534 | + result << " CV_FUSION_ADDR_DB.cube_m_stage_size = kConstTilingData.cube_m_stage_size;" << std::endl; | ||
| 4467 | result << "#else" << std::endl; | 4535 | result << "#else" << std::endl; |
| 4468 | result << " CV_FUSION_ADDR_DB.stage_size_name = t.stage_size_name;" << std::endl; | 4536 | result << " CV_FUSION_ADDR_DB.stage_size_name = t.stage_size_name;" << std::endl; |
| 4469 | result << " CV_FUSION_ADDR_DB.cube_ub_stage_size = t.cube_ub_stage_size;" << std::endl; | 4537 | result << " CV_FUSION_ADDR_DB.cube_ub_stage_size = t.cube_ub_stage_size;" << std::endl; |
| 4538 | + result << " CV_FUSION_ADDR_DB.cube_m_stage_size = t.cube_m_stage_size;" << std::endl; | ||
| 4470 | result << "#endif" << std::endl; | 4539 | result << "#endif" << std::endl; |
| 4471 | result << " }" << std::endl; | 4540 | result << " }" << std::endl; |
| 4472 | } else { | 4541 | } else { |
| @@ -1032,6 +1032,7 @@ Status Loop::Generate(const Tiler &tiler, const TPipe &tpipe, std::string &resul | |||
| 1032 | 1032 | ||
| 1033 | Status Loop::ActualSizeDefine(const Tiler &tiler, const TPipe &tpipe, std::string dtype_name, std::string &result) { | 1033 | Status Loop::ActualSizeDefine(const Tiler &tiler, const TPipe &tpipe, std::string dtype_name, std::string &result) { |
| 1034 | std::stringstream ss; | 1034 | std::stringstream ss; |
| 1035 | + (void)dtype_name; | ||
| 1035 | if (this->axis_id == af::kIdNone) { | 1036 | if (this->axis_id == af::kIdNone) { |
| 1036 | for (const auto &body : this->bodys) { | 1037 | for (const auto &body : this->bodys) { |
| 1037 | if (body.type == LoopType::LOOP) { | 1038 | if (body.type == LoopType::LOOP) { |
| @@ -1045,7 +1046,7 @@ Status Loop::ActualSizeDefine(const Tiler &tiler, const TPipe &tpipe, std::strin | |||
| 1045 | const auto &tile_inner = tiler.GetAxis(axis.split_pair_other_id); | 1046 | const auto &tile_inner = tiler.GetAxis(axis.split_pair_other_id); |
| 1046 | af::Expression actual_size = af::Symbol(tile_inner.actual_size.name.c_str()); | 1047 | af::Expression actual_size = af::Symbol(tile_inner.actual_size.name.c_str()); |
| 1047 | tpipe.tiler.actual_sizes.emplace_back(std::make_pair(tile_inner.size_expr, actual_size)); | 1048 | tpipe.tiler.actual_sizes.emplace_back(std::make_pair(tile_inner.size_expr, actual_size)); |
| 1048 | - ss << tile_inner.actual_size.AsArg() << " = stage_size / sizeof(" << dtype_name << ");" << std::endl; | 1049 | + ss << tile_inner.actual_size.AsArg() << " = stage_size;" << std::endl; |
| 1049 | } | 1050 | } |
| 1050 | result = ss.str(); | 1051 | result = ss.str(); |
| 1051 | return af::SUCCESS; | 1052 | return af::SUCCESS; |
| @@ -48,7 +48,7 @@ using namespace ascgen_utils; | |||
| 48 | namespace { | 48 | namespace { |
| 49 | constexpr uint64_t kMaxPgoTilingKeyCount = 10000U; | 49 | constexpr uint64_t kMaxPgoTilingKeyCount = 10000U; |
| 50 | constexpr uint64_t kInt64TilingKeyCapacity = static_cast<uint64_t>(std::numeric_limits<int64_t>::max()) + 1U; | 50 | constexpr uint64_t kInt64TilingKeyCapacity = static_cast<uint64_t>(std::numeric_limits<int64_t>::max()) + 1U; |
| 51 | -// fp32且K轴大于该阈值且未启用hf32时, CV融合UB模板存在精度问题, 需走common兜底模板 | 51 | +// fp32且K轴不小于该阈值且未启用hf32时, CV融合UB模板存在精度问题, 需走common兜底模板 |
| 52 | constexpr int64_t kFp32LargeKThreshold = 2048; | 52 | constexpr int64_t kFp32LargeKThreshold = 2048; |
| 53 | 53 | ||
| 54 | std::string GenUint64Literal(uint64_t value) { | 54 | std::string GenUint64Literal(uint64_t value) { |
| @@ -100,8 +100,14 @@ void GenInductorCvSafetyFallback(std::stringstream &ss, uint64_t count, const st | |||
| 100 | } | 100 | } |
| 101 | 101 | ||
| 102 | // 生成cv_tiling_data字段重置代码, ub_mode区分UB模板(1)与common模板(0) | 102 | // 生成cv_tiling_data字段重置代码, ub_mode区分UB模板(1)与common模板(0) |
| 103 | +// 仅在求解器成功选出STAGE_SIZE_NAME(a0a1t)后调用: 此时行数按预算覆盖为 | ||
| 104 | +// STAGE_SIZE_NAME / basen_align, kernel侧队列申请与vector计算长度均以此为准 | ||
| 103 | void GenCvTilingDataReset(std::stringstream &ss, const std::string &indent, uint32_t ub_mode) { | 105 | void GenCvTilingDataReset(std::stringstream &ss, const std::string &indent, uint32_t ub_mode) { |
| 104 | ss << indent << "tiling->stage_size_name = tiling->tiling_data.STAGE_SIZE_NAME;" << std::endl; | 106 | ss << indent << "tiling->stage_size_name = tiling->tiling_data.STAGE_SIZE_NAME;" << std::endl; |
| 107 | + ss << indent | ||
| 108 | + << "tiling->cube_m_stage_size = (uint32_t)std::max(1, static_cast<int32_t>(tiling->stage_size_name / " | ||
| 109 | + "basen_align));" | ||
| 110 | + << std::endl; | ||
| 105 | ss << indent << "tiling->cv_tiling_data.fusion_mode = 0;" << std::endl; | 111 | ss << indent << "tiling->cv_tiling_data.fusion_mode = 0;" << std::endl; |
| 106 | ss << indent << "tiling->cv_tiling_data.ub_mode = " << ub_mode << ";" << std::endl; | 112 | ss << indent << "tiling->cv_tiling_data.ub_mode = " << ub_mode << ";" << std::endl; |
| 107 | ss << indent << "tiling->cv_tiling_data.mix_mode = 0;" << std::endl; | 113 | ss << indent << "tiling->cv_tiling_data.mix_mode = 0;" << std::endl; |
| @@ -1138,14 +1144,14 @@ std::string TilingLib::GenFp32LargeKCondition(const MatMulCubeInfo &cube_info) c | |||
| 1138 | std::string k_str = std::string(k_expr.Str().get()); | 1144 | std::string k_str = std::string(k_expr.Str().get()); |
| 1139 | try { | 1145 | try { |
| 1140 | int64_t k_value = std::stoll(k_str); | 1146 | int64_t k_value = std::stoll(k_str); |
| 1141 | - return k_value > kFp32LargeKThreshold ? "true" : "false"; | 1147 | + return k_value >= kFp32LargeKThreshold ? "true" : "false"; |
| 1142 | } catch (...) { | 1148 | } catch (...) { |
| 1143 | return "false"; | 1149 | return "false"; |
| 1144 | } | 1150 | } |
| 1145 | } | 1151 | } |
| 1146 | // K轴为动态变量: 生成运行时判断 | 1152 | // K轴为动态变量: 生成运行时判断 |
| 1147 | std::string k_var = std::string(k_expr.Str().get()); | 1153 | std::string k_var = std::string(k_expr.Str().get()); |
| 1148 | - return "(static_cast<int64_t>(" + k_var + ") > " + std::to_string(kFp32LargeKThreshold) + ")"; | 1154 | + return "(static_cast<int64_t>(" + k_var + ") >= " + std::to_string(kFp32LargeKThreshold) + ")"; |
| 1149 | } | 1155 | } |
| 1150 | 1156 | ||
| 1151 | std::string TilingLib::GenCubeFusionTilingBodyInductor(const ascir::FusedScheduledResult &fused_schedule_result, | 1157 | std::string TilingLib::GenCubeFusionTilingBodyInductor(const ascir::FusedScheduledResult &fused_schedule_result, |
| @@ -1165,11 +1171,15 @@ std::string TilingLib::GenCubeFusionTilingBodyInductor(const ascir::FusedSchedul | |||
| 1165 | << std::endl; | 1171 | << std::endl; |
| 1166 | ss << " tiling->cube_tiling_key = cube_tiling_key;" << std::endl; | 1172 | ss << " tiling->cube_tiling_key = cube_tiling_key;" << std::endl; |
| 1167 | ss << " int64_t cube_tiling_key_ub = cube_tiling_key & ~0xF0;" << std::endl; | 1173 | ss << " int64_t cube_tiling_key_ub = cube_tiling_key & ~0xF0;" << std::endl; |
| 1168 | - ss << " const int32_t ub_align_value = 32 / " << cube_info.type_size << ";" << std::endl; | 1174 | + // 32元素对齐: 最坏dtype(bool)的32B行对齐行宽也是basen_align, 保证按行宽申请的 |
| 1175 | + // queue tensor在任何dtype下都不超过求解器按STAGE_SIZE_NAME给出的UB元素预算 | ||
| 1176 | + ss << " const int32_t ub_align_value = 32;" << std::endl; | ||
| 1169 | ss << " const int32_t basen_align = (basen + ub_align_value - 1) / ub_align_value * ub_align_value;" << std::endl; | 1177 | ss << " const int32_t basen_align = (basen + ub_align_value - 1) / ub_align_value * ub_align_value;" << std::endl; |
| 1170 | ss << " const int32_t basen_basem_align = (basem * basen_align) / 2 + basen_align;" << std::endl; | 1178 | ss << " const int32_t basen_basem_align = (basem * basen_align) / 2 + basen_align;" << std::endl; |
| 1171 | ss << " set_g_basen_basem_align(basen_basem_align);" << std::endl; | 1179 | ss << " set_g_basen_basem_align(basen_basem_align);" << std::endl; |
| 1172 | ss << " tiling->cube_ub_stage_size = (uint32_t)basen_basem_align;" << std::endl; | 1180 | ss << " tiling->cube_ub_stage_size = (uint32_t)basen_basem_align;" << std::endl; |
| 1181 | + // 兜底默认值(容量行数); 求解成功后在GenCvTilingDataReset中按预算行数覆盖 | ||
| 1182 | + ss << " tiling->cube_m_stage_size = (uint32_t)(basen_basem_align / basen_align);" << std::endl; | ||
| 1173 | ss << " tiling->tiling_data.set_block_dim(limit->aiv_num);" << std::endl; | 1183 | ss << " tiling->tiling_data.set_block_dim(limit->aiv_num);" << std::endl; |
| 1174 | ss << " tiling->tiling_data.set_ub_size(limit->ub_size - 256);" << std::endl; | 1184 | ss << " tiling->tiling_data.set_ub_size(limit->ub_size - 256);" << std::endl; |
| 1175 | 1185 | ||
| @@ -1530,6 +1540,7 @@ static std::string GenLocalMemorySizeCode() { | |||
| 1530 | 1540 | ||
| 1531 | static void AppendCubeFusionInitCode(std::stringstream &ss, const std::string &shape_dim_param, | 1541 | static void AppendCubeFusionInitCode(std::stringstream &ss, const std::string &shape_dim_param, |
| 1532 | const MatMulCubeInfo &cube_info) { | 1542 | const MatMulCubeInfo &cube_info) { |
| 1543 | + (void)cube_info; | ||
| 1533 | ss << " auto tiling_data = context->GetTilingData<CVAutofuseTilingData>();" << std::endl; | 1544 | ss << " auto tiling_data = context->GetTilingData<CVAutofuseTilingData>();" << std::endl; |
| 1534 | ss << " int64_t ws_size = 0;" << std::endl; | 1545 | ss << " int64_t ws_size = 0;" << std::endl; |
| 1535 | ss << " int64_t cube_tiling_key = 0;" << std::endl; | 1546 | ss << " int64_t cube_tiling_key = 0;" << std::endl; |
| @@ -1542,10 +1553,11 @@ static void AppendCubeFusionInitCode(std::stringstream &ss, const std::string &s | |||
| 1542 | ss << " ResLimit limit;" << std::endl << " limit.aiv_num = parse->aiv_num;" << std::endl; | 1553 | ss << " ResLimit limit;" << std::endl << " limit.aiv_num = parse->aiv_num;" << std::endl; |
| 1543 | ss << " limit.ub_size = (uint32_t)parse->ub_size;" << std::endl; | 1554 | ss << " limit.ub_size = (uint32_t)parse->ub_size;" << std::endl; |
| 1544 | ss << " auto ret = ge::GRAPH_SUCCESS;" << std::endl; | 1555 | ss << " auto ret = ge::GRAPH_SUCCESS;" << std::endl; |
| 1545 | - ss << " const int32_t ub_align_value = 32 / " << cube_info.type_size << ";" << std::endl; | 1556 | + ss << " const int32_t ub_align_value = 32;" << std::endl; |
| 1546 | ss << " const int32_t basen_align = (basen + ub_align_value - 1) / ub_align_value * ub_align_value;" << std::endl; | 1557 | ss << " const int32_t basen_align = (basen + ub_align_value - 1) / ub_align_value * ub_align_value;" << std::endl; |
| 1547 | ss << " const int32_t basen_basem_align = (basem * basen_align) / 2 + basen_align;" << std::endl; | 1558 | ss << " const int32_t basen_basem_align = (basem * basen_align) / 2 + basen_align;" << std::endl; |
| 1548 | ss << " tiling_data->cube_ub_stage_size = (uint32_t)basen_basem_align;" << std::endl; | 1559 | ss << " tiling_data->cube_ub_stage_size = (uint32_t)basen_basem_align;" << std::endl; |
| 1560 | + ss << " tiling_data->cube_m_stage_size = (uint32_t)(basen_basem_align / basen_align);" << std::endl; | ||
| 1549 | } | 1561 | } |
| 1550 | 1562 | ||
| 1551 | static void AppendCubeFusionUbModeCode(std::stringstream &ss) { | 1563 | static void AppendCubeFusionUbModeCode(std::stringstream &ss) { |
| @@ -1584,6 +1596,9 @@ static void AppendCubeFusionFallbackCode(std::stringstream &ss, const std::strin | |||
| 1584 | ss << "&(tiling_data->tiling_data), &workspace_size, &block_dim, &limit);" << std::endl; | 1596 | ss << "&(tiling_data->tiling_data), &workspace_size, &block_dim, &limit);" << std::endl; |
| 1585 | ss << " if (ret == 0) {" << std::endl; | 1597 | ss << " if (ret == 0) {" << std::endl; |
| 1586 | ss << " tiling_data->stage_size_name = tiling_data->tiling_data.STAGE_SIZE_NAME;" << std::endl; | 1598 | ss << " tiling_data->stage_size_name = tiling_data->tiling_data.STAGE_SIZE_NAME;" << std::endl; |
| 1599 | + ss << " tiling_data->cube_m_stage_size = (uint32_t)std::max(1, static_cast<int32_t>(tiling_data->" | ||
| 1600 | + "stage_size_name / basen_align));" | ||
| 1601 | + << std::endl; | ||
| 1587 | ss << " context->SetBlockDim(cube_block_dim);" << std::endl; | 1602 | ss << " context->SetBlockDim(cube_block_dim);" << std::endl; |
| 1588 | ss << " *context->GetWorkspaceSizes(1) = 16 * 1024 * 1024 + ws_size;" << std::endl; | 1603 | ss << " *context->GetWorkspaceSizes(1) = 16 * 1024 * 1024 + ws_size;" << std::endl; |
| 1589 | ss << " tiling_data->cv_tiling_data.fusion_mode = 0;" << std::endl; | 1604 | ss << " tiling_data->cv_tiling_data.fusion_mode = 0;" << std::endl; |
| @@ -1599,6 +1614,9 @@ static void AppendCubeFusionFallbackCode(std::stringstream &ss, const std::strin | |||
| 1599 | ss << " tiling_data->tiling_data.set_ub_size(limit.ub_size - 256);" << std::endl; | 1614 | ss << " tiling_data->tiling_data.set_ub_size(limit.ub_size - 256);" << std::endl; |
| 1600 | ss << " if (!optiling::GetTiling(tiling_data->tiling_data, 1)) {return ge::GRAPH_FAILED;}" << std::endl; | 1615 | ss << " if (!optiling::GetTiling(tiling_data->tiling_data, 1)) {return ge::GRAPH_FAILED;}" << std::endl; |
| 1601 | ss << " tiling_data->stage_size_name = tiling_data->tiling_data.STAGE_SIZE_NAME;" << std::endl; | 1616 | ss << " tiling_data->stage_size_name = tiling_data->tiling_data.STAGE_SIZE_NAME;" << std::endl; |
| 1617 | + ss << " tiling_data->cube_m_stage_size = (uint32_t)std::max(1, static_cast<int32_t>(tiling_data->" | ||
| 1618 | + "stage_size_name / basen_align));" | ||
| 1619 | + << std::endl; | ||
| 1602 | ss << " context->SetBlockDim(cube_block_dim);" << std::endl; | 1620 | ss << " context->SetBlockDim(cube_block_dim);" << std::endl; |
| 1603 | ss << " *context->GetWorkspaceSizes(1) = 16 * 1024 * 1024 + ws_size;" << std::endl; | 1621 | ss << " *context->GetWorkspaceSizes(1) = 16 * 1024 * 1024 + ws_size;" << std::endl; |
| 1604 | ss << " tiling_data->cv_tiling_data.fusion_mode = 0;" << std::endl; | 1622 | ss << " tiling_data->cv_tiling_data.fusion_mode = 0;" << std::endl; |
| @@ -29,6 +29,7 @@ void AppendCVAutofuseCommonTilingFields(std::stringstream &ss) { | |||
| 29 | ss << " };\n"; | 29 | ss << " };\n"; |
| 30 | ss << " uint32_t stage_size_name;\n"; | 30 | ss << " uint32_t stage_size_name;\n"; |
| 31 | ss << " uint32_t cube_ub_stage_size;\n"; | 31 | ss << " uint32_t cube_ub_stage_size;\n"; |
| 32 | + ss << " uint32_t cube_m_stage_size;\n"; | ||
| 32 | } | 33 | } |
| 33 | } // namespace | 34 | } // namespace |
| 34 | 35 | ||
| @@ -263,6 +264,7 @@ std::string codegen::TilingData::Generate(const ascir::FusedScheduledResult &fus | |||
| 263 | << " static constexpr size_t kMatmulTilingBytes = CV_TILING_ALIGN_UP(sizeof(MatmulTilingT), 8);\n" | 264 | << " static constexpr size_t kMatmulTilingBytes = CV_TILING_ALIGN_UP(sizeof(MatmulTilingT), 8);\n" |
| 264 | << " uint32_t stage_size_name;\n" | 265 | << " uint32_t stage_size_name;\n" |
| 265 | << " uint32_t cube_ub_stage_size;\n" | 266 | << " uint32_t cube_ub_stage_size;\n" |
| 267 | + << " uint32_t cube_m_stage_size;\n" | ||
| 266 | << " alignas(8) uint8_t matmul_tiling_data[kMatmulTilingBytes];\n" | 268 | << " alignas(8) uint8_t matmul_tiling_data[kMatmulTilingBytes];\n" |
| 267 | << "};\n" | 269 | << "};\n" |
| 268 | << "template <typename MatmulTilingT>\n" | 270 | << "template <typename MatmulTilingT>\n" |
| @@ -273,6 +275,7 @@ std::string codegen::TilingData::Generate(const ascir::FusedScheduledResult &fus | |||
| 273 | << " CVTilingData cv_tiling_data;\n" | 275 | << " CVTilingData cv_tiling_data;\n" |
| 274 | << " uint32_t stage_size_name;\n" | 276 | << " uint32_t stage_size_name;\n" |
| 275 | << " uint32_t cube_ub_stage_size;\n" | 277 | << " uint32_t cube_ub_stage_size;\n" |
| 278 | + << " uint32_t cube_m_stage_size;\n" | ||
| 276 | << " alignas(8) uint8_t matmul_tiling_data[kMatmulTilingBytes];\n" | 279 | << " alignas(8) uint8_t matmul_tiling_data[kMatmulTilingBytes];\n" |
| 277 | << "};\n" | 280 | << "};\n" |
| 278 | << "struct CVAutofuseExternTilingData {\n" | 281 | << "struct CVAutofuseExternTilingData {\n" |
| @@ -284,6 +287,7 @@ std::string codegen::TilingData::Generate(const ascir::FusedScheduledResult &fus | |||
| 284 | 287 | ||
| 285 | ss << " uint32_t stage_size_name;\n"; | 288 | ss << " uint32_t stage_size_name;\n"; |
| 286 | ss << " uint32_t cube_ub_stage_size;\n"; | 289 | ss << " uint32_t cube_ub_stage_size;\n"; |
| 290 | + ss << " uint32_t cube_m_stage_size;\n"; | ||
| 287 | ss << " alignas(8) uint8_t matmul_tiling_data[MATMUL_TILING_DATA_STORAGE_SIZE];\n"; | 291 | ss << " alignas(8) uint8_t matmul_tiling_data[MATMUL_TILING_DATA_STORAGE_SIZE];\n"; |
| 288 | 292 | ||
| 289 | ss << "};\n" | 293 | ss << "};\n" |
| @@ -707,7 +711,9 @@ std::string codegen::TilingData::GenCVConstReplace(const std::string &tiling_dat | |||
| 707 | ss << " k_const_tiling_data_ss << std::to_string((uint32_t)" << tiling_data_struct_name | 711 | ss << " k_const_tiling_data_ss << std::to_string((uint32_t)" << tiling_data_struct_name |
| 708 | << ".stage_size_name) << \", \";" << std::endl; | 712 | << ".stage_size_name) << \", \";" << std::endl; |
| 709 | ss << " k_const_tiling_data_ss << std::to_string((uint32_t)" << tiling_data_struct_name | 713 | ss << " k_const_tiling_data_ss << std::to_string((uint32_t)" << tiling_data_struct_name |
| 710 | - << ".cube_ub_stage_size) << \", {\";" << std::endl; | 714 | + << ".cube_ub_stage_size) << \", \";" << std::endl; |
| 715 | + ss << " k_const_tiling_data_ss << std::to_string((uint32_t)" << tiling_data_struct_name | ||
| 716 | + << ".cube_m_stage_size) << \", {\";" << std::endl; | ||
| 711 | ss << " k_const_tiling_data_ss << matmul_bytes << \"}};\";" << std::endl; | 717 | ss << " k_const_tiling_data_ss << matmul_bytes << \"}};\";" << std::endl; |
| 712 | ss << " replaceSubstring(tiling_data_const_gen_result, " << std::endl; | 718 | ss << " replaceSubstring(tiling_data_const_gen_result, " << std::endl; |
| 713 | ss << " \"const CVAutofuseTilingData kConstTilingData = {};\", " << std::endl; | 719 | ss << " \"const CVAutofuseTilingData kConstTilingData = {};\", " << std::endl; |
| @@ -1759,7 +1759,7 @@ def is_matmul_relu_fixpip(tiling_info, cube_info): | |||
| 1759 | 1759 | ||
| 1760 | 1760 | ||
| 1761 | def _is_fp32_large_k(a_trans, a_shape, cube_output_type_size, enable_hf32): | 1761 | def _is_fp32_large_k(a_trans, a_shape, cube_output_type_size, enable_hf32): |
| 1762 | - """fp32类型且K轴>2048且未启用hf32时, CV融合不能选择UB模板(精度问题), 需走common兜底""" | 1762 | + """fp32类型且K轴>=2048且未启用hf32时, CV融合不能选择UB模板(精度问题), 需走common兜底""" |
| 1763 | if cube_output_type_size != 4: # 非fp32 | 1763 | if cube_output_type_size != 4: # 非fp32 |
| 1764 | return False | 1764 | return False |
| 1765 | if enable_hf32: # 启用hf32时不走common兜底 | 1765 | if enable_hf32: # 启用hf32时不走common兜底 |
| @@ -1770,7 +1770,7 @@ def _is_fp32_large_k(a_trans, a_shape, cube_output_type_size, enable_hf32): | |||
| 1770 | logger.info( | 1770 | logger.info( |
| 1771 | "CV fusion op, fp32 large K check, a_trans=%s, k_value=%s", a_trans, k_value | 1771 | "CV fusion op, fp32 large K check, a_trans=%s, k_value=%s", a_trans, k_value |
| 1772 | ) | 1772 | ) |
| 1773 | - return k_value > 2048 | 1773 | + return k_value >= 2048 |
| 1774 | 1774 | ||
| 1775 | 1775 | ||
| 1776 | def is_fp32_large_k_matmul(tiling_info, cube_info, enable_hf32=False): | 1776 | def is_fp32_large_k_matmul(tiling_info, cube_info, enable_hf32=False): |
| @@ -2024,7 +2024,9 @@ GET_TILING_DATA_PTR_WITH_STRUCT({struct_name}, tmpTilingData, tmpTilingGM); | |||
| 2024 | """ | 2024 | """ |
| 2025 | 2025 | ||
| 2026 | # 构建class_body，使用data_prefix处理不同的数据访问路径 | 2026 | # 构建class_body，使用data_prefix处理不同的数据访问路径 |
| 2027 | - class_body = "const int32_t ub_align_value = 32 / cube_output_type_size;\n" | 2027 | + # ub_align_value固定32元素: 与kernel侧de445b95的上确界行宽对齐(basen_align |
| 2028 | + # 为所有dtype的32B对齐行宽上确界), 保证host侧basen_basem_align与kernel一致 | ||
| 2029 | + class_body = "const int32_t ub_align_value = 32;\n" | ||
| 2028 | class_body += ( | 2030 | class_body += ( |
| 2029 | f"const int32_t basen_align = ({data_prefix}.baseN + ub_align_value - 1) " | 2031 | f"const int32_t basen_align = ({data_prefix}.baseN + ub_align_value - 1) " |
| 2030 | f"/ ub_align_value * ub_align_value;\n" | 2032 | f"/ ub_align_value * ub_align_value;\n" |
| @@ -1151,7 +1151,7 @@ TEST(CodegenKernel, TPipe_TensorSizeCalc_AllocFromQue) { | |||
| 1151 | "const uint32_t local_1_que_buf_num = 4;\n"}); | 1151 | "const uint32_t local_1_que_buf_num = 4;\n"}); |
| 1152 | } | 1152 | } |
| 1153 | 1153 | ||
| 1154 | -TEST(CodegenKernel, TPipe_TensorSizeAssignForCvUbFuseShouldUseCubeOutputElementCount) { | 1154 | +TEST(CodegenKernel, TPipe_TensorSizeAssignForCvUbFuseShouldUseStageSizeElements) { |
| 1155 | af::AscGraph graph("test"); | 1155 | af::AscGraph graph("test"); |
| 1156 | af::ascir_op::Data x("x", graph); | 1156 | af::ascir_op::Data x("x", graph); |
| 1157 | af::ascir_op::Data y("y", graph); | 1157 | af::ascir_op::Data y("y", graph); |
| @@ -1182,8 +1182,8 @@ TEST(CodegenKernel, TPipe_TensorSizeAssignForCvUbFuseShouldUseCubeOutputElementC | |||
| 1182 | 1182 | ||
| 1183 | std::string result; | 1183 | std::string result; |
| 1184 | ASSERT_EQ(tpipe.TensorSizeAssign("float", result), af::SUCCESS); | 1184 | ASSERT_EQ(tpipe.TensorSizeAssign("float", result), af::SUCCESS); |
| 1185 | - EXPECT_EQ(result, std::string{"local_1_size = stage_size / sizeof(float);\n" | 1185 | + EXPECT_EQ(result, std::string{"local_1_size = stage_size;\n" |
| 1186 | - "local_2_size = stage_size / sizeof(float);\n" | 1186 | + "local_2_size = stage_size;\n" |
| 1187 | "\n"}); | 1187 | "\n"}); |
| 1188 | } | 1188 | } |
| 1189 | 1189 | ||
| @@ -2686,7 +2686,7 @@ TEST_F(TestCodegenTiling, GenFp32LargeKConditionShouldReturnRuntimeExprWhenDynam | |||
| 2686 | af::AscGraph graph("dynamic_k_matmul"); | 2686 | af::AscGraph graph("dynamic_k_matmul"); |
| 2687 | CreateMatmulGraphForFp32LargeK(graph, 0, true, false, true, ge::DT_FLOAT); | 2687 | CreateMatmulGraphForFp32LargeK(graph, 0, true, false, true, ge::DT_FLOAT); |
| 2688 | auto cube_info = EXTRACT_CUBE_INFO(graph); | 2688 | auto cube_info = EXTRACT_CUBE_INFO(graph); |
| 2689 | - EXPECT_EQ(this->GenFp32LargeKCondition(cube_info), "(static_cast<int64_t>(k_var) > 2048)"); | 2689 | + EXPECT_EQ(this->GenFp32LargeKCondition(cube_info), "(static_cast<int64_t>(k_var) >= 2048)"); |
| 2690 | } | 2690 | } |
| 2691 | 2691 | ||
| 2692 | TEST_F(TestCodegenTiling, GenFp32LargeKConditionShouldReturnTrueWhenNoTransposeStaticKGT2048) { | 2692 | TEST_F(TestCodegenTiling, GenFp32LargeKConditionShouldReturnTrueWhenNoTransposeStaticKGT2048) { |
| @@ -119,10 +119,13 @@ void ExpectToIntCvStageUsesDtypeAwareStrides(const std::string &api_name, const | |||
| 119 | 119 | ||
| 120 | std::string result; | 120 | std::string result; |
| 121 | EXPECT_EQ(call.Generate(tpipe, std::vector<af::AxisId>{z0.id}, result), 0); | 121 | EXPECT_EQ(call.Generate(tpipe, std::vector<af::AxisId>{z0.id}, result), 0); |
| 122 | - EXPECT_EQ(result, "AscendC::Cast(local_1[0], local_0[0], " + round_mode + | 122 | + EXPECT_EQ(result, |
| 123 | - ", {ConvertToUint32(curAivM), ConvertToUint32(curAivN)}, " | 123 | + "local_1_actual_size = curAivM * ((curAivN + 8 - 1) / 8 * 8);\n" |
| 124 | - "{ConvertToUint32(((curAivN + 8 - 1) / 8 * 8)), ConvertToUint32(1)}, " | 124 | + "AscendC::Cast(local_1[0], local_0[0], " + |
| 125 | - "{ConvertToUint32(((curAivN + 16 - 1) / 16 * 16)), ConvertToUint32(1)});\n"); | 125 | + round_mode + |
| 126 | + ", {ConvertToUint32(curAivM), ConvertToUint32(curAivN)}, " | ||
| 127 | + "{ConvertToUint32(((curAivN + 8 - 1) / 8 * 8)), ConvertToUint32(1)}, " | ||
| 128 | + "{ConvertToUint32(((curAivN + 16 - 1) / 16 * 16)), ConvertToUint32(1)});\n"); | ||
| 126 | } | 129 | } |
| 127 | } // namespace | 130 | } // namespace |
| 128 | 131 | ||
| @@ -408,7 +411,7 @@ TEST(CastV2ApiCallTest, CastV2ApiCallCvUbFuseUsesDtypeAlignedStrides) { | |||
| 408 | 411 | ||
| 409 | std::string result; | 412 | std::string result; |
| 410 | call.Generate(tpipe, std::vector<af::AxisId>{z0.id}, result); | 413 | call.Generate(tpipe, std::vector<af::AxisId>{z0.id}, result); |
| 411 | - EXPECT_EQ(result, std::string{"local_1_actual_size = curAivM * curAivN;\n" | 414 | + EXPECT_EQ(result, std::string{"local_1_actual_size = curAivM * ((curAivN + 8 - 1) / 8 * 8);\n" |
| 412 | "CastExtend(local_1[0], local_0[0], {ConvertToUint32(curAivM), " | 415 | "CastExtend(local_1[0], local_0[0], {ConvertToUint32(curAivM), " |
| 413 | "ConvertToUint32(curAivN)}, {ConvertToUint32(((curAivN + 8 - 1) / 8 * 8)), " | 416 | "ConvertToUint32(curAivN)}, {ConvertToUint32(((curAivN + 8 - 1) / 8 * 8)), " |
| 414 | "ConvertToUint32(1)}, {ConvertToUint32(((curAivN + 16 - 1) / 16 * 16)), " | 417 | "ConvertToUint32(1)}, {ConvertToUint32(((curAivN + 16 - 1) / 16 * 16)), " |
| @@ -160,7 +160,12 @@ TEST(CodegenKernel, UnaryApicallIsNanCVStage) { | |||
| 160 | std::string result; | 160 | std::string result; |
| 161 | call_0.Generate(tpipe, vector<af::AxisId>{}, result); | 161 | call_0.Generate(tpipe, vector<af::AxisId>{}, result); |
| 162 | EXPECT_EQ(result, std::string{"LocalTensor<bool> local_1_cast = local_1.template ReinterpretCast<bool>();\n" | 162 | EXPECT_EQ(result, std::string{"LocalTensor<bool> local_1_cast = local_1.template ReinterpretCast<bool>();\n" |
| 163 | - "IsNan(local_1_cast[0], local_0[0], ((local_0_actual_size + 8 - 1) / 8 * 8));\n"}); | 163 | + "local_1_actual_size = curAivM * ((curAivN + 16 - 1) / 16 * 16);\n" |
| 164 | + "for (uint32_t cv_row = 0; cv_row < ConvertToUint32(curAivM); cv_row++) {\n" | ||
| 165 | + " IsNan(local_1_cast[cv_row * ConvertToUint32(((curAivN + 16 - 1) / 16 * 16))], " | ||
| 166 | + "local_0[cv_row * ConvertToUint32(((curAivN + 8 - 1) / 8 * 8))], " | ||
| 167 | + "ConvertToUint32(curAivN));\n" | ||
| 168 | + "}\n"}); | ||
| 164 | } | 169 | } |
| 165 | 170 | ||
| 166 | TEST(CodegenKernel, UnaryApicallIsNanCvStageUsesInputDtypeAlignedCount) { | 171 | TEST(CodegenKernel, UnaryApicallIsNanCvStageUsesInputDtypeAlignedCount) { |
| @@ -191,7 +196,12 @@ TEST(CodegenKernel, UnaryApicallIsNanCvStageUsesInputDtypeAlignedCount) { | |||
| 191 | std::string result; | 196 | std::string result; |
| 192 | EXPECT_EQ(call.Generate(tpipe, vector<af::AxisId>{}, result), 0); | 197 | EXPECT_EQ(call.Generate(tpipe, vector<af::AxisId>{}, result), 0); |
| 193 | EXPECT_EQ(result, std::string{"LocalTensor<bool> local_1_cast = local_1.template ReinterpretCast<bool>();\n" | 198 | EXPECT_EQ(result, std::string{"LocalTensor<bool> local_1_cast = local_1.template ReinterpretCast<bool>();\n" |
| 194 | - "IsNan(local_1_cast[0], local_0[0], ((local_0_actual_size + 8 - 1) / 8 * 8));\n"}); | 199 | + "local_1_actual_size = curAivM * ((curAivN + 32 - 1) / 32 * 32);\n" |
| 200 | + "for (uint32_t cv_row = 0; cv_row < ConvertToUint32(curAivM); cv_row++) {\n" | ||
| 201 | + " IsNan(local_1_cast[cv_row * ConvertToUint32(((curAivN + 32 - 1) / 32 * 32))], " | ||
| 202 | + "local_0[cv_row * ConvertToUint32(((curAivN + 8 - 1) / 8 * 8))], " | ||
| 203 | + "ConvertToUint32(curAivN));\n" | ||
| 204 | + "}\n"}); | ||
| 195 | } | 205 | } |
| 196 | 206 | ||
| 197 | TEST(CodegenKernel, UnaryApicallIsNanThrowingFor) { | 207 | TEST(CodegenKernel, UnaryApicallIsNanThrowingFor) { |
| @@ -308,8 +308,13 @@ TEST(WhereRegApiCallTest, WhereRegApiCall_Scalar_x2x3_CVStage) { | |||
| 308 | 308 | ||
| 309 | std::string result; | 309 | std::string result; |
| 310 | EXPECT_EQ(call.Generate(tpipe, current_axis, result), 0); | 310 | EXPECT_EQ(call.Generate(tpipe, current_axis, result), 0); |
| 311 | - EXPECT_EQ(result, std::string{"Where(local_3[0], local_0[0], local_1, local_2, " | 311 | + EXPECT_EQ(result, std::string{"local_3_actual_size = curAivM * ((curAivN + 16 - 1) / 16 * 16);\n" |
| 312 | - "((local_0_actual_size + 16 - 1) / 16 * 16));\n"}); | 312 | + "Where<true, true, 2>(local_3[0], local_0[0], local_blk_tensor_of_local_1[0], " |
| 313 | + "local_blk_tensor_of_local_2[0], {static_cast<uint16_t>(curAivM), " | ||
| 314 | + "static_cast<uint16_t>(curAivN)}, {static_cast<uint16_t>(((curAivN + 16 - 1) / " | ||
| 315 | + "16 * 16)), static_cast<uint16_t>(1)}, {static_cast<uint16_t>(((curAivN + 8 - 1) " | ||
| 316 | + "/ 8 * 8)), static_cast<uint16_t>(1)}, {static_cast<uint16_t>(((curAivN + 8 - 1) " | ||
| 317 | + "/ 8 * 8)), static_cast<uint16_t>(1)});\n"}); | ||
| 313 | } | 318 | } |
| 314 | 319 | ||
| 315 | TEST(WhereRegApiCallTest, WhereRegApiCall_x2_x3_is_ub_scalar) { | 320 | TEST(WhereRegApiCallTest, WhereRegApiCall_x2_x3_is_ub_scalar) { |
| @@ -1145,9 +1145,9 @@ TEST(CodegenKernel, CvUbFuseTensorSizeAssignUsesTensorDtype) { | |||
| 1145 | 1145 | ||
| 1146 | std::string result; | 1146 | std::string result; |
| 1147 | ASSERT_EQ(tpipe.TensorSizeAssign("float", result), af::SUCCESS); | 1147 | ASSERT_EQ(tpipe.TensorSizeAssign("float", result), af::SUCCESS); |
| 1148 | - EXPECT_NE(result.find("local_0_size = stage_size / sizeof(float);"), std::string::npos); | 1148 | + EXPECT_NE(result.find("local_0_size = stage_size;"), std::string::npos); |
| 1149 | - EXPECT_NE(result.find("local_1_size = stage_size / sizeof(float);"), std::string::npos); | 1149 | + EXPECT_NE(result.find("local_1_size = stage_size;"), std::string::npos); |
| 1150 | - EXPECT_NE(result.find("local_2_size = stage_size / sizeof(float);"), std::string::npos); | 1150 | + EXPECT_NE(result.find("local_2_size = stage_size;"), std::string::npos); |
| 1151 | } | 1151 | } |
| 1152 | 1152 | ||
| 1153 | TEST(CodegenKernel, VfCall_TwoDimLoad_VFLoop) { | 1153 | TEST(CodegenKernel, VfCall_TwoDimLoad_VFLoop) { |
| @@ -105,10 +105,10 @@ Status CastV2ApiCall::Generate(const TPipe &tpipe, const std::vector<ascir::Axis | |||
| 105 | } | 105 | } |
| 106 | GE_ASSERT_SUCCESS(FillCastNodeParams(this->node, output_dims, output_strides, input_strides)); | 106 | GE_ASSERT_SUCCESS(FillCastNodeParams(this->node, output_dims, output_strides, input_strides)); |
| 107 | stringstream ss; | 107 | stringstream ss; |
| 108 | - if (IsCVFusionStage(this->api_call_context)) { | 108 | + if (IsCVFusionStage(this->api_call_context) || tpipe.cv_fusion_type == ascir::CubeTemplateType::kUBFuse) { |
| 109 | const auto cv_params = BuildCvApi2DParams(tpipe, x, y); | 109 | const auto cv_params = BuildCvApi2DParams(tpipe, x, y); |
| 110 | const std::string input_tensor = x.is_constant ? ("local_blk_tensor_of_" + x.name) : x.Str(); | 110 | const std::string input_tensor = x.is_constant ? ("local_blk_tensor_of_" + x.name) : x.Str(); |
| 111 | - ss << y.actual_size << " = " << cv_params.first_dim << " * " << cv_params.last_dim << ";" << std::endl; | 111 | + ss << y.actual_size << " = " << cv_params.output_compute_size << ";" << std::endl; |
| 112 | ss << this->api_name_ << "(" << y << "[0], " << input_tensor << "[0], " << GenCvUint32Dims(cv_params) << ", " | 112 | ss << this->api_name_ << "(" << y << "[0], " << input_tensor << "[0], " << GenCvUint32Dims(cv_params) << ", " |
| 113 | << GenCvUint32Stride(cv_params.output_stride) << ", " << GenCvUint32Stride(cv_params.input_stride) << ");" | 113 | << GenCvUint32Stride(cv_params.output_stride) << ", " << GenCvUint32Stride(cv_params.input_stride) << ");" |
| 114 | << std::endl; | 114 | << std::endl; |
| @@ -133,8 +133,9 @@ Status CompareV2ApiCall::Generate(const TPipe &tpipe, const std::vector<ascir::A | |||
| 133 | std::string scalar_local_blk_tensor_name_x2 = x2.IsConstScalar() ? "local_blk_tensor_of_" + x2.name : x2.name; | 133 | std::string scalar_local_blk_tensor_name_x2 = x2.IsConstScalar() ? "local_blk_tensor_of_" + x2.name : x2.name; |
| 134 | size_t outer_repeats_size = param.outer_repeats.size(); | 134 | size_t outer_repeats_size = param.outer_repeats.size(); |
| 135 | if (outer_repeats_size == 0U) { | 135 | if (outer_repeats_size == 0U) { |
| 136 | - if (IsCVFusionStage(this->api_call_context)) { | 136 | + if (IsCVFusionStage(this->api_call_context) || tpipe.cv_fusion_type == ascir::CubeTemplateType::kUBFuse) { |
| 137 | const auto cv_params = BuildCvApi2DParams(tpipe, x1, y); | 137 | const auto cv_params = BuildCvApi2DParams(tpipe, x1, y); |
| 138 | + ss << y.actual_size << " = " << cv_params.output_compute_size << ";" << std::endl; | ||
| 138 | ss << "CompareScalarExtend<" << dtype_name << ", 2, CMPMODE::" << this->api_name_ << ">(" << y << "[" | 139 | ss << "CompareScalarExtend<" << dtype_name << ", 2, CMPMODE::" << this->api_name_ << ">(" << y << "[" |
| 139 | << tpipe.tiler.TensorVectorizedOffset(current_axis, y) << "], " << x1 << "[" | 140 | << tpipe.tiler.TensorVectorizedOffset(current_axis, y) << "], " << x1 << "[" |
| 140 | << tpipe.tiler.TensorVectorizedOffset(current_axis, x1) << "], " << x2_scalar << ", " | 141 | << tpipe.tiler.TensorVectorizedOffset(current_axis, x1) << "], " << x2_scalar << ", " |
| @@ -187,8 +188,9 @@ Status CompareV2ApiCall::Generate(const TPipe &tpipe, const std::vector<ascir::A | |||
| 187 | FillCompareNodeParams(this->node, false, outer_call_count, output_dims, output_strides, input_strides)); | 188 | FillCompareNodeParams(this->node, false, outer_call_count, output_dims, output_strides, input_strides)); |
| 188 | size_t outer_repeats_size = param.outer_repeats.size(); | 189 | size_t outer_repeats_size = param.outer_repeats.size(); |
| 189 | if (outer_repeats_size == 0U) { | 190 | if (outer_repeats_size == 0U) { |
| 190 | - if (IsCVFusionStage(this->api_call_context)) { | 191 | + if (IsCVFusionStage(this->api_call_context) || tpipe.cv_fusion_type == ascir::CubeTemplateType::kUBFuse) { |
| 191 | const auto cv_params = BuildCvApi2DParams(tpipe, x1, y); | 192 | const auto cv_params = BuildCvApi2DParams(tpipe, x1, y); |
| 193 | + ss << y.actual_size << " = " << cv_params.output_compute_size << ";" << std::endl; | ||
| 192 | ss << "CompareExtend<" << dtype_name << ", 2, CMPMODE::" << this->api_name_ << ">(" << y << "[" | 194 | ss << "CompareExtend<" << dtype_name << ", 2, CMPMODE::" << this->api_name_ << ">(" << y << "[" |
| 193 | << tpipe.tiler.TensorVectorizedOffset(current_axis, y) << "], " << x1 << "[" | 195 | << tpipe.tiler.TensorVectorizedOffset(current_axis, y) << "], " << x1 << "[" |
| 194 | << tpipe.tiler.TensorVectorizedOffset(current_axis, x1) << "], " << x2 << "[" | 196 | << tpipe.tiler.TensorVectorizedOffset(current_axis, x1) << "], " << x2 << "[" |
| @@ -37,8 +37,9 @@ Status FloorToIntApiCall::Generate(const TPipe &tpipe, const std::vector<ascir:: | |||
| 37 | GELOGI("FloorToInt x_dtype:%d, y_dtype:%d.", static_cast<int32_t>(x.dtype), static_cast<int32_t>(y.dtype)); | 37 | GELOGI("FloorToInt x_dtype:%d, y_dtype:%d.", static_cast<int32_t>(x.dtype), static_cast<int32_t>(y.dtype)); |
| 38 | stringstream ss; | 38 | stringstream ss; |
| 39 | 39 | ||
| 40 | - if (IsCVFusionStage(this->api_call_context)) { | 40 | + if (IsCVFusionStage(this->api_call_context) || tpipe.cv_fusion_type == ascir::CubeTemplateType::kUBFuse) { |
| 41 | const auto cv_params = BuildCvApi2DParams(tpipe, x, y); | 41 | const auto cv_params = BuildCvApi2DParams(tpipe, x, y); |
| 42 | + ss << y.actual_size << " = " << cv_params.output_compute_size << ";" << std::endl; | ||
| 42 | ss << "AscendC::Cast(" << y << "[0], " << x << "[0], AscendC::RoundMode::CAST_FLOOR, " << GenCvUint32Dims(cv_params) | 43 | ss << "AscendC::Cast(" << y << "[0], " << x << "[0], AscendC::RoundMode::CAST_FLOOR, " << GenCvUint32Dims(cv_params) |
| 43 | << ", " << GenCvUint32Stride(cv_params.output_stride) << ", " << GenCvUint32Stride(cv_params.input_stride) | 44 | << ", " << GenCvUint32Stride(cv_params.output_stride) << ", " << GenCvUint32Stride(cv_params.input_stride) |
| 44 | << ");" << std::endl; | 45 | << ");" << std::endl; |
| @@ -30,6 +30,9 @@ CvApi2DParams BuildCvApi2DParams(const TPipe &tpipe, const Tensor &input, const | |||
| 30 | const bool input_is_cube_output = | 30 | const bool input_is_cube_output = |
| 31 | tpipe.cv_fusion_type == ascir::CubeTemplateType::kUBFuse && input.id == tpipe.cube_output_tensor_id; | 31 | tpipe.cv_fusion_type == ascir::CubeTemplateType::kUBFuse && input.id == tpipe.cube_output_tensor_id; |
| 32 | params.input_stride = input_is_cube_output ? "curAlignN" : GenBlockAlignNExpr(input, params.last_dim); | 32 | params.input_stride = input_is_cube_output ? "curAlignN" : GenBlockAlignNExpr(input, params.last_dim); |
| 33 | + params.valid_size = params.first_dim + " * " + params.last_dim; | ||
| 34 | + params.output_compute_size = params.first_dim + " * " + params.output_stride; | ||
| 35 | + params.input_compute_size = params.first_dim + " * " + params.input_stride; | ||
| 33 | return params; | 36 | return params; |
| 34 | } | 37 | } |
| 35 | 38 | ||
| @@ -65,6 +65,9 @@ struct CvApi2DParams { | |||
| 65 | std::string last_dim; | 65 | std::string last_dim; |
| 66 | std::string output_stride; | 66 | std::string output_stride; |
| 67 | std::string input_stride; | 67 | std::string input_stride; |
| 68 | + std::string valid_size; | ||
| 69 | + std::string output_compute_size; | ||
| 70 | + std::string input_compute_size; | ||
| 68 | }; | 71 | }; |
| 69 | 72 | ||
| 70 | CvApi2DParams BuildCvApi2DParams(const TPipe &tpipe, const Tensor &input, const Tensor &output); | 73 | CvApi2DParams BuildCvApi2DParams(const TPipe &tpipe, const Tensor &input, const Tensor &output); |
| @@ -16,6 +16,7 @@ | |||
| 16 | #include "ascir_node_param/ascir_node_param.h" | 16 | #include "ascir_node_param/ascir_node_param.h" |
| 17 | #include "api_call/utils/api_call_factory.h" | 17 | #include "api_call/utils/api_call_factory.h" |
| 18 | #include "api_call/utils/api_call_utils.h" | 18 | #include "api_call/utils/api_call_utils.h" |
| 19 | +#include "reg_api_call_utils.h" | ||
| 19 | 20 | ||
| 20 | namespace codegen { | 21 | namespace codegen { |
| 21 | using namespace std; | 22 | using namespace std; |
| @@ -156,6 +157,31 @@ Status WhereRegApiCall::GenerateNoLoopCase(const TPipe &tpipe, const std::vector | |||
| 156 | const bool x2_is_scalar_scene = x2.IsAnyScalar(); | 157 | const bool x2_is_scalar_scene = x2.IsAnyScalar(); |
| 157 | const bool x3_is_scalar_scene = x3.IsAnyScalar(); | 158 | const bool x3_is_scalar_scene = x3.IsAnyScalar(); |
| 158 | 159 | ||
| 160 | + if (IsCVFusionStage(this->api_call_context) || tpipe.cv_fusion_type == ascir::CubeTemplateType::kUBFuse) { | ||
| 161 | + // CV UBFuse: mask/src/output have different dtype-aligned row strides, must use 2D API. | ||
| 162 | + // For examples: (dst, mask, src0, src1, {curAivM, curAivN}, {out_stride,1}, {mask_stride,1}, {in_stride,1}). | ||
| 163 | + const auto cv_params = BuildCvApi2DParams(tpipe, x2, y); | ||
| 164 | + ss << y.actual_size << " = " << cv_params.output_compute_size << ";" << std::endl; | ||
| 165 | + ss << this->api_name_ << "<" << (x2_is_scalar_scene ? "true" : "false") << ", " | ||
| 166 | + << (x3_is_scalar_scene ? "true" : "false") << ", 2>(" << y << "[" | ||
| 167 | + << tpipe.tiler.TensorVectorizedOffset(current_axis, y) << "], " << x1 << "[" | ||
| 168 | + << tpipe.tiler.TensorVectorizedOffset(current_axis, x1) << "], "; | ||
| 169 | + if (x2_is_scalar_scene) { | ||
| 170 | + ss << (x2.IsConstScalar() ? "local_blk_tensor_of_" + x2.name : x2.Str()) << "[0], "; | ||
| 171 | + } else { | ||
| 172 | + ss << x2 << "[" << tpipe.tiler.TensorVectorizedOffset(current_axis, x2) << "], "; | ||
| 173 | + } | ||
| 174 | + if (x3_is_scalar_scene) { | ||
| 175 | + ss << (x3.IsConstScalar() ? "local_blk_tensor_of_" + x3.name : x3.Str()) << "[0], "; | ||
| 176 | + } else { | ||
| 177 | + ss << x3 << "[" << tpipe.tiler.TensorVectorizedOffset(current_axis, x3) << "], "; | ||
| 178 | + } | ||
| 179 | + ss << GenCvUint16Dims(cv_params) << ", " << GenCvUint16Stride(cv_params.output_stride) << ", " | ||
| 180 | + << GenCvUint16Stride(GenBlockAlignNExpr(x1, cv_params.last_dim)) << ", " | ||
| 181 | + << GenCvUint16Stride(cv_params.input_stride) << ");" << std::endl; | ||
| 182 | + return af::SUCCESS; | ||
| 183 | + } | ||
| 184 | + | ||
| 159 | ss << this->api_name_ << "(" << y << "[" << tpipe.tiler.TensorVectorizedOffset(current_axis, y) << "], "; | 185 | ss << this->api_name_ << "(" << y << "[" << tpipe.tiler.TensorVectorizedOffset(current_axis, y) << "], "; |
| 160 | ss << x1 << "[" << tpipe.tiler.TensorVectorizedOffset(current_axis, x1) << "], "; | 186 | ss << x1 << "[" << tpipe.tiler.TensorVectorizedOffset(current_axis, x1) << "], "; |
| 161 | 187 | ||
| @@ -40,8 +40,9 @@ Status RoundToIntApiCall::Generate(const TPipe &tpipe, const std::vector<ascir:: | |||
| 40 | 40 | ||
| 41 | stringstream ss; | 41 | stringstream ss; |
| 42 | 42 | ||
| 43 | - if (IsCVFusionStage(this->api_call_context)) { | 43 | + if (IsCVFusionStage(this->api_call_context) || tpipe.cv_fusion_type == ascir::CubeTemplateType::kUBFuse) { |
| 44 | const auto cv_params = BuildCvApi2DParams(tpipe, x, y); | 44 | const auto cv_params = BuildCvApi2DParams(tpipe, x, y); |
| 45 | + ss << y.actual_size << " = " << cv_params.output_compute_size << ";" << std::endl; | ||
| 45 | ss << "AscendC::Cast(" << y << "[0], " << x << "[0], AscendC::RoundMode::CAST_RINT, " << GenCvUint32Dims(cv_params) | 46 | ss << "AscendC::Cast(" << y << "[0], " << x << "[0], AscendC::RoundMode::CAST_RINT, " << GenCvUint32Dims(cv_params) |
| 46 | << ", " << GenCvUint32Stride(cv_params.output_stride) << ", " << GenCvUint32Stride(cv_params.input_stride) | 47 | << ", " << GenCvUint32Stride(cv_params.output_stride) << ", " << GenCvUint32Stride(cv_params.input_stride) |
| 47 | << ");" << std::endl; | 48 | << ");" << std::endl; |
| @@ -40,8 +40,9 @@ Status TruncToIntApiCall::Generate(const TPipe &tpipe, const std::vector<ascir:: | |||
| 40 | 40 | ||
| 41 | stringstream ss; | 41 | stringstream ss; |
| 42 | 42 | ||
| 43 | - if (IsCVFusionStage(this->api_call_context)) { | 43 | + if (IsCVFusionStage(this->api_call_context) || tpipe.cv_fusion_type == ascir::CubeTemplateType::kUBFuse) { |
| 44 | const auto cv_params = BuildCvApi2DParams(tpipe, x, y); | 44 | const auto cv_params = BuildCvApi2DParams(tpipe, x, y); |
| 45 | + ss << y.actual_size << " = " << cv_params.output_compute_size << ";" << std::endl; | ||
| 45 | ss << "AscendC::Cast(" << y << "[0], " << x << "[0], AscendC::RoundMode::CAST_TRUNC, " << GenCvUint32Dims(cv_params) | 46 | ss << "AscendC::Cast(" << y << "[0], " << x << "[0], AscendC::RoundMode::CAST_TRUNC, " << GenCvUint32Dims(cv_params) |
| 46 | << ", " << GenCvUint32Stride(cv_params.output_stride) << ", " << GenCvUint32Stride(cv_params.input_stride) | 47 | << ", " << GenCvUint32Stride(cv_params.output_stride) << ", " << GenCvUint32Stride(cv_params.input_stride) |
| 47 | << ");" << std::endl; | 48 | << ");" << std::endl; |
| @@ -83,6 +83,20 @@ Status UnaryBitWidthChangeApiCallV2::Generate(const TPipe &tpipe, const std::vec | |||
| 83 | GE_ASSERT_TRUE(status, "GenerateVectorizedAxisMergeStatus failed"); | 83 | GE_ASSERT_TRUE(status, "GenerateVectorizedAxisMergeStatus failed"); |
| 84 | SaveApiLoopAxisParams(merge_info, param); | 84 | SaveApiLoopAxisParams(merge_info, param); |
| 85 | GE_ASSERT_SUCCESS(FillUnaryBitWidthChangeNodeParams(this->node, param, merge_info)); | 85 | GE_ASSERT_SUCCESS(FillUnaryBitWidthChangeNodeParams(this->node, param, merge_info)); |
| 86 | + if (IsCVFusionStage(this->api_call_context) || tpipe.cv_fusion_type == ascir::CubeTemplateType::kUBFuse) { | ||
| 87 | + const auto cv_params = BuildCvApi2DParams(tpipe, x, y); | ||
| 88 | + ss << y.actual_size << " = " << cv_params.output_compute_size << ";" << std::endl; | ||
| 89 | + // 该类 API(如 IsFinite)输出与输入 dtype 的 32B 行对齐粒度不同(bool 为 32 元素, float 为 | ||
| 90 | + // 8 元素), 1D flat 调用会在行宽非输出对齐整数倍的尾块错位。CANN 仅有 1D 形式, 按 | ||
| 91 | + // per-dtype 物理行距逐行调用, 输出按输出 dtype 的行距落位, 与 stage 写出侧行距一致。 | ||
| 92 | + ss << "for (uint32_t cv_row = 0; cv_row < ConvertToUint32(" << cv_params.first_dim << "); cv_row++) {" << std::endl; | ||
| 93 | + ss << " " << this->api_name_ << "(" << y << "_cast[cv_row * ConvertToUint32(" << cv_params.output_stride << ")], " | ||
| 94 | + << x << "[cv_row * ConvertToUint32(" << cv_params.input_stride << ")], ConvertToUint32(" << cv_params.last_dim | ||
| 95 | + << "));" << std::endl; | ||
| 96 | + ss << "}" << std::endl; | ||
| 97 | + result = ss.str(); | ||
| 98 | + return af::SUCCESS; | ||
| 99 | + } | ||
| 86 | if (param.outer_repeats.size() == 0) { | 100 | if (param.outer_repeats.size() == 0) { |
| 87 | ss << this->api_name_ << "(" << y << "_cast[" << tpipe.tiler.TensorVectorizedOffset(current_axis, y) << "], " << x | 101 | ss << this->api_name_ << "(" << y << "_cast[" << tpipe.tiler.TensorVectorizedOffset(current_axis, y) << "], " << x |
| 88 | << "[" << tpipe.tiler.TensorVectorizedOffset(current_axis, x) << "], " | 102 | << "[" << tpipe.tiler.TensorVectorizedOffset(current_axis, x) << "], " |