已合并
fix: complete Arange fallback path (#302) #1980
Jett_Woo创建于 23 天前
fix: complete Arange fallback path (#302) #1980
已合并
共 29 个文件变更+1186-84
| @@ -1365,7 +1365,7 @@ Status Kernel::ParseOptimizeInfo(const ascir::NodeView &node, const ascir::Tenso | |||
| 1365 | 1365 | ||
| 1366 | Status Kernel::ParseScalarNeedGenBlkTensors(const ascir::NodeView &node, ascir::TensorId id) { | 1366 | Status Kernel::ParseScalarNeedGenBlkTensors(const ascir::NodeView &node, ascir::TensorId id) { |
| 1367 | // 是scalar的节点,判断下是否支持 blk tensor 输入的 Ascir | 1367 | // 是scalar的节点,判断下是否支持 blk tensor 输入的 Ascir |
| 1368 | - if (!IsOps<Scalar>(node) && !IsOps<ScalarData>(node)) { | 1368 | + if (!IsOps<Scalar>(node) && !IsOps<ScalarData>(node) && !IsOps<IndexExpr>(node)) { |
| 1369 | return af::SUCCESS; | 1369 | return af::SUCCESS; |
| 1370 | } | 1370 | } |
| 1371 | for (auto &out : node->outputs()) { | 1371 | for (auto &out : node->outputs()) { |
| @@ -1704,8 +1704,10 @@ Status TPipe::BlkTensorAllocAndInit(std::string &result) const { | |||
| 1704 | ss << "LocalTensor<" << tensor_ptr->type << "> " << scalar_local_blk_tensor_name << " = " << scalar_t_buf_name | 1704 | ss << "LocalTensor<" << tensor_ptr->type << "> " << scalar_local_blk_tensor_name << " = " << scalar_t_buf_name |
| 1705 | << ".Get<" << tensor_ptr->type << ">();" << std::endl; | 1705 | << ".Get<" << tensor_ptr->type << ">();" << std::endl; |
| 1706 | 1706 | ||
| 1707 | - ss << "Duplicate(" << scalar_local_blk_tensor_name << "[0], static_cast<" << tensor_ptr->type << ">(" | 1707 | + const auto const_value = |
| 1708 | - << tensor_ptr->const_value << "), static_cast<uint64_t>(32/" | 1708 | + tensor_ptr->const_value.empty() ? tiler.Size(tensor_ptr->const_value_expr, true) : tensor_ptr->const_value; |
| 1709 | + ss << "Duplicate(" << scalar_local_blk_tensor_name << "[0], static_cast<" << tensor_ptr->type << ">(" << const_value | ||
| 1710 | + << "), static_cast<uint64_t>(32/" | ||
| 1709 | << "sizeof(" << tensor_ptr->type << ")));" << std::endl; | 1711 | << "sizeof(" << tensor_ptr->type << ")));" << std::endl; |
| 1710 | ss << "AscendC::PipeBarrier<PIPE_V>();" << std::endl; | 1712 | ss << "AscendC::PipeBarrier<PIPE_V>();" << std::endl; |
| 1711 | } | 1713 | } |
| @@ -2292,6 +2294,7 @@ Status Kernel::ParseGraph(const ascir::ImplGraph &graph, const ascir::FusedSched | |||
| 2292 | return af::FAILED; | 2294 | return af::FAILED; |
| 2293 | } | 2295 | } |
| 2294 | GE_CHK_STATUS_RET(kernel.tpipe.AddTensor(*output, expr, tensor_name), "Codegen add tensor failed"); | 2296 | GE_CHK_STATUS_RET(kernel.tpipe.AddTensor(*output, expr, tensor_name), "Codegen add tensor failed"); |
| 2297 | + GE_CHK_STATUS_RET(kernel.ParseOptimizeInfo(node, *output)); | ||
| 2295 | } else if (IsOps<Workspace>(node)) { | 2298 | } else if (IsOps<Workspace>(node)) { |
| 2296 | GE_CHK_STATUS_RET(kernel.ParseWorkspaceTensor(output, fused_schedule_result, output_indices, | 2299 | GE_CHK_STATUS_RET(kernel.ParseWorkspaceTensor(output, fused_schedule_result, output_indices, |
| 2297 | output_tensorid_to_index, output_index_to_name), | 2300 | output_tensorid_to_index, output_index_to_name), |
| @@ -417,16 +417,26 @@ PyObject *SizeExpr::FloorDiv(PyObject *self, PyObject *args) { | |||
| 417 | } | 417 | } |
| 418 | 418 | ||
| 419 | PyObject *SizeExpr::Compare(PyObject *self, PyObject *other, int op) { | 419 | PyObject *SizeExpr::Compare(PyObject *self, PyObject *other, int op) { |
| 420 | - if (op == Py_EQ) { | 420 | + if (op == Py_EQ || op == Py_NE) { |
| 421 | af::Expression left = SizeExpr::AsSizeExpr(self); | 421 | af::Expression left = SizeExpr::AsSizeExpr(self); |
| 422 | af::Expression right = SizeExpr::AsSizeExpr(other); | 422 | af::Expression right = SizeExpr::AsSizeExpr(other); |
| 423 | - if (left == right) { | 423 | + // richcompare 必须返回新引用: 直接 return Py_True/Py_False 会把单例的借用引用 |
| 424 | - return Py_True; | 424 | + // 当新引用交给调用方, 调用方释放时对 True/False 多减一次引用计数; |
| 425 | + // Python 3.10 及以前 True/False 非永生对象, 减穿后在解释器退出阶段触发 | ||
| 426 | + // 对 _Py_TrueStruct 的非法 free (munmap_chunk invalid pointer)。 | ||
| 427 | + const bool equal = (left == right); | ||
| 428 | + if (op == Py_NE) { | ||
| 429 | + if (!equal) { | ||
| 430 | + Py_RETURN_TRUE; | ||
| 431 | + } | ||
| 432 | + Py_RETURN_FALSE; | ||
| 425 | } | 433 | } |
| 426 | - return Py_False; | 434 | + if (equal) { |
| 427 | - } else { | 435 | + Py_RETURN_TRUE; |
| 428 | - return nullptr; | 436 | + } |
| 437 | + Py_RETURN_FALSE; | ||
| 429 | } | 438 | } |
| 439 | + Py_RETURN_NOTIMPLEMENTED; | ||
| 430 | } | 440 | } |
| 431 | } // namespace pyascir | 441 | } // namespace pyascir |
| 432 | 442 | ||
| @@ -1097,28 +1107,28 @@ PyObject *FusedScheduledResult::GetOutputNum(PyObject *self_pyobject) { | |||
| 1097 | PyObject *FusedScheduledResult::IsConvType(PyObject *self_pyobject) { | 1107 | PyObject *FusedScheduledResult::IsConvType(PyObject *self_pyobject) { |
| 1098 | auto self = reinterpret_cast<FusedScheduledResult::Object *>(self_pyobject); | 1108 | auto self = reinterpret_cast<FusedScheduledResult::Object *>(self_pyobject); |
| 1099 | if (self == nullptr) { | 1109 | if (self == nullptr) { |
| 1100 | - return Py_False; | 1110 | + Py_RETURN_FALSE; |
| 1101 | } | 1111 | } |
| 1102 | if (ascgen_utils::IsConv2DFusedScheduled(self->fused_schedule_result)) { | 1112 | if (ascgen_utils::IsConv2DFusedScheduled(self->fused_schedule_result)) { |
| 1103 | - return Py_True; | 1113 | + Py_RETURN_TRUE; |
| 1104 | } | 1114 | } |
| 1105 | 1115 | ||
| 1106 | - return Py_False; | 1116 | + Py_RETURN_FALSE; |
| 1107 | } | 1117 | } |
| 1108 | 1118 | ||
| 1109 | PyObject *FusedScheduledResult::IsCubeType(PyObject *self_pyobject) { | 1119 | PyObject *FusedScheduledResult::IsCubeType(PyObject *self_pyobject) { |
| 1110 | auto self = reinterpret_cast<FusedScheduledResult::Object *>(self_pyobject); | 1120 | auto self = reinterpret_cast<FusedScheduledResult::Object *>(self_pyobject); |
| 1111 | if (self == nullptr) { | 1121 | if (self == nullptr) { |
| 1112 | - return Py_False; | 1122 | + Py_RETURN_FALSE; |
| 1113 | } | 1123 | } |
| 1114 | for (auto scheduled_results : self->fused_schedule_result.node_idx_to_scheduled_results) { | 1124 | for (auto scheduled_results : self->fused_schedule_result.node_idx_to_scheduled_results) { |
| 1115 | for (auto scheduled_result : scheduled_results) { | 1125 | for (auto scheduled_result : scheduled_results) { |
| 1116 | if (scheduled_result.cube_type != ascir::CubeTemplateType::kDefault) { | 1126 | if (scheduled_result.cube_type != ascir::CubeTemplateType::kDefault) { |
| 1117 | - return Py_True; | 1127 | + Py_RETURN_TRUE; |
| 1118 | } | 1128 | } |
| 1119 | } | 1129 | } |
| 1120 | } | 1130 | } |
| 1121 | - return Py_False; | 1131 | + Py_RETURN_FALSE; |
| 1122 | } | 1132 | } |
| 1123 | 1133 | ||
| 1124 | PyObject *FusedScheduledResult::GetCubeAttributes(PyObject *self_pyobject) { | 1134 | PyObject *FusedScheduledResult::GetCubeAttributes(PyObject *self_pyobject) { |
| @@ -310,12 +310,14 @@ def arange( | |||
| 310 | "Arange requires unit physical stride on the vectorized axis" | 310 | "Arange requires unit physical stride on the vectorized axis" |
| 311 | ) | 311 | ) |
| 312 | effective_size = size if size is not None else [item.size for item in axis] | 312 | effective_size = size if size is not None else [item.size for item in axis] |
| 313 | - singleton_start = 0 if last_nonzero is None else last_nonzero[0] + 1 | ||
| 314 | if any( | 313 | if any( |
| 315 | - not (value == ascir.SizeExpr(1)) | 314 | + value == ascir.SizeExpr(0) and not (dimension == ascir.SizeExpr(1)) |
| 316 | - for value in effective_size[singleton_start:] | 315 | + for dimension, value in zip(effective_size, stride) |
| 317 | ): | 316 | ): |
| 318 | - raise ValueError("Arange zero strides require singleton dimensions") | 317 | + raise ValueError( |
| 318 | + "Arange does not support non-singleton zero strides; " | ||
| 319 | + "use 1D Arange followed by explicit Broadcast" | ||
| 320 | + ) | ||
| 319 | meta = _get_metadata(owner_graph) | 321 | meta = _get_metadata(owner_graph) |
| 320 | name = _generate_op_name(owner_graph, "arange") | 322 | name = _generate_op_name(owner_graph, "arange") |
| 321 | op = ascir.ops.Arange(name, owner_graph) | 323 | op = ascir.ops.Arange(name, owner_graph) |
| @@ -21,6 +21,7 @@ | |||
| 21 | 21 | ||
| 22 | 22 | ||
| 23 | 23 | ||
| 24 | + | ||
| 24 | 25 | ||
| 25 | using namespace af::ascir_op; | 26 | using namespace af::ascir_op; |
| 26 | using namespace af::ops; | 27 | using namespace af::ops; |
| @@ -904,6 +905,10 @@ Status BufQueAllocator::TopoSortByLoadPriority(af::AscGraph &graph) { | |||
| 904 | 905 | ||
| 905 | Status BufQueAllocator::ProcessSingleImplGraph(af::AscGraph &impl_graph, BasePlatform &platform, size_t max_que_num, | 906 | Status BufQueAllocator::ProcessSingleImplGraph(af::AscGraph &impl_graph, BasePlatform &platform, size_t max_que_num, |
| 906 | bool is_reduce_mem_reuse) { | 907 | bool is_reduce_mem_reuse) { |
| 908 | + // UBFuse 上下文通过内部 graph attr 传递给平台分区逻辑, 不扩展 BasePlatform 公共接口。 | ||
| 909 | + GE_ASSERT_TRUE(af::AttrUtils::SetBool(af::AscGraphUtils::GetComputeGraph(impl_graph), kGraphAttrIsUBFuseContext, | ||
| 910 | + cube_type == ascir::CubeTemplateType::kUBFuse), | ||
| 911 | + "Failed to set UBFuse context attr for graph %s.", impl_graph.GetName().c_str()); | ||
| 907 | GE_ASSERT_SUCCESS(platform.PartitionSubFunctions(impl_graph), "Failed to partition vf func for graph %s.", | 912 | GE_ASSERT_SUCCESS(platform.PartitionSubFunctions(impl_graph), "Failed to partition vf func for graph %s.", |
| 908 | impl_graph.GetName().c_str()); | 913 | impl_graph.GetName().c_str()); |
| 909 | if (cube_type == ascir::CubeTemplateType::kUBFuse) { | 914 | if (cube_type == ascir::CubeTemplateType::kUBFuse) { |
| @@ -28,6 +28,7 @@ | |||
| 28 | 28 | ||
| 29 | 29 | ||
| 30 | 30 | ||
| 31 | + | ||
| 31 | 32 | ||
| 32 | 33 | ||
| 33 | 34 | ||
| @@ -964,6 +965,9 @@ Status Optimizer::OptimizeForHintGraph(af::AscGraph &hint_graph, | |||
| 964 | utils::DumpGraph(optimize_graph, "AfterGraphPass"); | 965 | utils::DumpGraph(optimize_graph, "AfterGraphPass"); |
| 965 | // cube拆分后再做合轴 | 966 | // cube拆分后再做合轴 |
| 966 | if (!ScheduleUtils::HasComputeType(optimize_graph, af::ComputeType::kComputeCube)) { | 967 | if (!ScheduleUtils::HasComputeType(optimize_graph, af::ComputeType::kComputeCube)) { |
| 968 | + // 前端显式 1D Arange -> Broadcast: 合轴前补齐退化前缀轴, 保持输入输出同轴数 | ||
| 969 | + GE_ASSERT_SUCCESS(optimize::NormalizeArangeBroadcastViews(optimize_graph), | ||
| 970 | + "Normalize explicit Arange Broadcast views failed."); | ||
| 967 | // 这里concat已经打破了一套轴的约束 | 971 | // 这里concat已经打破了一套轴的约束 |
| 968 | GE_ASSERT_SUCCESS(RemoveAllZeroStrideLoopAxis(optimize_graph), "Remove All zero stride axis failed."); | 972 | GE_ASSERT_SUCCESS(RemoveAllZeroStrideLoopAxis(optimize_graph), "Remove All zero stride axis failed."); |
| 969 | GE_ASSERT_SUCCESS(MergeContinuousAxis(optimize_graph), "Merge continuous axes failed."); | 973 | GE_ASSERT_SUCCESS(MergeContinuousAxis(optimize_graph), "Merge continuous axes failed."); |
| @@ -539,7 +539,7 @@ TEST(CodegenKernel, OutputTensorIsScalarDuplicate_test) { | |||
| 539 | "TBuf<TPosition::VECCALC> global_1_tbuf;\n" | 539 | "TBuf<TPosition::VECCALC> global_1_tbuf;\n" |
| 540 | "tpipe.InitBuffer(global_1_tbuf, 32);\n" | 540 | "tpipe.InitBuffer(global_1_tbuf, 32);\n" |
| 541 | "LocalTensor<GlobalTensor<float>> local_blk_tensor_of_global_1 = global_1_tbuf.Get<GlobalTensor<float>>();\n" | 541 | "LocalTensor<GlobalTensor<float>> local_blk_tensor_of_global_1 = global_1_tbuf.Get<GlobalTensor<float>>();\n" |
| 542 | - "Duplicate(local_blk_tensor_of_global_1[0], static_cast<GlobalTensor<float>>(), " | 542 | + "Duplicate(local_blk_tensor_of_global_1[0], static_cast<GlobalTensor<float>>(0), " |
| 543 | "static_cast<uint64_t>(32/sizeof(GlobalTensor<float>)));\n" | 543 | "static_cast<uint64_t>(32/sizeof(GlobalTensor<float>)));\n" |
| 544 | "AscendC::PipeBarrier<PIPE_V>();\n"}); | 544 | "AscendC::PipeBarrier<PIPE_V>();\n"}); |
| 545 | } | 545 | } |
| @@ -344,6 +344,76 @@ TEST_F(BufQueAllocatorUT, AllocBufQueKeepsOldOneShotBehavior) { | |||
| 344 | ASSERT_EQ(fused_result.output_nodes.size(), 1UL); | 344 | ASSERT_EQ(fused_result.output_nodes.size(), 1UL); |
| 345 | } | 345 | } |
| 346 | 346 | ||
| 347 | +static af::AscGraph MakePartitionReadyArangeGraph(const std::string &name) { | ||
| 348 | + af::AscGraph graph(name.c_str()); | ||
| 349 | + const af::Expression size = graph.CreateSizeVar(64); | ||
| 350 | + auto axis = graph.CreateAxis("z0", size); | ||
| 351 | + | ||
| 352 | + af::ascir_op::Arange arange("arange", graph); | ||
| 353 | + arange.ir_attr.SetBase(af::Symbol(0)); | ||
| 354 | + arange.ir_attr.SetStep(af::Symbol(1)); | ||
| 355 | + arange.y.dtype = ge::DT_INT32; | ||
| 356 | + *arange.y.axis = {axis.id}; | ||
| 357 | + *arange.y.repeats = {size}; | ||
| 358 | + *arange.y.strides = {af::ops::One}; | ||
| 359 | + *arange.y.vectorized_axis = {axis.id}; | ||
| 360 | + *arange.y.vectorized_strides = {af::ops::One}; | ||
| 361 | + | ||
| 362 | + af::ascir_op::Store store("store"); | ||
| 363 | + store.x = arange.y; | ||
| 364 | + store.attr.api.compute_type = af::ComputeType::kComputeStore; | ||
| 365 | + store.attr.api.unit = af::ComputeUnit::kUnitMTE2; | ||
| 366 | + store.y.dtype = ge::DT_INT32; | ||
| 367 | + *store.y.axis = {axis.id}; | ||
| 368 | + *store.y.repeats = {size}; | ||
| 369 | + *store.y.strides = {af::ops::One}; | ||
| 370 | + | ||
| 371 | + af::ascir_op::Output output("output"); | ||
| 372 | + output.x = store.y; | ||
| 373 | + output.ir_attr.SetIndex(0); | ||
| 374 | + output.y.dtype = ge::DT_INT32; | ||
| 375 | + | ||
| 376 | + for (const auto &node : graph.GetAllNodes()) { | ||
| 377 | + node->attr.sched.axis = {axis.id}; | ||
| 378 | + } | ||
| 379 | + optimize::AscGraphInfoComplete::CompleteApiInfo(graph); | ||
| 380 | + return graph; | ||
| 381 | +} | ||
| 382 | + | ||
| 383 | +TEST_F(BufQueAllocatorUT, UBFuseContextKeepsArangeInRootGraph) { | ||
| 384 | + ge::PlatformContext::GetInstance().SetPlatform("3510"); | ||
| 385 | + auto fused_result = MakeFusedScheduledResultWithGraphs({MakePartitionReadyArangeGraph("ubfuse_arange")}); | ||
| 386 | + fused_result.node_idx_to_scheduled_results[0][0].cube_type = ascir::CubeTemplateType::kUBFuse; | ||
| 387 | + | ||
| 388 | + BufQueAllocator allocator; | ||
| 389 | + ASSERT_EQ(allocator.PrepareImplGraphMemoryPlan(fused_result), af::SUCCESS); | ||
| 390 | + | ||
| 391 | + auto &impl_graph = fused_result.node_idx_to_scheduled_results[0][0].schedule_groups[0].impl_graphs[0]; | ||
| 392 | + // UBFuse 上下文下 Arange 不参与 VF 分区, 保留在根图走普通 ArangeApiCall。 | ||
| 393 | + EXPECT_NE(impl_graph.FindNode("arange"), nullptr); | ||
| 394 | + EXPECT_EQ(impl_graph.FindNode("ubfuse_arange_VfNode_0"), nullptr); | ||
| 395 | + std::vector<af::AscGraph> subgraphs; | ||
| 396 | + ASSERT_EQ(impl_graph.GetAllSubGraphs(subgraphs), af::SUCCESS); | ||
| 397 | + EXPECT_TRUE(subgraphs.empty()); | ||
| 398 | +} | ||
| 399 | + | ||
| 400 | +TEST_F(BufQueAllocatorUT, DefaultContextStillPartitionsArangeIntoVf) { | ||
| 401 | + ge::PlatformContext::GetInstance().SetPlatform("3510"); | ||
| 402 | + auto fused_result = MakeFusedScheduledResultWithGraphs({MakePartitionReadyArangeGraph("default_arange")}); | ||
| 403 | + | ||
| 404 | + BufQueAllocator allocator; | ||
| 405 | + ASSERT_EQ(allocator.PrepareImplGraphMemoryPlan(fused_result), af::SUCCESS); | ||
| 406 | + | ||
| 407 | + auto &impl_graph = fused_result.node_idx_to_scheduled_results[0][0].schedule_groups[0].impl_graphs[0]; | ||
| 408 | + // 非 UBFuse 上下文行为不变: Arange 仍按既有行为分区进 VF 子图。 | ||
| 409 | + EXPECT_EQ(impl_graph.FindNode("arange"), nullptr); | ||
| 410 | + EXPECT_NE(impl_graph.FindNode("default_arange_VfNode_0"), nullptr); | ||
| 411 | + std::vector<af::AscGraph> subgraphs; | ||
| 412 | + ASSERT_EQ(impl_graph.GetAllSubGraphs(subgraphs), af::SUCCESS); | ||
| 413 | + ASSERT_EQ(subgraphs.size(), 1UL); | ||
| 414 | + EXPECT_NE(subgraphs[0].FindNode("arange"), nullptr); | ||
| 415 | +} | ||
| 416 | + | ||
| 347 | TEST_F(BufQueAllocatorUT, test_reuse_id_vecacc) { | 417 | TEST_F(BufQueAllocatorUT, test_reuse_id_vecacc) { |
| 348 | af::AscGraph graph("test_reuse_id_vecacc"); | 418 | af::AscGraph graph("test_reuse_id_vecacc"); |
| 349 | const af::Expression s0 = graph.CreateSizeVar("s0"); | 419 | const af::Expression s0 = graph.CreateSizeVar("s0"); |
| @@ -756,8 +756,10 @@ TEST_P(Int64ArangeConsumerCodegenTest, GeneratesInt64ConsumerWithoutNarrowing) { | |||
| 756 | ASSERT_FALSE(vf_signature.empty()); | 756 | ASSERT_FALSE(vf_signature.empty()); |
| 757 | EXPECT_EQ(vf_signature, | 757 | EXPECT_EQ(vf_signature, |
| 758 | "inline __simd_vf__ void VFCallint64_consumer_0_B0Y0_S0G0C0_VfNode_0(__local_mem__ int64_t " | 758 | "inline __simd_vf__ void VFCallint64_consumer_0_B0Y0_S0G0C0_VfNode_0(__local_mem__ int64_t " |
| 759 | - "*local_1_addr, uint32_t output_dims_0, int64_t arange_offset_0, int64_t arange_base_0, int64_t " | 759 | + "*local_1_addr, uint32_t output_dims_0, uint32_t local_1_stride_0, int64_t arange_offset_0, " |
| 760 | - "arange_step_0, int64_t arange_offset_1, int64_t arange_base_1, int64_t arange_step_1)"); | 760 | + "int64_t arange_base_0, int64_t arange_step_0, int64_t arange_stride_0_0, int64_t arange_stride_0_1, " |
| 761 | + "int64_t arange_stride_0_2, int64_t arange_offset_1, int64_t arange_base_1, int64_t arange_step_1, " | ||
| 762 | + "int64_t arange_stride_1_0, int64_t arange_stride_1_1, int64_t arange_stride_1_2)"); | ||
| 761 | } | 763 | } |
| 762 | } | 764 | } |
| 763 | 765 | ||
| @@ -200,6 +200,19 @@ class TestAscir: | |||
| 200 | assert not hasattr(values, "vectorized_axis") | 200 | assert not hasattr(values, "vectorized_axis") |
| 201 | assert not hasattr(values, "vectorized_strides") | 201 | assert not hasattr(values, "vectorized_strides") |
| 202 | 202 | ||
| 203 | + rows = graph.create_size("rows") | ||
| 204 | + row_axis = graph.create_axis("row", rows) | ||
| 205 | + expanded = ascir_api.Broadcast( | ||
| 206 | + graph, | ||
| 207 | + values, | ||
| 208 | + axis=[row_axis, axis], | ||
| 209 | + size=[rows, size], | ||
| 210 | + stride=[size, ascir.SizeExpr(1)], | ||
| 211 | + ) | ||
| 212 | + assert expanded.axis == [row_axis.id, axis.id] | ||
| 213 | + assert expanded.size == [rows, size] | ||
| 214 | + assert expanded.strides == [size, ascir.SizeExpr(1)] | ||
| 215 | + | ||
| 203 | singleton = graph.create_axis("singleton", 1) | 216 | singleton = graph.create_axis("singleton", 1) |
| 204 | trailing_singleton = ascir_api.arange( | 217 | trailing_singleton = ascir_api.arange( |
| 205 | graph, | 218 | graph, |
| @@ -222,6 +235,108 @@ class TestAscir: | |||
| 222 | ascir_api.Output(graph, stored, dtype=ascir.dtypes.int64) | 235 | ascir_api.Output(graph, stored, dtype=ascir.dtypes.int64) |
| 223 | Autofuser(AutofuserOptions()).schedule(graph) | 236 | Autofuser(AutofuserOptions()).schedule(graph) |
| 224 | 237 | ||
| 238 | + broadcast_graph = ascir.HintGraph("explicit_arange_broadcast") | ||
| 239 | + b_rows = broadcast_graph.create_size("rows") | ||
| 240 | + b_cols = broadcast_graph.create_size("cols") | ||
| 241 | + b_row_axis = broadcast_graph.create_axis("row", b_rows) | ||
| 242 | + b_col_axis = broadcast_graph.create_axis("col", b_cols) | ||
| 243 | + arange_value = ascir_api.arange( | ||
| 244 | + broadcast_graph, | ||
| 245 | + dtype=ascir.dtypes.int64, | ||
| 246 | + base=ascir.SizeExpr(0), | ||
| 247 | + step=ascir.SizeExpr(1), | ||
| 248 | + axis=[b_col_axis], | ||
| 249 | + size=[b_cols], | ||
| 250 | + stride=[ascir.SizeExpr(1)], | ||
| 251 | + ) | ||
| 252 | + expanded_value = ascir_api.Broadcast( | ||
| 253 | + broadcast_graph, | ||
| 254 | + arange_value, | ||
| 255 | + axis=[b_row_axis, b_col_axis], | ||
| 256 | + size=[b_rows, b_cols], | ||
| 257 | + stride=[b_cols, ascir.SizeExpr(1)], | ||
| 258 | + ) | ||
| 259 | + broadcast_stored = ascir_api.Store( | ||
| 260 | + broadcast_graph, | ||
| 261 | + expanded_value, | ||
| 262 | + axis=[b_row_axis, b_col_axis], | ||
| 263 | + size=[b_rows, b_cols], | ||
| 264 | + stride=[b_cols, ascir.SizeExpr(1)], | ||
| 265 | + ) | ||
| 266 | + ascir_api.Output( | ||
| 267 | + broadcast_graph, broadcast_stored, dtype=ascir.dtypes.int64 | ||
| 268 | + ) | ||
| 269 | + broadcast_fuser = Autofuser(AutofuserOptions()) | ||
| 270 | + scheduled = broadcast_fuser.schedule(broadcast_graph) | ||
| 271 | + _, _, broadcast_kernel = broadcast_fuser.codegen(scheduled) | ||
| 272 | + assert broadcast_kernel | ||
| 273 | + assert "for (int64_t arange_b" not in broadcast_kernel | ||
| 274 | + assert "BroadcastExtend" in broadcast_kernel | ||
| 275 | + | ||
| 276 | + degenerate_graph = ascir.HintGraph("degenerate_arange_broadcast") | ||
| 277 | + d_rows = degenerate_graph.create_size("rows") | ||
| 278 | + d_cols = degenerate_graph.create_size("cols") | ||
| 279 | + d_row_axis = degenerate_graph.create_axis("row", d_rows) | ||
| 280 | + d_col_axis = degenerate_graph.create_axis("col", d_cols) | ||
| 281 | + # Pattern A: 退化前缀轴 Arange([1, cols]/[0, 1]) -> Broadcast。 | ||
| 282 | + col_arange = ascir_api.arange( | ||
| 283 | + degenerate_graph, | ||
| 284 | + dtype=ascir.dtypes.int64, | ||
| 285 | + base=ascir.SizeExpr(0), | ||
| 286 | + step=ascir.SizeExpr(1), | ||
| 287 | + axis=[d_row_axis, d_col_axis], | ||
| 288 | + size=[ascir.SizeExpr(1), d_cols], | ||
| 289 | + stride=[ascir.SizeExpr(0), ascir.SizeExpr(1)], | ||
| 290 | + ) | ||
| 291 | + # Pattern B: 退化尾轴 Arange([rows, 1]/[1, 0]) -> Broadcast。 | ||
| 292 | + row_arange = ascir_api.arange( | ||
| 293 | + degenerate_graph, | ||
| 294 | + dtype=ascir.dtypes.int64, | ||
| 295 | + base=ascir.SizeExpr(0), | ||
| 296 | + step=d_cols, | ||
| 297 | + axis=[d_row_axis, d_col_axis], | ||
| 298 | + size=[d_rows, ascir.SizeExpr(1)], | ||
| 299 | + stride=[ascir.SizeExpr(1), ascir.SizeExpr(0)], | ||
| 300 | + ) | ||
| 301 | + col_expanded = ascir_api.Broadcast( | ||
| 302 | + degenerate_graph, | ||
| 303 | + col_arange, | ||
| 304 | + axis=[d_row_axis, d_col_axis], | ||
| 305 | + size=[d_rows, d_cols], | ||
| 306 | + stride=[d_cols, ascir.SizeExpr(1)], | ||
| 307 | + ) | ||
| 308 | + row_expanded = ascir_api.Broadcast( | ||
| 309 | + degenerate_graph, | ||
| 310 | + row_arange, | ||
| 311 | + axis=[d_row_axis, d_col_axis], | ||
| 312 | + size=[d_rows, d_cols], | ||
| 313 | + stride=[d_cols, ascir.SizeExpr(1)], | ||
| 314 | + ) | ||
| 315 | + add_value = ascir.ops.Add("add", degenerate_graph) | ||
| 316 | + add_value.x1 = col_expanded | ||
| 317 | + add_value.x2 = row_expanded | ||
| 318 | + add_value.attr.sched.axis = [d_row_axis, d_col_axis] | ||
| 319 | + add_value.y.axis = [d_row_axis, d_col_axis] | ||
| 320 | + add_value.y.size = [d_rows, d_cols] | ||
| 321 | + add_value.y.strides = [d_cols, ascir.SizeExpr(1)] | ||
| 322 | + add_value.y.dtype = ascir.dtypes.int64 | ||
| 323 | + degenerate_stored = ascir_api.Store( | ||
| 324 | + degenerate_graph, | ||
| 325 | + add_value.y, | ||
| 326 | + axis=[d_row_axis, d_col_axis], | ||
| 327 | + size=[d_rows, d_cols], | ||
| 328 | + stride=[d_cols, ascir.SizeExpr(1)], | ||
| 329 | + ) | ||
| 330 | + ascir_api.Output( | ||
| 331 | + degenerate_graph, degenerate_stored, dtype=ascir.dtypes.int64 | ||
| 332 | + ) | ||
| 333 | + degenerate_fuser = Autofuser(AutofuserOptions()) | ||
| 334 | + degenerate_scheduled = degenerate_fuser.schedule(degenerate_graph) | ||
| 335 | + _, _, degenerate_kernel = degenerate_fuser.codegen(degenerate_scheduled) | ||
| 336 | + assert degenerate_kernel | ||
| 337 | + assert "for (int64_t arange_b" not in degenerate_kernel | ||
| 338 | + assert "BroadcastExtend" in degenerate_kernel | ||
| 339 | + | ||
| 225 | with pytest.raises(ValueError, match="axis must not be empty"): | 340 | with pytest.raises(ValueError, match="axis must not be empty"): |
| 226 | ascir_api.arange( | 341 | ascir_api.arange( |
| 227 | graph, | 342 | graph, |
| @@ -240,7 +355,7 @@ class TestAscir: | |||
| 240 | size=[size], | 355 | size=[size], |
| 241 | stride=[ascir.SizeExpr(2)], | 356 | stride=[ascir.SizeExpr(2)], |
| 242 | ) | 357 | ) |
| 243 | - with pytest.raises(ValueError, match="zero strides require singleton"): | 358 | + with pytest.raises(ValueError, match="explicit Broadcast"): |
| 244 | ascir_api.arange( | 359 | ascir_api.arange( |
| 245 | graph, | 360 | graph, |
| 246 | dtype=ascir.dtypes.int32, | 361 | dtype=ascir.dtypes.int32, |
Mautofuse/tests/v35/st/backend_e2e_v2/load_erfinv_store_test/load_erfinv_store_backend_generator.cpp+7-1
| @@ -66,7 +66,13 @@ TEST_F(TestBackendLoadErfinvStoreE2e, LoadErfinvStoreE2eCodegen) { | |||
| 66 | codegen::CodegenResult erfinv_result; | 66 | codegen::CodegenResult erfinv_result; |
| 67 | EXPECT_EQ(erfinv_codegen.Generate(erfinv_shape_info, erfinv_fused_result, erfinv_result), 0); | 67 | EXPECT_EQ(erfinv_codegen.Generate(erfinv_shape_info, erfinv_fused_result, erfinv_result), 0); |
| 68 | EXPECT_NE(erfinv_result.kernel.find("Erfinv"), std::string::npos); | 68 | EXPECT_NE(erfinv_result.kernel.find("Erfinv"), std::string::npos); |
| 69 | - EXPECT_EQ(erfinv_result.kernel.find("tmp_buf_"), std::string::npos); | 69 | + const auto erfinv_call_begin = erfinv_result.kernel.find("Erfinv(local_"); |
| 70 | + ASSERT_NE(erfinv_call_begin, std::string::npos); | ||
| 71 | + const auto erfinv_call_end = erfinv_result.kernel.find(';', erfinv_call_begin); | ||
| 72 | + ASSERT_NE(erfinv_call_end, std::string::npos); | ||
| 73 | + // Store may need scratch even though the Erfinv call does not. | ||
| 74 | + EXPECT_EQ(erfinv_result.kernel.substr(erfinv_call_begin, erfinv_call_end - erfinv_call_begin).find("tmp_buf_"), | ||
| 75 | + std::string::npos); | ||
| 70 | erfinv_kernel_stream << erfinv_tiling_stub << RemoveSubDirInclude(erfinv_result.kernel); | 76 | erfinv_kernel_stream << erfinv_tiling_stub << RemoveSubDirInclude(erfinv_result.kernel); |
| 71 | erfinv_tiling_stream << erfinv_result.tiling; | 77 | erfinv_tiling_stream << erfinv_result.tiling; |
| 72 | erfinv_data_stream << erfinv_result.tiling_data; | 78 | erfinv_data_stream << erfinv_result.tiling_data; |
| @@ -48,9 +48,7 @@ TEST_F(TestBackendScalarCastAddE2e, ScalarCastAddE2eCodegen) { | |||
| 48 | std::map<std::string, std::string> shape_info({{"s0", "stub_s0"}, {"s1", "stub_s1"}, {"s2", "stub_s2"}}); | 48 | std::map<std::string, std::string> shape_info({{"s0", "stub_s0"}, {"s1", "stub_s1"}, {"s2", "stub_s2"}}); |
| 49 | auto graph = ascir::ShareGraph::ScalarCastAddFusedGraph(3, af::DT_FLOAT16, af::DT_FLOAT); | 49 | auto graph = ascir::ShareGraph::ScalarCastAddFusedGraph(3, af::DT_FLOAT16, af::DT_FLOAT); |
| 50 | GenerateBackendKernelWithCheck(graph, shape_info, tilig_stub, [](const std::string &kernel) { | 50 | GenerateBackendKernelWithCheck(graph, shape_info, tilig_stub, [](const std::string &kernel) { |
| 51 | - EXPECT_NE(kernel.find("CastExtend(local_3[0], local_blk_tensor_of_scalar_2[0], " | 51 | + EXPECT_NE(kernel.find("Duplicate(local_3[0], static_cast<float>(1.0), local_3_actual_size);"), std::string::npos); |
| 52 | - "{ConvertToUint32(local_3_actual_size)}, {ConvertToUint32(1)}, {ConvertToUint32(1)});"), | ||
| 53 | - std::string::npos); | ||
| 54 | EXPECT_NE(kernel.find("Add(local_5[0], local_4[0], local_3[0], local_4_actual_size);"), std::string::npos); | 52 | EXPECT_NE(kernel.find("Add(local_5[0], local_4[0], local_3[0], local_4_actual_size);"), std::string::npos); |
| 55 | }); | 53 | }); |
| 56 | } | 54 | } |
| @@ -38,6 +38,7 @@ | |||
| 38 | 38 | ||
| 39 | 39 | ||
| 40 | 40 | ||
| 41 | + | ||
| 41 | 42 | ||
| 42 | 43 | ||
| 43 | 44 | ||
| @@ -1359,6 +1360,57 @@ TEST_F(OptimizerStV2, TwoAxisSliceNeedAlign) { | |||
| 1359 | EXPECT_EQ(load_node->outputs[0].attr.vectorized_strides, golden_stride); | 1360 | EXPECT_EQ(load_node->outputs[0].attr.vectorized_strides, golden_stride); |
| 1360 | } | 1361 | } |
| 1361 | 1362 | ||
| 1363 | +TEST_F(OptimizerStV2, HighRankArangePaddedStoreAlignsUbLayout) { | ||
| 1364 | + struct Case { | ||
| 1365 | + std::vector<int64_t> sizes; | ||
| 1366 | + std::vector<int64_t> logical_strides; | ||
| 1367 | + std::vector<int64_t> store_strides; | ||
| 1368 | + std::vector<int64_t> aligned_strides; | ||
| 1369 | + }; | ||
| 1370 | + const std::vector<Case> cases = { | ||
| 1371 | + {{2, 3, 4, 5, 7}, {420, 140, 35, 7, 1}, {593, 196, 48, 9, 1}, {480, 160, 40, 8, 1}}, | ||
| 1372 | + {{2, 2, 2, 8, 1}, {32, 16, 8, 1, 1}, {121, 58, 27, 3, 1}, {256, 128, 64, 8, 0}}, | ||
| 1373 | + {{2, 2, 2, 8, 1, 1}, {32, 16, 8, 1, 1, 1}, {224, 109, 52, 6, 3, 1}, {256, 128, 64, 8, 0, 0}}, | ||
| 1374 | + }; | ||
| 1375 | + for (size_t case_index = 0; case_index < cases.size(); ++case_index) { | ||
| 1376 | + const auto to_exprs = [](const std::vector<int64_t> &values) { | ||
| 1377 | + std::vector<Expression> result; | ||
| 1378 | + for (const auto value : values) result.push_back(af::Symbol(value)); | ||
| 1379 | + return result; | ||
| 1380 | + }; | ||
| 1381 | + const auto sizes = to_exprs(cases[case_index].sizes); | ||
| 1382 | + af::AscGraph graph(("high_rank_arange_padded_store_" + std::to_string(case_index)).c_str()); | ||
| 1383 | + std::vector<af::AxisId> axes; | ||
| 1384 | + for (size_t index = 0; index < sizes.size(); ++index) { | ||
| 1385 | + axes.push_back(graph.CreateAxis("axis" + std::to_string(index), sizes[index]).id); | ||
| 1386 | + } | ||
| 1387 | + Arange arange("arange", graph); | ||
| 1388 | + arange.attr.sched.axis = axes; | ||
| 1389 | + arange.y.dtype = af::DT_INT32; | ||
| 1390 | + *arange.y.axis = axes; | ||
| 1391 | + *arange.y.repeats = sizes; | ||
| 1392 | + *arange.y.strides = to_exprs(cases[case_index].logical_strides); | ||
| 1393 | + *arange.y.vectorized_axis = axes; | ||
| 1394 | + Store store("store"); | ||
| 1395 | + store.attr.sched.axis = axes; | ||
| 1396 | + store.x = arange.y; | ||
| 1397 | + store.y.dtype = af::DT_INT32; | ||
| 1398 | + *store.y.axis = axes; | ||
| 1399 | + *store.y.repeats = sizes; | ||
| 1400 | + *store.y.strides = to_exprs(cases[case_index].store_strides); | ||
| 1401 | + *store.y.vectorized_axis = axes; | ||
| 1402 | + Output output("output"); | ||
| 1403 | + output.x = store.y; | ||
| 1404 | + output.y.dtype = af::DT_INT32; | ||
| 1405 | + output.attr.api.type = af::ApiType::kAPITypeBuffer; | ||
| 1406 | + output.ir_attr.SetIndex(0); | ||
| 1407 | + ::optimize::UnAlignmentStrategy strategy; | ||
| 1408 | + ASSERT_EQ(strategy.AlignVectorizedStrides(graph), af::SUCCESS); | ||
| 1409 | + EXPECT_EQ(graph.FindNode("arange")->outputs[0].attr.vectorized_strides, | ||
| 1410 | + to_exprs(cases[case_index].aligned_strides)); | ||
| 1411 | + } | ||
| 1412 | +} | ||
| 1413 | + | ||
| 1362 | TEST_F(OptimizerStV2, NoNeedAlign_AABToARA) { | 1414 | TEST_F(OptimizerStV2, NoNeedAlign_AABToARA) { |
| 1363 | const Expression s0 = af::Symbol(3); | 1415 | const Expression s0 = af::Symbol(3); |
| 1364 | 1416 | ||
| @@ -673,7 +673,10 @@ TEST_F(VectorFuncSt, cast_bit_with) { | |||
| 673 | std::vector<af::AscGraph> asc_graphs; | 673 | std::vector<af::AscGraph> asc_graphs; |
| 674 | fused_scheduled_result.node_idx_to_scheduled_results[0][0].schedule_groups[0].impl_graphs[0].GetAllSubGraphs( | 674 | fused_scheduled_result.node_idx_to_scheduled_results[0][0].schedule_groups[0].impl_graphs[0].GetAllSubGraphs( |
| 675 | asc_graphs); | 675 | asc_graphs); |
| 676 | - EXPECT_EQ(asc_graphs.size(), 2UL); | 676 | + // float<->int64 casts cross the 8<->4 byte boundary and stay in the root |
| 677 | + // graph (CastExtend), so the remaining f16/float nodes merge into a single | ||
| 678 | + // VF cluster instead of being split by the int64 bit-width gap. | ||
| 679 | + EXPECT_EQ(asc_graphs.size(), 1UL); | ||
| 677 | } | 680 | } |
| 678 | 681 | ||
| 679 | TEST_F(VectorFuncSt, cycle_bugfix) { | 682 | TEST_F(VectorFuncSt, cycle_bugfix) { |
| @@ -1508,7 +1511,10 @@ TEST_F(VectorFuncSt, CastNotFusion) { | |||
| 1508 | ++cast_num; | 1511 | ++cast_num; |
| 1509 | } | 1512 | } |
| 1510 | } | 1513 | } |
| 1511 | - EXPECT_EQ(cast_num, 1UL); | 1514 | + // float->int64 casts cross the 8<->4 byte boundary: the capability gate |
| 1515 | + // keeps them in the root graph (CastExtend) instead of fusing into VF, | ||
| 1516 | + // so both casts remain visible in the impl graph. | ||
| 1517 | + EXPECT_EQ(cast_num, 2UL); | ||
| 1512 | } | 1518 | } |
| 1513 | 1519 | ||
| 1514 | TEST_F(VectorFuncSt, MaximumNotFusion) { | 1520 | TEST_F(VectorFuncSt, MaximumNotFusion) { |
| @@ -0,0 +1,374 @@ | |||
| 1 | +/** | ||
| 2 | + * Copyright (c) 2026 Huawei Technologies Co., Ltd. | ||
| 3 | + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of | ||
| 4 | + * CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 5 | + * Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 6 | + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 7 | + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 8 | + * See LICENSE in the root of the software repository for the full text of the License. | ||
| 9 | + */ | ||
| 10 | + | ||
| 11 | + | ||
| 12 | + | ||
| 13 | + | ||
| 14 | + | ||
| 15 | + | ||
| 16 | + | ||
| 17 | + | ||
| 18 | + | ||
| 19 | + | ||
| 20 | + | ||
| 21 | + | ||
| 22 | + | ||
| 23 | + | ||
| 24 | + | ||
| 25 | + | ||
| 26 | +namespace codegen { | ||
| 27 | +namespace { | ||
| 28 | + | ||
| 29 | +class ArangeApiCallTest : public testing::Test { | ||
| 30 | + protected: | ||
| 31 | + ArangeApiCallTest() { | ||
| 32 | + ge::PlatformContext::GetInstance().Reset(); | ||
| 33 | + ge::PlatformContext::GetInstance().SetPlatform("3510"); | ||
| 34 | + ge::RuntimeStub::SetInstance(std::make_shared<ge::RuntimeStubV2Common>()); | ||
| 35 | + } | ||
| 36 | + | ||
| 37 | + ~ArangeApiCallTest() override { | ||
| 38 | + ge::RuntimeStub::Reset(); | ||
| 39 | + ge::PlatformContext::GetInstance().Reset(); | ||
| 40 | + } | ||
| 41 | +}; | ||
| 42 | + | ||
| 43 | +struct ArangeGraph { | ||
| 44 | + af::AscGraph graph{"arange_graph"}; | ||
| 45 | + af::AscNodePtr node; | ||
| 46 | + Tiler tiler; | ||
| 47 | + TPipe tpipe{"tpipe", tiler}; | ||
| 48 | + ascir::AxisId axis_id{af::kIdNone}; | ||
| 49 | + | ||
| 50 | + ArangeGraph(const af::Expression &base, const af::Expression &step, ge::DataType dtype, | ||
| 51 | + bool nonvectorized_outer_axis = false, bool vectorized_outer_axis = false) { | ||
| 52 | + const auto size = graph.CreateSizeVar("size"); | ||
| 53 | + const auto axis = graph.CreateAxis("axis", size); | ||
| 54 | + axis_id = axis.id; | ||
| 55 | + af::ascir_op::Arange op("arange"); | ||
| 56 | + graph.AddNode(op); | ||
| 57 | + op.ir_attr.SetBase(base); | ||
| 58 | + op.ir_attr.SetStep(step); | ||
| 59 | + *op.y.axis = {axis.id}; | ||
| 60 | + *op.y.repeats = {size}; | ||
| 61 | + *op.y.strides = {af::ops::One}; | ||
| 62 | + if (nonvectorized_outer_axis || vectorized_outer_axis) { | ||
| 63 | + const auto outer_axis = graph.CreateAxis("outer_axis", af::Symbol(2)); | ||
| 64 | + axis_id = outer_axis.id; | ||
| 65 | + *op.y.axis = {outer_axis.id, axis.id}; | ||
| 66 | + *op.y.repeats = {af::Symbol(2), size}; | ||
| 67 | + *op.y.strides = {size, af::ops::One}; | ||
| 68 | + (void)tiler.AddAxis(outer_axis); | ||
| 69 | + } | ||
| 70 | + node = graph.FindNode("arange"); | ||
| 71 | + if (node == nullptr) { | ||
| 72 | + return; | ||
| 73 | + } | ||
| 74 | + auto &out = node->outputs[0].attr; | ||
| 75 | + out.dtype = dtype; | ||
| 76 | + out.vectorized_axis = {axis.id}; | ||
| 77 | + out.vectorized_strides = {af::ops::One}; | ||
| 78 | + if (vectorized_outer_axis) { | ||
| 79 | + out.vectorized_axis = *op.y.axis; | ||
| 80 | + out.vectorized_strides = *op.y.strides; | ||
| 81 | + } | ||
| 82 | + out.mem.tensor_id = 0; | ||
| 83 | + out.mem.position = af::Position::kPositionVecOut; | ||
| 84 | + out.mem.alloc_type = af::AllocType::kAllocTypeBuffer; | ||
| 85 | + out.buf.id = 1; | ||
| 86 | + out.opt.merge_scope = af::kIdNone; | ||
| 87 | + node->attr.api.compute_type = af::ComputeType::kComputeElewise; | ||
| 88 | + node->attr.api.type = af::ApiType::kAPITypeCompute; | ||
| 89 | + node->attr.api.unit = af::ComputeUnit::kUnitVector; | ||
| 90 | + (void)tpipe.AddTensor(node->outputs[0]); | ||
| 91 | + (void)tiler.AddAxis(axis); | ||
| 92 | + tiler.AddSizeVar(af::SizeVar(size)); | ||
| 93 | + } | ||
| 94 | +}; | ||
| 95 | + | ||
| 96 | +std::unique_ptr<ApiCall> MakeInitedArangeCall(const ArangeGraph &g) { | ||
| 97 | + std::unique_ptr<ApiCall> call(CreateApiCallObject(g.node)); | ||
| 98 | + if (call == nullptr) { | ||
| 99 | + return nullptr; | ||
| 100 | + } | ||
| 101 | + if (call->Init(g.node) != af::SUCCESS) { | ||
| 102 | + return nullptr; | ||
| 103 | + } | ||
| 104 | + return call; | ||
| 105 | +} | ||
| 106 | +} // namespace | ||
| 107 | + | ||
| 108 | +TEST_F(ArangeApiCallTest, FactoryCreatesRegisteredOrdinaryApiCall) { | ||
| 109 | + ArangeGraph g(af::Expression::Parse("base"), af::Expression::Parse("step"), ge::DT_INT32); | ||
| 110 | + ASSERT_NE(g.node, nullptr); | ||
| 111 | + std::unique_ptr<ApiCall> call(CreateApiCallObject(g.node)); | ||
| 112 | + ASSERT_NE(call, nullptr); | ||
| 113 | + EXPECT_EQ(call->api_name_, "Arange"); | ||
| 114 | + EXPECT_EQ(call->Init(g.node), af::SUCCESS); | ||
| 115 | +} | ||
| 116 | + | ||
| 117 | +TEST_F(ArangeApiCallTest, CodegenImplUsesArangeApiCall) { | ||
| 118 | + af::ascir::ArangeAscIrCodegenImplV2 impl; | ||
| 119 | + EXPECT_EQ(impl.GetApiCallName(), "ArangeApiCall"); | ||
| 120 | + EXPECT_EQ(impl.GetApiName(), "Arange"); | ||
| 121 | +} | ||
| 122 | + | ||
| 123 | +TEST_F(ArangeApiCallTest, GeneratesInt32LoopWithDynamicBaseAndStep) { | ||
| 124 | + ArangeGraph g(af::Expression::Parse("base"), af::Expression::Parse("step"), ge::DT_INT32); | ||
| 125 | + const auto *output = g.tpipe.GetTensor(g.node->outputs[0].attr.mem.tensor_id); | ||
| 126 | + ASSERT_NE(output, nullptr); | ||
| 127 | + auto call = MakeInitedArangeCall(g); | ||
| 128 | + ASSERT_NE(call, nullptr); | ||
| 129 | + | ||
| 130 | + std::string result; | ||
| 131 | + ASSERT_EQ(call->Generate(g.tpipe, {}, result), af::SUCCESS); | ||
| 132 | + RecordProperty("generated_code", result); | ||
| 133 | + EXPECT_NE(result.find("for (int64_t arange_i = 0; arange_i < " + output->actual_size.Str() + "; ++arange_i) {"), | ||
| 134 | + std::string::npos); | ||
| 135 | + EXPECT_NE( | ||
| 136 | + result.find( | ||
| 137 | + output->Str() + | ||
| 138 | + ".SetValue(static_cast<uint32_t>(0 + arange_i), static_cast<int32_t>((base) + (0 + arange_i) * (step)));"), | ||
| 139 | + std::string::npos); | ||
| 140 | +} | ||
| 141 | + | ||
| 142 | +TEST_F(ArangeApiCallTest, GeneratesInt64LoopWithNegativeBaseAndStep) { | ||
| 143 | + ArangeGraph g(af::Symbol(-3), af::Symbol(-2), ge::DT_INT64); | ||
| 144 | + auto call = MakeInitedArangeCall(g); | ||
| 145 | + ASSERT_NE(call, nullptr); | ||
| 146 | + | ||
| 147 | + std::string result; | ||
| 148 | + ASSERT_EQ(call->Generate(g.tpipe, {}, result), af::SUCCESS); | ||
| 149 | + RecordProperty("generated_code", result); | ||
| 150 | + EXPECT_NE(result.find( | ||
| 151 | + ".SetValue(static_cast<uint32_t>(0 + arange_i), static_cast<int64_t>((-3) + (0 + arange_i) * (-2)));"), | ||
| 152 | + std::string::npos); | ||
| 153 | +} | ||
| 154 | + | ||
| 155 | +TEST_F(ArangeApiCallTest, GeneratesLoopWithZeroStep) { | ||
| 156 | + ArangeGraph g(af::Symbol(5), af::Symbol(0), ge::DT_INT32); | ||
| 157 | + auto call = MakeInitedArangeCall(g); | ||
| 158 | + ASSERT_NE(call, nullptr); | ||
| 159 | + | ||
| 160 | + std::string result; | ||
| 161 | + ASSERT_EQ(call->Generate(g.tpipe, {}, result), af::SUCCESS); | ||
| 162 | + RecordProperty("generated_code", result); | ||
| 163 | + EXPECT_NE( | ||
| 164 | + result.find(".SetValue(static_cast<uint32_t>(0 + arange_i), static_cast<int32_t>((5) + (0 + arange_i) * (0)));"), | ||
| 165 | + std::string::npos); | ||
| 166 | +} | ||
| 167 | + | ||
| 168 | +TEST_F(ArangeApiCallTest, GeneratesLoopWithUnitStep) { | ||
| 169 | + ArangeGraph g(af::Symbol(7), af::Symbol(1), ge::DT_INT32); | ||
| 170 | + auto call = MakeInitedArangeCall(g); | ||
| 171 | + ASSERT_NE(call, nullptr); | ||
| 172 | + | ||
| 173 | + std::string result; | ||
| 174 | + ASSERT_EQ(call->Generate(g.tpipe, {}, result), af::SUCCESS); | ||
| 175 | + RecordProperty("generated_code", result); | ||
| 176 | + EXPECT_NE( | ||
| 177 | + result.find(".SetValue(static_cast<uint32_t>(0 + arange_i), static_cast<int32_t>((7) + (0 + arange_i) * (1)));"), | ||
| 178 | + std::string::npos); | ||
| 179 | +} | ||
| 180 | + | ||
| 181 | +TEST_F(ArangeApiCallTest, GeneratesBoundedCurrentAxisScalarWrite) { | ||
| 182 | + ArangeGraph g(af::Expression::Parse("base"), af::Expression::Parse("step"), ge::DT_INT32); | ||
| 183 | + const auto *output = g.tpipe.GetTensor(g.node->outputs[0].attr.mem.tensor_id); | ||
| 184 | + ASSERT_NE(output, nullptr); | ||
| 185 | + auto call = MakeInitedArangeCall(g); | ||
| 186 | + ASSERT_NE(call, nullptr); | ||
| 187 | + | ||
| 188 | + std::string result; | ||
| 189 | + ASSERT_EQ(call->Generate(g.tpipe, {g.axis_id}, result), af::SUCCESS); | ||
| 190 | + RecordProperty("generated_code", result); | ||
| 191 | + EXPECT_NE(result.find("arange_i < 1;"), std::string::npos); | ||
| 192 | + EXPECT_NE(result.find(".SetValue(static_cast<uint32_t>((int64_t)axis + arange_i), " | ||
| 193 | + "static_cast<int32_t>((base) + ((int64_t)axis + arange_i) * (step)));"), | ||
| 194 | + std::string::npos); | ||
| 195 | +} | ||
| 196 | + | ||
| 197 | +TEST_F(ArangeApiCallTest, GeneratesBoundedVectorizedOuterAxisSlice) { | ||
| 198 | + ArangeGraph g(af::Symbol(7), af::Symbol(1), ge::DT_INT32, false, true); | ||
| 199 | + const auto *output = g.tpipe.GetTensor(g.node->outputs[0].attr.mem.tensor_id); | ||
| 200 | + ASSERT_NE(output, nullptr); | ||
| 201 | + ASSERT_EQ(output->vectorized_axis.size(), 2U); | ||
| 202 | + EXPECT_EQ(g.tpipe.tiler.TensorActualSize(*output), "(2 - 1) * t->size + (axis_actual_size - 1) + 1"); | ||
| 203 | + auto call = MakeInitedArangeCall(g); | ||
| 204 | + ASSERT_NE(call, nullptr); | ||
| 205 | + std::string result; | ||
| 206 | + ASSERT_EQ(call->Generate(g.tpipe, {g.axis_id}, result), af::SUCCESS); | ||
| 207 | + RecordProperty("generated_code", result); | ||
| 208 | + EXPECT_NE(result.find("arange_i < (axis_actual_size - 1) + 1;"), std::string::npos); | ||
| 209 | + EXPECT_NE(result.find(".SetValue(static_cast<uint32_t>((int64_t)outer_axis * (int64_t)t->size + arange_i), " | ||
| 210 | + "static_cast<int32_t>((7) + ((int64_t)outer_axis * (int64_t)t->size + arange_i) * (1)));"), | ||
| 211 | + std::string::npos); | ||
| 212 | +} | ||
| 213 | + | ||
| 214 | +TEST_F(ArangeApiCallTest, GeneratesBoundedMultipleCurrentAxesScalarWrite) { | ||
| 215 | + ArangeGraph g(af::Symbol(-3), af::Symbol(-2), ge::DT_INT64, false, true); | ||
| 216 | + const auto *output = g.tpipe.GetTensor(g.node->outputs[0].attr.mem.tensor_id); | ||
| 217 | + ASSERT_NE(output, nullptr); | ||
| 218 | + auto call = MakeInitedArangeCall(g); | ||
| 219 | + ASSERT_NE(call, nullptr); | ||
| 220 | + std::string result; | ||
| 221 | + ASSERT_EQ(call->Generate(g.tpipe, output->vectorized_axis, result), af::SUCCESS); | ||
| 222 | + RecordProperty("generated_code", result); | ||
| 223 | + EXPECT_NE(result.find("arange_i < 1;"), std::string::npos); | ||
| 224 | + EXPECT_NE(result.find(".SetValue(static_cast<uint32_t>((int64_t)outer_axis * (int64_t)t->size + " | ||
| 225 | + "(int64_t)axis + arange_i),"), | ||
| 226 | + std::string::npos); | ||
| 227 | +} | ||
| 228 | + | ||
| 229 | +TEST_F(ArangeApiCallTest, GeneratesBroadcastOuterAxisRepeatedSlice) { | ||
| 230 | + ArangeGraph g(af::Symbol(0), af::Symbol(1), ge::DT_INT32, false, true); | ||
| 231 | + // 广播行轴: 逻辑 stride 为 0, 物化布局为连续展开(与分区器 MaterializeArangeViewContiguous 对齐)。 | ||
| 232 | + g.tpipe.tensors.at(0).axis_strides = {af::ops::Zero, af::ops::One}; | ||
| 233 | + auto call = MakeInitedArangeCall(g); | ||
| 234 | + ASSERT_NE(call, nullptr); | ||
| 235 | + std::string result; | ||
| 236 | + ASSERT_EQ(call->Generate(g.tpipe, {}, result), af::SUCCESS); | ||
| 237 | + RecordProperty("generated_code", result); | ||
| 238 | + // 非固定广播轴在调用内部展开: 每帧物理偏移按连续布局前进, 取值按逻辑偏移(广播轴贡献 0)重复。 | ||
| 239 | + EXPECT_NE(result.find("for (int64_t arange_b0 = 0; arange_b0 < 2; ++arange_b0) {"), std::string::npos); | ||
| 240 | + EXPECT_NE(result.find("arange_i < (axis_actual_size - 1) + 1;"), std::string::npos); | ||
| 241 | + EXPECT_NE(result.find(".SetValue(static_cast<uint32_t>(0 + arange_b0 * t->size + arange_i), " | ||
| 242 | + "static_cast<int32_t>((0) + (0 + arange_i) * (1)));"), | ||
| 243 | + std::string::npos); | ||
| 244 | +} | ||
| 245 | + | ||
| 246 | +TEST_F(ArangeApiCallTest, RejectsLogicalStrideDifferentFromPhysicalStride) { | ||
| 247 | + ArangeGraph g(af::Symbol(7), af::Symbol(1), ge::DT_INT32); | ||
| 248 | + g.tpipe.tensors.at(0).axis_strides = {af::Symbol(2)}; | ||
| 249 | + auto call = MakeInitedArangeCall(g); | ||
| 250 | + ASSERT_NE(call, nullptr); | ||
| 251 | + std::string result; | ||
| 252 | + EXPECT_EQ(call->Generate(g.tpipe, {}, result), af::FAILED); | ||
| 253 | + EXPECT_TRUE(result.empty()); | ||
| 254 | +} | ||
| 255 | + | ||
| 256 | +TEST_F(ArangeApiCallTest, RejectsPhysicalHolesAtZeroOffset) { | ||
| 257 | + ArangeGraph g(af::Symbol(7), af::Symbol(1), ge::DT_INT32); | ||
| 258 | + g.tpipe.tensors.at(0).vectorized_strides = {af::Symbol(2)}; | ||
| 259 | + auto call = MakeInitedArangeCall(g); | ||
| 260 | + ASSERT_NE(call, nullptr); | ||
| 261 | + std::string result; | ||
| 262 | + EXPECT_EQ(call->Generate(g.tpipe, {}, result), af::FAILED); | ||
| 263 | + EXPECT_TRUE(result.empty()); | ||
| 264 | +} | ||
| 265 | + | ||
| 266 | +TEST_F(ArangeApiCallTest, RejectsLoopRangeLargerThanTensorExtent) { | ||
| 267 | + ArangeGraph g(af::Symbol(7), af::Symbol(1), ge::DT_INT32); | ||
| 268 | + g.tpipe.tensors.at(0).axis_size = {af::Symbol(1)}; | ||
| 269 | + auto call = MakeInitedArangeCall(g); | ||
| 270 | + ASSERT_NE(call, nullptr); | ||
| 271 | + std::string result; | ||
| 272 | + EXPECT_EQ(call->Generate(g.tpipe, {g.axis_id}, result), af::FAILED); | ||
| 273 | + EXPECT_TRUE(result.empty()); | ||
| 274 | +} | ||
| 275 | + | ||
| 276 | +TEST_F(ArangeApiCallTest, RejectsUnprovenActualSpanAtZeroOffset) { | ||
| 277 | + ArangeGraph g(af::Symbol(7), af::Symbol(1), ge::DT_INT32); | ||
| 278 | + g.tpipe.tensors.at(0).axis_size = {af::Symbol(1)}; | ||
| 279 | + g.tiler.axis_map.at(g.axis_id).type = ascir::Axis::Type::kAxisTypeTileInner; | ||
| 280 | + auto call = MakeInitedArangeCall(g); | ||
| 281 | + ASSERT_NE(call, nullptr); | ||
| 282 | + std::string result; | ||
| 283 | + EXPECT_EQ(call->Generate(g.tpipe, {}, result), af::FAILED); | ||
| 284 | + EXPECT_TRUE(result.empty()); | ||
| 285 | +} | ||
| 286 | + | ||
| 287 | +TEST_F(ArangeApiCallTest, NonvectorizedOuterAxisOnlyContributesToLogicalOffset) { | ||
| 288 | + ArangeGraph g(af::Expression::Parse("base"), af::Expression::Parse("step"), ge::DT_INT32, true); | ||
| 289 | + const auto *output = g.tpipe.GetTensor(g.node->outputs[0].attr.mem.tensor_id); | ||
| 290 | + ASSERT_NE(output, nullptr); | ||
| 291 | + ASSERT_EQ(output->axis.size(), 2U); | ||
| 292 | + ASSERT_EQ(output->vectorized_axis.size(), 1U); | ||
| 293 | + ASSERT_NE(output->vectorized_axis.front(), g.axis_id); | ||
| 294 | + auto call = MakeInitedArangeCall(g); | ||
| 295 | + ASSERT_NE(call, nullptr); | ||
| 296 | + | ||
| 297 | + std::string result; | ||
| 298 | + ASSERT_EQ(call->Generate(g.tpipe, {g.axis_id}, result), af::SUCCESS); | ||
| 299 | + RecordProperty("generated_code", result); | ||
| 300 | + EXPECT_NE(result.find(".SetValue(static_cast<uint32_t>(0 + arange_i), static_cast<int32_t>((base) + " | ||
| 301 | + "((int64_t)outer_axis * (int64_t)t->size + arange_i) * (step)));"), | ||
| 302 | + std::string::npos); | ||
| 303 | +} | ||
| 304 | + | ||
| 305 | +TEST_F(ArangeApiCallTest, KernelGenerateOrdinaryFallbackWithLoopAndStore) { | ||
| 306 | + for (const bool full_local_view : {false, true}) { | ||
| 307 | + SCOPED_TRACE(full_local_view); | ||
| 308 | + ArangeGraph g(af::Symbol(7), af::Symbol(-2), ge::DT_INT32, !full_local_view, full_local_view); | ||
| 309 | + ASSERT_NE(g.node, nullptr); | ||
| 310 | + g.node->attr.sched.axis = g.node->outputs[0].attr.axis; | ||
| 311 | + g.node->attr.sched.loop_axis = g.axis_id; | ||
| 312 | + | ||
| 313 | + af::ascir_op::Store store("store"); | ||
| 314 | + af::ascir_op::Output output("output"); | ||
| 315 | + g.graph.AddNode(store); | ||
| 316 | + g.graph.AddNode(output); | ||
| 317 | + ASSERT_EQ(af::GraphUtils::AddEdge(g.node->GetOutDataAnchor(0), g.graph.FindNode("store")->GetInDataAnchor(0)), | ||
| 318 | + af::GRAPH_SUCCESS); | ||
| 319 | + output.x = store.y; | ||
| 320 | + output.ir_attr.SetIndex(0); | ||
| 321 | + const auto store_node = g.graph.FindNode("store"); | ||
| 322 | + const auto output_node = g.graph.FindNode("output"); | ||
| 323 | + ASSERT_NE(store_node, nullptr); | ||
| 324 | + ASSERT_NE(output_node, nullptr); | ||
| 325 | + store_node->outputs[0].attr = g.node->outputs[0].attr; | ||
| 326 | + store_node->outputs[0].attr.mem.tensor_id = 1; | ||
| 327 | + store_node->outputs[0].attr.mem.alloc_type = af::AllocType::kAllocTypeGlobal; | ||
| 328 | + store_node->outputs[0].attr.mem.position = af::Position::kPositionGM; | ||
| 329 | + store_node->attr.api.unit = af::ComputeUnit::kUnitMTE3; | ||
| 330 | + store_node->attr.api.type = af::ApiType::kAPITypeCompute; | ||
| 331 | + store_node->attr.api.compute_type = af::ComputeType::kComputeStore; | ||
| 332 | + store_node->attr.sched.axis = g.node->attr.sched.axis; | ||
| 333 | + store_node->attr.sched.loop_axis = full_local_view ? af::kIdNone : g.axis_id; | ||
| 334 | + | ||
| 335 | + ascir::FusedScheduledResult fused; | ||
| 336 | + fused.output_nodes.push_back(output_node); | ||
| 337 | + Kernel kernel(g.graph.GetName()); | ||
| 338 | + ASSERT_EQ(Kernel::ParseGraph(g.graph, fused, kernel), af::SUCCESS); | ||
| 339 | + std::string result; | ||
| 340 | + ASSERT_EQ(kernel.Generate(g.graph.GetName(), "AutofuseTilingData", result, g.graph), af::SUCCESS); | ||
| 341 | + RecordProperty(full_local_view ? "full_view_kernel" : "row_view_kernel", result); | ||
| 342 | + EXPECT_EQ(result.find("VFCall"), std::string::npos); | ||
| 343 | + EXPECT_EQ(result.find("Reg::Arange"), std::string::npos); | ||
| 344 | + const auto loop_pos = result.find("for (int outer_axis = 0;"); | ||
| 345 | + const auto alloc_pos = result.find("LocalTensor<int32_t> local_0 = b1.Get<int32_t>();"); | ||
| 346 | + const auto actual_size_pos = result.find("const uint32_t local_0_actual_size ="); | ||
| 347 | + const auto write_pos = result.find("local_0.SetValue("); | ||
| 348 | + const auto store_pos = result.find("DataCopy"); | ||
| 349 | + ASSERT_NE(loop_pos, std::string::npos) << result; | ||
| 350 | + ASSERT_NE(alloc_pos, std::string::npos) << result; | ||
| 351 | + ASSERT_NE(actual_size_pos, std::string::npos) << result; | ||
| 352 | + ASSERT_NE(write_pos, std::string::npos) << result; | ||
| 353 | + ASSERT_NE(store_pos, std::string::npos) << result; | ||
| 354 | + EXPECT_LT(alloc_pos, write_pos); | ||
| 355 | + EXPECT_LT(actual_size_pos, write_pos); | ||
| 356 | + EXPECT_LT(loop_pos, write_pos); | ||
| 357 | + EXPECT_LT(write_pos, store_pos); | ||
| 358 | + EXPECT_NE(result.find("((int64_t)outer_axis * (int64_t)t->size + arange_i) * (-2)"), std::string::npos); | ||
| 359 | + if (full_local_view) { | ||
| 360 | + EXPECT_LT(actual_size_pos, loop_pos); | ||
| 361 | + EXPECT_NE(result.find("arange_i < (axis_actual_size - 1) + 1;"), std::string::npos); | ||
| 362 | + EXPECT_NE(result.find("static_cast<uint32_t>((int64_t)outer_axis * (int64_t)t->size + arange_i)"), | ||
| 363 | + std::string::npos); | ||
| 364 | + EXPECT_NE(result.find("KernelUtils::BlkAlign<int32_t>((2 - 1) * t->size + (t->size - 1) + 1)"), | ||
| 365 | + std::string::npos); | ||
| 366 | + } else { | ||
| 367 | + EXPECT_LT(loop_pos, actual_size_pos); | ||
| 368 | + EXPECT_NE(result.find("arange_i < local_0_actual_size;"), std::string::npos); | ||
| 369 | + EXPECT_NE(result.find("static_cast<uint32_t>(0 + arange_i)"), std::string::npos); | ||
| 370 | + EXPECT_NE(result.find("KernelUtils::BlkAlign<int32_t>((t->size - 1) + 1)"), std::string::npos); | ||
| 371 | + } | ||
| 372 | + } | ||
| 373 | +} | ||
| 374 | +} // namespace codegen | ||
| @@ -136,7 +136,7 @@ TEST(CodegenKernel, StoreRegApiCall_TwoStoreOneOutput) { | |||
| 136 | call_1.inputs.push_back(&x1); | 136 | call_1.inputs.push_back(&x1); |
| 137 | call_1.Generate(tpipe, vector<af::AxisId>{}, result); | 137 | call_1.Generate(tpipe, vector<af::AxisId>{}, result); |
| 138 | EXPECT_EQ(result, std::string{"DataCopyPadExtend<float, AscendC::PaddingMode::Normal>(local_1[0 + 1], local_0[0], " | 138 | EXPECT_EQ(result, std::string{"DataCopyPadExtend<float, AscendC::PaddingMode::Normal>(local_1[0 + 1], local_0[0], " |
| 139 | - "z0_t_size, 1, (16 - 1), 0);\n"}); | 139 | + "z0_t_size, 1, (8 - 1), 0);\n"}); |
| 140 | } | 140 | } |
| 141 | 141 | ||
| 142 | namespace { | 142 | namespace { |
| @@ -630,7 +630,8 @@ TEST(VFLoopTest, UsesLogicalOuterStrideForAlignedArangeLayout) { | |||
| 630 | af::SUCCESS); | 630 | af::SUCCESS); |
| 631 | const auto arange_call = GetLineContaining(result, "AscendC::Reg::Arange"); | 631 | const auto arange_call = GetLineContaining(result, "AscendC::Reg::Arange"); |
| 632 | ASSERT_FALSE(arange_call.empty()) << result; | 632 | ASSERT_FALSE(arange_call.empty()) << result; |
| 633 | - EXPECT_NE(arange_call.find("row * t->cols"), std::string::npos); | 633 | + EXPECT_NE(arange_call.find("row * arange_stride_0_0"), std::string::npos); |
| 634 | + EXPECT_EQ(arange_call.find("t->"), std::string::npos); | ||
| 634 | EXPECT_NE(arange_call.find("col * ELEMENT_PER_VECTOR_LENGTH"), std::string::npos); | 635 | EXPECT_NE(arange_call.find("col * ELEMENT_PER_VECTOR_LENGTH"), std::string::npos); |
| 635 | EXPECT_EQ(arange_call.find("row * 16"), std::string::npos); | 636 | EXPECT_EQ(arange_call.find("row * 16"), std::string::npos); |
| 636 | loop.Destruct(); | 637 | loop.Destruct(); |
| @@ -1079,6 +1080,7 @@ TEST(CodegenKernel, VfCall_TwoDimLoad) { | |||
| 1079 | 1080 | ||
| 1080 | std::stringstream func_def; | 1081 | std::stringstream func_def; |
| 1081 | EXPECT_EQ(call.GenerateFuncDefinition(tpipe, tiler, func_def), 0); | 1082 | EXPECT_EQ(call.GenerateFuncDefinition(tpipe, tiler, func_def), 0); |
| 1083 | + EXPECT_NE(func_def.str().find("sizeof(float)"), std::string::npos); | ||
| 1082 | 1084 | ||
| 1083 | std::string result; | 1085 | std::string result; |
| 1084 | call.Generate(tpipe, vector<af::AxisId>{}, result); | 1086 | call.Generate(tpipe, vector<af::AxisId>{}, result); |
| @@ -1088,7 +1090,7 @@ TEST(CodegenKernel, VfCall_TwoDimLoad) { | |||
| 1088 | "(defined(__NPU_ARCH__) && (__NPU_ARCH__ == 5102 || __NPU_ARCH__ == 3510 || __NPU_ARCH__ == 9202))\n" | 1090 | "(defined(__NPU_ARCH__) && (__NPU_ARCH__ == 5102 || __NPU_ARCH__ == 3510 || __NPU_ARCH__ == 9202))\n" |
| 1089 | "AscendC::SetCtrlSpr<60, 60>(0);\n" | 1091 | "AscendC::SetCtrlSpr<60, 60>(0);\n" |
| 1090 | "VFCallvf((__local_mem__ float *)local_1[0].GetPhyAddr(), (__local_mem__ float " | 1092 | "VFCallvf((__local_mem__ float *)local_1[0].GetPhyAddr(), (__local_mem__ float " |
| 1091 | - "*)local_0[0].GetPhyAddr(), t->s0 * t->s1);\n" | 1093 | + "*)local_0[0].GetPhyAddr(), t->s0 * t->s1, 1, 1);\n" |
| 1092 | "#endif\n"}); | 1094 | "#endif\n"}); |
| 1093 | } | 1095 | } |
| 1094 | 1096 | ||
| @@ -1478,7 +1480,7 @@ TEST(CodegenKernel, VfCall_TwoDim_Scalar) { | |||
| 1478 | "(defined(__NPU_ARCH__) && (__NPU_ARCH__ == 5102 || __NPU_ARCH__ == 3510 || __NPU_ARCH__ == 9202))\n" | 1480 | "(defined(__NPU_ARCH__) && (__NPU_ARCH__ == 5102 || __NPU_ARCH__ == 3510 || __NPU_ARCH__ == 9202))\n" |
| 1479 | "AscendC::SetCtrlSpr<60, 60>(0);\n" | 1481 | "AscendC::SetCtrlSpr<60, 60>(0);\n" |
| 1480 | "VFCallvf((__local_mem__ float *)local_1[0].GetPhyAddr(), (__local_mem__ float " | 1482 | "VFCallvf((__local_mem__ float *)local_1[0].GetPhyAddr(), (__local_mem__ float " |
| 1481 | - "*)local_1[0].GetPhyAddr(), scalar_0, t->s0 * t->s1);\n" | 1483 | + "*)local_1[0].GetPhyAddr(), scalar_0, t->s0 * t->s1, 1, 1);\n" |
| 1482 | "#endif\n"}); | 1484 | "#endif\n"}); |
| 1483 | } | 1485 | } |
| 1484 | 1486 | ||
| @@ -1697,7 +1699,7 @@ TEST(CodegenKernel, VfCall_ThreeDimLoad) { | |||
| 1697 | "(defined(__NPU_ARCH__) && (__NPU_ARCH__ == 5102 || __NPU_ARCH__ == 3510 || __NPU_ARCH__ == 9202))\n" | 1699 | "(defined(__NPU_ARCH__) && (__NPU_ARCH__ == 5102 || __NPU_ARCH__ == 3510 || __NPU_ARCH__ == 9202))\n" |
| 1698 | "AscendC::SetCtrlSpr<60, 60>(0);\n" | 1700 | "AscendC::SetCtrlSpr<60, 60>(0);\n" |
| 1699 | "VFCallvf((__local_mem__ float *)local_1[0].GetPhyAddr(), (__local_mem__ float " | 1701 | "VFCallvf((__local_mem__ float *)local_1[0].GetPhyAddr(), (__local_mem__ float " |
| 1700 | - "*)local_0[0].GetPhyAddr(), t->s0 * t->s1, t->s2, (2 * t->s2), (2 * t->s2));\n" | 1702 | + "*)local_0[0].GetPhyAddr(), t->s0 * t->s1, t->s2, (2 * t->s2), 1, (2 * t->s2), 1);\n" |
| 1701 | "#endif\n"}); | 1703 | "#endif\n"}); |
| 1702 | } | 1704 | } |
| 1703 | 1705 | ||
| @@ -1885,7 +1887,8 @@ TEST(CodegenKernel, VfCall_FiveDimLoad) { | |||
| 1885 | "*)local_1[outer_for_0 * (5 * t->s1 * t->s2 * t->s3 * t->s4)].GetPhyAddr(), (__local_mem__ float " | 1887 | "*)local_1[outer_for_0 * (5 * t->s1 * t->s2 * t->s3 * t->s4)].GetPhyAddr(), (__local_mem__ float " |
| 1886 | "*)local_0[outer_for_0 * (t->s1 * t->s2 * t->s3 * t->s4)].GetPhyAddr(), t->s1, t->s2, t->s3, t->s4, (4 * " | 1888 | "*)local_0[outer_for_0 * (t->s1 * t->s2 * t->s3 * t->s4)].GetPhyAddr(), t->s1, t->s2, t->s3, t->s4, (4 * " |
| 1887 | "t->s2 " | 1889 | "t->s2 " |
| 1888 | - "* t->s3 * t->s4), (3 * t->s3 * t->s4), (2 * t->s4), (t->s2 * t->s3 * t->s4), (t->s3 * t->s4), t->s4);\n\n" | 1890 | + "* t->s3 * t->s4), (3 * t->s3 * t->s4), (2 * t->s4), 1, (t->s2 * t->s3 * t->s4), (t->s3 * t->s4), t->s4, " |
| 1891 | + "1);\n\n" | ||
| 1889 | "}\n" | 1892 | "}\n" |
| 1890 | "#endif\n"}); | 1893 | "#endif\n"}); |
| 1891 | } | 1894 | } |
| @@ -2262,23 +2262,25 @@ TEST_F(VfPartition, cast_high_to_low_no_fuse_with_output) { | |||
| 2262 | 2262 | ||
| 2263 | af::ascir_op::Load load("load"); | 2263 | af::ascir_op::Load load("load"); |
| 2264 | load.x = data0.y; | 2264 | load.x = data0.y; |
| 2265 | - load.y.dtype = af::DT_INT64; | 2265 | + load.y.dtype = af::DT_FLOAT; |
| 2266 | 2266 | ||
| 2267 | af::ascir_op::Abs abs("abs"); | 2267 | af::ascir_op::Abs abs("abs"); |
| 2268 | abs.x = load.y; | 2268 | abs.x = load.y; |
| 2269 | - abs.y.dtype = af::DT_INT64; | 2269 | + abs.y.dtype = af::DT_FLOAT; |
| 2270 | 2270 | ||
| 2271 | + // 8字节<->4字节的 Cast 已在能力检查中拒绝 VF 融合(设备上仅填充半数 lane), | ||
| 2272 | + // 高→低场景改用 4字节->2字节组合验证簇合并逻辑。 | ||
| 2271 | af::ascir_op::Cast cast("cast"); | 2273 | af::ascir_op::Cast cast("cast"); |
| 2272 | cast.x = abs.y; | 2274 | cast.x = abs.y; |
| 2273 | - cast.y.dtype = af::DT_INT32; // INT64 -> INT32, 高→低 | 2275 | + cast.y.dtype = af::DT_FLOAT16; // FLOAT -> FLOAT16, 高→低 |
| 2274 | 2276 | ||
| 2275 | - af::ascir_op::Exp exp("exp"); | 2277 | + af::ascir_op::Abs abs2("abs2"); |
| 2276 | - exp.x = cast.y; | 2278 | + abs2.x = cast.y; |
| 2277 | - exp.y.dtype = af::DT_INT32; | 2279 | + abs2.y.dtype = af::DT_FLOAT16; |
| 2278 | 2280 | ||
| 2279 | af::ascir_op::Store store("store"); | 2281 | af::ascir_op::Store store("store"); |
| 2280 | - store.x = exp.y; | 2282 | + store.x = abs2.y; |
| 2281 | - store.y.dtype = af::DT_INT32; | 2283 | + store.y.dtype = af::DT_FLOAT16; |
| 2282 | 2284 | ||
| 2283 | af::ascir_op::Output out("out"); | 2285 | af::ascir_op::Output out("out"); |
| 2284 | out.x = store.y; | 2286 | out.x = store.y; |
| @@ -2295,7 +2297,7 @@ TEST_F(VfPartition, cast_high_to_low_no_fuse_with_output) { | |||
| 2295 | 2297 | ||
| 2296 | // 高→低 Cast 的检查是在合并 Cluster 时进行的 | 2298 | // 高→低 Cast 的检查是在合并 Cluster 时进行的 |
| 2297 | // 这里只有一个分支,所以所有节点应该能融合 | 2299 | // 这里只有一个分支,所以所有节点应该能融合 |
| 2298 | - EXPECT_EQ(sub_graphs.size(), 1UL); | 2300 | + ASSERT_EQ(sub_graphs.size(), 1UL); |
| 2299 | auto cast_node = sub_graphs[0].FindNode("cast"); | 2301 | auto cast_node = sub_graphs[0].FindNode("cast"); |
| 2300 | EXPECT_NE(cast_node, nullptr); | 2302 | EXPECT_NE(cast_node, nullptr); |
| 2301 | } | 2303 | } |
| @@ -242,7 +242,7 @@ class ArangeAscIrCodegenImplV2 : public AscIrCodegenV2 { | |||
| 242 | return true; | 242 | return true; |
| 243 | } | 243 | } |
| 244 | [[nodiscard]] std::string GetApiCallName() const override { | 244 | [[nodiscard]] std::string GetApiCallName() const override { |
| 245 | - return ""; | 245 | + return "ArangeApiCall"; |
| 246 | } | 246 | } |
| 247 | [[nodiscard]] std::string GetApiName() const override { | 247 | [[nodiscard]] std::string GetApiName() const override { |
| 248 | return "Arange"; | 248 | return "Arange"; |
| @@ -434,10 +434,9 @@ class CastAscIrCodegenImplV2 : public AscIrCodegenV2 { | |||
| 434 | 434 | ||
| 435 | [[nodiscard]] bool IsVectorFunctionSupported(const ge::AscNode &node) const override { | 435 | [[nodiscard]] bool IsVectorFunctionSupported(const ge::AscNode &node) const override { |
| 436 | std::map<ge::DataType, std::set<ge::DataType>> supported_map = { | 436 | std::map<ge::DataType, std::set<ge::DataType>> supported_map = { |
| 437 | - {DT_FLOAT, {DT_FLOAT16, DT_INT64, DT_INT16, DT_INT32, DT_BF16}}, | 437 | + {DT_FLOAT, {DT_FLOAT16, DT_INT16, DT_INT32, DT_BF16}}, |
| 438 | {DT_FLOAT16, {DT_UINT8, DT_INT8, DT_FLOAT}}, | 438 | {DT_FLOAT16, {DT_UINT8, DT_INT8, DT_FLOAT}}, |
| 439 | {DT_INT32, {DT_FLOAT, DT_INT16}}, | 439 | {DT_INT32, {DT_FLOAT, DT_INT16}}, |
| 440 | - {DT_INT64, {DT_INT32, DT_FLOAT}}, | ||
| 441 | {DT_BF16, {DT_FLOAT}}, | 440 | {DT_BF16, {DT_FLOAT}}, |
| 442 | {DT_UINT8, {DT_FLOAT16}}, | 441 | {DT_UINT8, {DT_FLOAT16}}, |
| 443 | {DT_INT8, {DT_FLOAT16, DT_INT16}}, | 442 | {DT_INT8, {DT_FLOAT16, DT_INT16}}, |
| @@ -451,6 +450,13 @@ class CastAscIrCodegenImplV2 : public AscIrCodegenV2 { | |||
| 451 | if ((input_dtype_size > output_dtype_size * 2U) || (output_dtype_size > input_dtype_size * 2U)) { | 450 | if ((input_dtype_size > output_dtype_size * 2U) || (output_dtype_size > input_dtype_size * 2U)) { |
| 452 | return false; | 451 | return false; |
| 453 | } | 452 | } |
| 453 | + // 8字节与4字节之间的转换在 MicroAPI 层要求源/目的使用 RegTraitNumTwo 双寄存器 | ||
| 454 | + // (dav_m510 的 s642f32/f322s64 实现按 srcReg.reg[0]/reg[1] 取数)。VF codegen 只 | ||
| 455 | + // 生成单寄存器张量, 该组合在设备上仅填充半数 lane(真机验证: 恰好 50% mismatch, | ||
| 456 | + // 交替 -0.0 模式), 因此拒绝 VF 融合, 保留在根图走 CastExtend(真机验证正确)。 | ||
| 457 | + if ((input_dtype_size == 8U) != (output_dtype_size == 8U)) { | ||
| 458 | + return false; | ||
| 459 | + } | ||
| 454 | 460 | ||
| 455 | auto iter = supported_map.find(node_inputs[0].attr.dtype); | 461 | auto iter = supported_map.find(node_inputs[0].attr.dtype); |
| 456 | if (iter != supported_map.end()) { | 462 | if (iter != supported_map.end()) { |
| @@ -0,0 +1,131 @@ | |||
| 1 | +/** | ||
| 2 | + * Copyright (c) 2026 Huawei Technologies Co., Ltd. | ||
| 3 | + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of | ||
| 4 | + * CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 5 | + * Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 6 | + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 7 | + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 8 | + * See LICENSE in the root of the software repository for the full text of the License. | ||
| 9 | + */ | ||
| 10 | + | ||
| 11 | + | ||
| 12 | + | ||
| 13 | + | ||
| 14 | + | ||
| 15 | + | ||
| 16 | + | ||
| 17 | + | ||
| 18 | +namespace codegen { | ||
| 19 | +Status ArangeApiCall::ParseAttr(const ascir::NodeView &node) { | ||
| 20 | + GE_CHK_GRAPH_STATUS_RET(node->attr.ir_attr->GetAttrValue("base", base_), "Failed to get Arange base attr"); | ||
| 21 | + GE_CHK_GRAPH_STATUS_RET(node->attr.ir_attr->GetAttrValue("step", step_), "Failed to get Arange step attr"); | ||
| 22 | + return af::SUCCESS; | ||
| 23 | +} | ||
| 24 | + | ||
| 25 | +Status ArangeApiCall::Generate(const TPipe &tpipe, const std::vector<ascir::AxisId> ¤t_axis, | ||
| 26 | + const std::vector<std::reference_wrapper<const Tensor>> &inputs, | ||
| 27 | + const std::vector<std::reference_wrapper<const Tensor>> &outputs, | ||
| 28 | + std::string &result) const { | ||
| 29 | + GE_ASSERT_TRUE(inputs.empty(), "Arange ApiCall must have no inputs, but got %zu inputs", inputs.size()); | ||
| 30 | + GE_ASSERT_TRUE(outputs.size() == 1U, "Arange ApiCall must have exactly one output, but got %zu outputs", | ||
| 31 | + outputs.size()); | ||
| 32 | + const auto &output = outputs[0].get(); | ||
| 33 | + GE_ASSERT_TRUE(output.dtype == ge::DT_INT32 || output.dtype == ge::DT_INT64, | ||
| 34 | + "Arange supports only int32 and int64, dtype:%d", static_cast<int32_t>(output.dtype)); | ||
| 35 | + | ||
| 36 | + std::string dtype_name; | ||
| 37 | + GE_CHK_STATUS_RET(Tensor::DtypeName(output.dtype, dtype_name), "Get Arange dtype failed, dtype:%d", | ||
| 38 | + static_cast<int32_t>(output.dtype)); | ||
| 39 | + const auto base = tpipe.tiler.ActualSize(base_); | ||
| 40 | + const auto step = tpipe.tiler.ActualSize(step_); | ||
| 41 | + GE_ASSERT_TRUE(!base.empty() && !step.empty(), "Failed to generate Arange base or step expression"); | ||
| 42 | + const auto logical_offset = tpipe.tiler.Offset(current_axis, output.axis, output.axis_strides); | ||
| 43 | + const auto write_offset = tpipe.tiler.TensorVectorizedOffset(current_axis, output); | ||
| 44 | + // A Loop fixes a prefix of the local view. TensorActualSize(output) still | ||
| 45 | + // includes that prefix, so only the remaining contiguous suffix is written. | ||
| 46 | + auto slice = output; | ||
| 47 | + slice.vectorized_axis.clear(); | ||
| 48 | + slice.vectorized_axis_pos.clear(); | ||
| 49 | + slice.vectorized_strides.clear(); | ||
| 50 | + af::Expression stride = af::Symbol(1); | ||
| 51 | + bool has_fixed_axis = false; | ||
| 52 | + std::vector<std::pair<std::string, std::string>> broadcast_frames; | ||
| 53 | + GE_ASSERT_TRUE(output.vectorized_axis.size() == output.vectorized_axis_pos.size() && | ||
| 54 | + output.vectorized_axis.size() == output.vectorized_strides.size(), | ||
| 55 | + "Arange vectorized layout metadata size mismatch"); | ||
| 56 | + for (size_t i = output.vectorized_axis.size(); i > 0U; --i) { | ||
| 57 | + const auto index = i - 1U; | ||
| 58 | + const auto id = output.vectorized_axis[index]; | ||
| 59 | + const auto pos = output.vectorized_axis_pos[index]; | ||
| 60 | + GE_ASSERT_TRUE(pos < output.axis_size.size() && pos < output.axis_strides.size(), | ||
| 61 | + "Arange vectorized axis position is out of range"); | ||
| 62 | + // 退化轴(size 1 且逻辑 stride 0): 对物理偏移(size 1)和逻辑取值(stride 0) | ||
| 63 | + // 均无贡献, 物化结果即扁平一维序列; 跳过该轴, 扩维由 Broadcast 通用路径处理。 | ||
| 64 | + if (af::SymbolicUtils::StaticCheckEq(output.axis_size[pos], af::sym::kSymbolOne) == af::TriBool::kTrue && | ||
| 65 | + af::SymbolicUtils::StaticCheckEq(output.axis_strides[pos], af::sym::kSymbolZero) == af::TriBool::kTrue) { | ||
| 66 | + continue; | ||
| 67 | + } | ||
| 68 | + const auto &axis = tpipe.tiler.GetAxis(id); | ||
| 69 | + GE_CHK_BOOL_RET_STATUS( | ||
| 70 | + af::SymbolicUtils::StaticCheckEq(output.axis_size[pos], axis.size) == af::TriBool::kTrue && | ||
| 71 | + af::SymbolicUtils::StaticCheckEq(output.vectorized_strides[index], stride) == af::TriBool::kTrue, | ||
| 72 | + af::FAILED, "Arange ApiCall requires a bounded contiguous vectorized layout"); | ||
| 73 | + const bool fixed = std::find(current_axis.begin(), current_axis.end(), id) != current_axis.end(); | ||
| 74 | + for (const auto current : current_axis) { | ||
| 75 | + GE_CHK_BOOL_RET_STATUS(current == id || !tpipe.tiler.IsFrom(current, id), af::FAILED, | ||
| 76 | + "Arange ApiCall cannot prove a derived current-axis local range"); | ||
| 77 | + } | ||
| 78 | + if (fixed) { | ||
| 79 | + GE_CHK_BOOL_RET_STATUS(axis.type == Axis::Type::kAxisTypeOriginal || axis.IsInner(), af::FAILED, | ||
| 80 | + "Arange ApiCall cannot prove the current-axis loop bound"); | ||
| 81 | + has_fixed_axis = true; | ||
| 82 | + } else if (af::SymbolicUtils::StaticCheckEq(output.axis_strides[pos], af::sym::kSymbolZero) == af::TriBool::kTrue) { | ||
| 83 | + // 广播轴: 取值沿该轴重复, 逻辑偏移不前进。作为连续片段边界处理; | ||
| 84 | + // 调用点外层循环不迭代该轴时, 在调用内部展开逐帧写入, | ||
| 85 | + // 物理偏移按连续布局前进, 取值按逻辑 stride(0)保持重复。 | ||
| 86 | + has_fixed_axis = true; | ||
| 87 | + std::string frame_size; | ||
| 88 | + const bool size_equal = | ||
| 89 | + af::SymbolicUtils::StaticCheckEq(output.axis_size[pos], axis.size_expr) == af::TriBool::kTrue; | ||
| 90 | + if (axis.type == Axis::Type::kAxisTypeTileInner || size_equal) { | ||
| 91 | + frame_size = axis.actual_size.Str(); | ||
| 92 | + } else { | ||
| 93 | + frame_size = tpipe.tiler.Size(output.axis_size[pos]); | ||
| 94 | + } | ||
| 95 | + broadcast_frames.emplace_back(frame_size, tpipe.tiler.Size(output.vectorized_strides[index])); | ||
| 96 | + } else { | ||
| 97 | + GE_CHK_BOOL_RET_STATUS( | ||
| 98 | + !has_fixed_axis && af::SymbolicUtils::StaticCheckEq(output.axis_strides[pos], stride) == af::TriBool::kTrue, | ||
| 99 | + af::FAILED, "Arange ApiCall requires a contiguous logical suffix"); | ||
| 100 | + GE_CHK_BOOL_RET_STATUS( | ||
| 101 | + slice.vectorized_axis.empty() || !tpipe.tiler.GetAxis(slice.vectorized_axis.front()).IsInner(), af::FAILED, | ||
| 102 | + "Arange ApiCall cannot flatten a multi-axis inner tail"); | ||
| 103 | + slice.vectorized_axis.insert(slice.vectorized_axis.begin(), id); | ||
| 104 | + slice.vectorized_axis_pos.insert(slice.vectorized_axis_pos.begin(), pos); | ||
| 105 | + slice.vectorized_strides.insert(slice.vectorized_strides.begin(), output.vectorized_strides[index]); | ||
| 106 | + } | ||
| 107 | + stride = af::sym::Mul(stride, output.axis_size[pos]); | ||
| 108 | + } | ||
| 109 | + const auto count = has_fixed_axis ? tpipe.tiler.TensorActualSize(slice) : output.actual_size.Str(); | ||
| 110 | + | ||
| 111 | + std::stringstream ss; | ||
| 112 | + std::string write_addr = write_offset; | ||
| 113 | + for (size_t frame = 0UL; frame < broadcast_frames.size(); ++frame) { | ||
| 114 | + const auto &var = "arange_b" + std::to_string(frame); | ||
| 115 | + ss << "for (int64_t " << var << " = 0; " << var << " < " << broadcast_frames[frame].first << "; ++" << var << ") {" | ||
| 116 | + << std::endl; | ||
| 117 | + write_addr += " + " + var + " * " + broadcast_frames[frame].second; | ||
| 118 | + } | ||
| 119 | + ss << "for (int64_t arange_i = 0; arange_i < " << count << "; ++arange_i) {" << std::endl; | ||
| 120 | + ss << " " << output << ".SetValue(static_cast<uint32_t>(" << write_addr << " + arange_i), static_cast<" << dtype_name | ||
| 121 | + << ">((" << base << ") + (" << logical_offset << " + arange_i) * (" << step << ")));" << std::endl; | ||
| 122 | + ss << "}" << std::endl; | ||
| 123 | + for (size_t frame = 0UL; frame < broadcast_frames.size(); ++frame) { | ||
| 124 | + ss << "}" << std::endl; | ||
| 125 | + } | ||
| 126 | + result = ss.str(); | ||
| 127 | + return af::SUCCESS; | ||
| 128 | +} | ||
| 129 | + | ||
| 130 | +static ApiCallRegister<ArangeApiCall> register_arange_api_call("ArangeApiCall"); | ||
| 131 | +} // namespace codegen | ||
| @@ -0,0 +1,33 @@ | |||
| 1 | +/** | ||
| 2 | + * Copyright (c) 2026 Huawei Technologies Co., Ltd. | ||
| 3 | + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of | ||
| 4 | + * CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 5 | + * Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 6 | + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 7 | + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 8 | + * See LICENSE in the root of the software repository for the full text of the License. | ||
| 9 | + */ | ||
| 10 | + | ||
| 11 | + | ||
| 12 | + | ||
| 13 | + | ||
| 14 | +namespace codegen { | ||
| 15 | +class ArangeApiCall final : public ApiCall { | ||
| 16 | + public: | ||
| 17 | + using ApiCall::Generate; | ||
| 18 | + explicit ArangeApiCall(const std::string &api_name) : ApiCall(api_name) {} | ||
| 19 | + ~ArangeApiCall() final = default; | ||
| 20 | + | ||
| 21 | + Status Generate(const TPipe &tpipe, const std::vector<ascir::AxisId> ¤t_axis, | ||
| 22 | + const std::vector<std::reference_wrapper<const Tensor>> &inputs, | ||
| 23 | + const std::vector<std::reference_wrapper<const Tensor>> &outputs, std::string &result) const override; | ||
| 24 | + | ||
| 25 | + protected: | ||
| 26 | + Status ParseAttr(const ascir::NodeView &node) override; | ||
| 27 | + | ||
| 28 | + private: | ||
| 29 | + af::Expression base_; | ||
| 30 | + af::Expression step_; | ||
| 31 | +}; | ||
| 32 | +} // namespace codegen | ||
| 33 | + | ||
| @@ -121,10 +121,8 @@ Status CastV2ApiCall::Generate(const TPipe &tpipe, const std::vector<ascir::Axis | |||
| 121 | if (outer_repeats_size == 0U) { | 121 | if (outer_repeats_size == 0U) { |
| 122 | GELOGD("outer_repeats_size is 0, x_dtype = %s, y_dtype = %s", x_dtype.c_str(), y_dtype.c_str()); | 122 | GELOGD("outer_repeats_size is 0, x_dtype = %s, y_dtype = %s", x_dtype.c_str(), y_dtype.c_str()); |
| 123 | if (x.is_constant) { | 123 | if (x.is_constant) { |
| 124 | - ss << this->api_name_ << "(" << y << "[" << tpipe.tiler.TensorVectorizedOffset(current_axis, y) << "], " | 124 | + ss << "Duplicate(" << y << "[" << tpipe.tiler.TensorVectorizedOffset(current_axis, y) << "], static_cast<" |
| 125 | - << scalar_local_blk_tensor_name << "[0], " | 125 | + << y_dtype << ">(" << x.GetScalarValue() << "), " << y.actual_size << ");" << std::endl; |
| 126 | - << "{" << "ConvertToUint32(" << y.actual_size << ")" << "}, " | ||
| 127 | - << "{ConvertToUint32(1)}, {ConvertToUint32(1)});" << std::endl; | ||
| 128 | } else { | 126 | } else { |
| 129 | ss << this->api_name_ << "(" << y << "[" << tpipe.tiler.TensorVectorizedOffset(current_axis, y) << "], " << x | 127 | ss << this->api_name_ << "(" << y << "[" << tpipe.tiler.TensorVectorizedOffset(current_axis, y) << "], " << x |
| 130 | << "[" << tpipe.tiler.TensorVectorizedOffset(current_axis, x) << "], " | 128 | << "[" << tpipe.tiler.TensorVectorizedOffset(current_axis, x) << "], " |
| @@ -359,7 +359,7 @@ Status BuildDataCopyApiParamInNormal(const TPipe &tpipe, CodegenApiParam &api_pa | |||
| 359 | DmaSpecificParams &dma_specific_params, const Tensor &src, const Tensor &dst, | 359 | DmaSpecificParams &dma_specific_params, const Tensor &src, const Tensor &dst, |
| 360 | std::string &gm_offset, bool copy_in, bool has_transpose) { | 360 | std::string &gm_offset, bool copy_in, bool has_transpose) { |
| 361 | DataCopyParams data_copy_param; | 361 | DataCopyParams data_copy_param; |
| 362 | - GE_ASSERT_TRUE(CalculateDmaParams(tpipe, dst, dst, data_copy_param), "CalculateDmaParams failed"); | 362 | + GE_ASSERT_TRUE(CalculateDmaParams(tpipe, dst, copy_in ? dst : src, data_copy_param), "CalculateDmaParams failed"); |
| 363 | size_t total_len = data_copy_param.repeats.size(); | 363 | size_t total_len = data_copy_param.repeats.size(); |
| 364 | const Tensor &ub_tensor = copy_in ? dst : src; | 364 | const Tensor &ub_tensor = copy_in ? dst : src; |
| 365 | std::string padding_mode = GetPaddingMode(ub_tensor, data_copy_param, has_transpose); | 365 | std::string padding_mode = GetPaddingMode(ub_tensor, data_copy_param, has_transpose); |
| @@ -80,10 +80,6 @@ void CreateSingleStridesParamsInfo(const Tensor &tensor, const std::vector<ascir | |||
| 80 | size_t stride_size = strides.size(); | 80 | size_t stride_size = strides.size(); |
| 81 | size_t start_idx = stride_size <= kVFMaxLoop ? 0 : stride_size - kVFMaxLoop; | 81 | size_t start_idx = stride_size <= kVFMaxLoop ? 0 : stride_size - kVFMaxLoop; |
| 82 | for (; start_idx < stride_size; start_idx++) { | 82 | for (; start_idx < stride_size; start_idx++) { |
| 83 | - // 在生成函数体时,能从图上判断出来0和1的stride,这些轴对应的stride信息可以在代码生成时,直接生成到代码中,不需要额外进行传递 | ||
| 84 | - if (strides[start_idx].Simplify() == af::ops::One || strides[start_idx].Simplify() == af::ops::Zero) { | ||
| 85 | - continue; | ||
| 86 | - } | ||
| 87 | ss << "uint32_t " << tensor << "_stride_" << start_idx << ", "; | 83 | ss << "uint32_t " << tensor << "_stride_" << start_idx << ", "; |
| 88 | } | 84 | } |
| 89 | } | 85 | } |
| @@ -92,15 +88,7 @@ void CreateSingleStridesInfo(const TPipe &tpipe, const std::vector<ascir::SizeEx | |||
| 92 | size_t stride_size = strides.size(); | 88 | size_t stride_size = strides.size(); |
| 93 | size_t start_idx = stride_size <= kVFMaxLoop ? 0 : stride_size - kVFMaxLoop; | 89 | size_t start_idx = stride_size <= kVFMaxLoop ? 0 : stride_size - kVFMaxLoop; |
| 94 | for (; start_idx < stride_size; start_idx++) { | 90 | for (; start_idx < stride_size; start_idx++) { |
| 95 | - // 在生成函数体时,能从图上判断出来0和1的stride,这些轴对应的stride信息可以在代码生成时,直接生成到代码中,不需要额外进行传递 | ||
| 96 | auto current_stride = strides[start_idx].Simplify(); | 91 | auto current_stride = strides[start_idx].Simplify(); |
| 97 | - bool current_stride_is_one = | ||
| 98 | - (af::SymbolicUtils::StaticCheckEq(current_stride, af::sym::kSymbolOne) == af::TriBool::kTrue); | ||
| 99 | - bool current_stride_is_zero = | ||
| 100 | - (af::SymbolicUtils::StaticCheckEq(current_stride, af::sym::kSymbolZero) == af::TriBool::kTrue); | ||
| 101 | - if (current_stride_is_one || current_stride_is_zero) { | ||
| 102 | - continue; | ||
| 103 | - } | ||
| 104 | ss << tpipe.tiler.Size(strides[start_idx]) << ", "; | 92 | ss << tpipe.tiler.Size(strides[start_idx]) << ", "; |
| 105 | } | 93 | } |
| 106 | } | 94 | } |
| @@ -192,6 +180,9 @@ void CreateVFCall(const TPipe &tpipe, const std::string &vf_call_name, const std | |||
| 192 | CreateDimAndStrideParmas(tpipe, merge_info, ss); | 180 | CreateDimAndStrideParmas(tpipe, merge_info, ss); |
| 193 | for (const auto ¶m : arange_params) { | 181 | for (const auto ¶m : arange_params) { |
| 194 | ss << ", " << param.offset << ", " << param.base << ", " << param.step; | 182 | ss << ", " << param.offset << ", " << param.base << ", " << param.step; |
| 183 | + for (const auto &stride : param.logical_strides) { | ||
| 184 | + ss << ", " << stride; | ||
| 185 | + } | ||
| 195 | } | 186 | } |
| 196 | ss << ");" << std::endl; | 187 | ss << ");" << std::endl; |
| 197 | } | 188 | } |
| @@ -427,7 +418,6 @@ void GenerateStridesEqualCheck(const std::vector<Tensor> &inputs, const std::vec | |||
| 427 | ss << " bool strides_equal = false;\n"; | 418 | ss << " bool strides_equal = false;\n"; |
| 428 | ss << " uint32_t strides_align = static_cast<uint32_t>(" << all_stride_names[0] << ");\n"; | 419 | ss << " uint32_t strides_align = static_cast<uint32_t>(" << all_stride_names[0] << ");\n"; |
| 429 | if (all_stride_names.size() == 1) { | 420 | if (all_stride_names.size() == 1) { |
| 430 | - ss << " strides_equal = true;\n"; | ||
| 431 | return; | 421 | return; |
| 432 | } | 422 | } |
| 433 | ss << " if ("; | 423 | ss << " if ("; |
| @@ -444,7 +434,8 @@ void GenerateStridesEqualCheck(const std::vector<Tensor> &inputs, const std::vec | |||
| 444 | return; | 434 | return; |
| 445 | } | 435 | } |
| 446 | 436 | ||
| 447 | -void OptimizeMergeParamsAndLoopSize(const std::vector<std::string> &loop_size_vec, std::stringstream &ss) { | 437 | +void OptimizeMergeParamsAndLoopSize(const std::vector<std::string> &loop_size_vec, |
| 438 | + const VectorizedAxisLoopMergeStatus &merge_info, std::stringstream &ss) { | ||
| 448 | if (loop_size_vec.size() < MAX_VF_AXIS_MERGE_SIZE) { | 439 | if (loop_size_vec.size() < MAX_VF_AXIS_MERGE_SIZE) { |
| 449 | return; | 440 | return; |
| 450 | } | 441 | } |
| @@ -452,9 +443,22 @@ void OptimizeMergeParamsAndLoopSize(const std::vector<std::string> &loop_size_ve | |||
| 452 | const auto &loop_size_0 = loop_size_vec[0]; | 443 | const auto &loop_size_0 = loop_size_vec[0]; |
| 453 | const auto &loop_size_1 = loop_size_vec[1]; | 444 | const auto &loop_size_1 = loop_size_vec[1]; |
| 454 | 445 | ||
| 455 | - ss << " if (strides_equal) {\n"; | 446 | + // A single merged output axis is already flattened. VFLoop still emits the |
| 447 | + // logical axis as an outer loop around the vector loop, so it must be | ||
| 448 | + // collapsed even when its only stride is dynamic and cannot establish | ||
| 449 | + // strides_equal. | ||
| 450 | + const bool is_single_merged_axis = merge_info.merge_axis_ids.size() == 1; | ||
| 451 | + if (is_single_merged_axis) { | ||
| 452 | + ss << " {\n"; | ||
| 453 | + } else { | ||
| 454 | + ss << " if (strides_equal) {\n"; | ||
| 455 | + } | ||
| 456 | ss << " " << loop_size_0 << " = 1;\n"; | 456 | ss << " " << loop_size_0 << " = 1;\n"; |
| 457 | - ss << " element_count = static_cast<uint32_t>(strides_align * output_dims_0);\n"; | 457 | + if (is_single_merged_axis) { |
| 458 | + ss << " element_count = static_cast<uint32_t>(output_dims_0);\n"; | ||
| 459 | + } else { | ||
| 460 | + ss << " element_count = static_cast<uint32_t>(strides_align * output_dims_0);\n"; | ||
| 461 | + } | ||
| 458 | ss << " " << loop_size_1 | 462 | ss << " " << loop_size_1 |
| 459 | << " = static_cast<uint16_t>((element_count + ELEMENT_PER_VECTOR_LENGTH - 1) / ELEMENT_PER_VECTOR_LENGTH);\n"; | 463 | << " = static_cast<uint16_t>((element_count + ELEMENT_PER_VECTOR_LENGTH - 1) / ELEMENT_PER_VECTOR_LENGTH);\n"; |
| 460 | ss << " }\n"; | 464 | ss << " }\n"; |
| @@ -469,7 +473,8 @@ void GenerateVectorFuncParams(const std::string &max_dtype_size, int32_t stride_ | |||
| 469 | if (merge_axis_ids.size() == 0) { | 473 | if (merge_axis_ids.size() == 0) { |
| 470 | ss << " uint32_t element_count = 1;\n"; | 474 | ss << " uint32_t element_count = 1;\n"; |
| 471 | } else { | 475 | } else { |
| 472 | - ss << " uint32_t element_count = static_cast<uint32_t>(" << "output_dims_" << stride_depth << ");\n"; | 476 | + const auto output_dim_index = std::min<size_t>(static_cast<size_t>(stride_depth), merge_axis_ids.size() - 1UL); |
| 477 | + ss << " uint32_t element_count = static_cast<uint32_t>(output_dims_" << output_dim_index << ");\n"; | ||
| 473 | } | 478 | } |
| 474 | ss << " uint16_t loop_times = static_cast<uint16_t>((element_count + ELEMENT_PER_VECTOR_LENGTH - 1) / " | 479 | ss << " uint16_t loop_times = static_cast<uint16_t>((element_count + ELEMENT_PER_VECTOR_LENGTH - 1) / " |
| 475 | "ELEMENT_PER_VECTOR_LENGTH);\n"; | 480 | "ELEMENT_PER_VECTOR_LENGTH);\n"; |
| @@ -502,6 +507,9 @@ void GenerateVfCallFuncHeader(const TPipe &tpipe, const std::string &vf_call_nam | |||
| 502 | for (const auto &arange_param : arange_params) { | 507 | for (const auto &arange_param : arange_params) { |
| 503 | ss << ", int64_t arange_offset_" << arange_param.tensor_id << ", int64_t arange_base_" << arange_param.tensor_id | 508 | ss << ", int64_t arange_offset_" << arange_param.tensor_id << ", int64_t arange_base_" << arange_param.tensor_id |
| 504 | << ", int64_t arange_step_" << arange_param.tensor_id; | 509 | << ", int64_t arange_step_" << arange_param.tensor_id; |
| 510 | + for (size_t index = 0; index < arange_param.logical_strides.size(); ++index) { | ||
| 511 | + ss << ", int64_t arange_stride_" << arange_param.tensor_id << "_" << index; | ||
| 512 | + } | ||
| 505 | } | 513 | } |
| 506 | } | 514 | } |
| 507 | ss << ")" << std::endl; | 515 | ss << ")" << std::endl; |
| @@ -539,12 +547,12 @@ void GenerateLocalMemTensorPtrs(const std::vector<Tensor> &outputs, const std::v | |||
| 539 | 547 | ||
| 540 | void GenerateVfCallBodyPreamble(const TPipe &tpipe, const TensorManager &tensor_mgr, const VFLoop &root_loop, | 548 | void GenerateVfCallBodyPreamble(const TPipe &tpipe, const TensorManager &tensor_mgr, const VFLoop &root_loop, |
| 541 | const std::vector<Tensor> &outputs, const std::vector<Tensor> &inputs, | 549 | const std::vector<Tensor> &outputs, const std::vector<Tensor> &inputs, |
| 542 | - const std::string &max_dtype_size, std::stringstream &vf_body) { | 550 | + const std::string &loop_dtype, std::stringstream &vf_body) { |
| 543 | GenerateLocalMemTensorPtrs(outputs, inputs, vf_body); | 551 | GenerateLocalMemTensorPtrs(outputs, inputs, vf_body); |
| 544 | GenerateTensorDefs(tpipe, tensor_mgr, root_loop, vf_body); | 552 | GenerateTensorDefs(tpipe, tensor_mgr, root_loop, vf_body); |
| 545 | - vf_body << "\nAscendC::MicroAPI::MaskReg preg_main = AscendC::MicroAPI::CreateMask<" << max_dtype_size | 553 | + vf_body << "\nAscendC::MicroAPI::MaskReg preg_main = AscendC::MicroAPI::CreateMask<" << loop_dtype |
| 546 | << ", AscendC::MicroAPI::MaskPattern::ALL>();\n"; | 554 | << ", AscendC::MicroAPI::MaskPattern::ALL>();\n"; |
| 547 | - vf_body << "AscendC::MicroAPI::MaskReg preg_vl1 = AscendC::MicroAPI::CreateMask<" << max_dtype_size | 555 | + vf_body << "AscendC::MicroAPI::MaskReg preg_vl1 = AscendC::MicroAPI::CreateMask<" << loop_dtype |
| 548 | << ", AscendC::MicroAPI::MaskPattern::VL1>();\n"; | 556 | << ", AscendC::MicroAPI::MaskPattern::VL1>();\n"; |
| 549 | } | 557 | } |
| 550 | 558 | ||
| @@ -566,6 +574,13 @@ Status GenerateVfCallLoopBody(const TPipe &tpipe, const TensorManager &tensor_mg | |||
| 566 | ArangeOffsetMap arange_offsets; | 574 | ArangeOffsetMap arange_offsets; |
| 567 | std::vector<ArangeParam> arange_params; | 575 | std::vector<ArangeParam> arange_params; |
| 568 | root_loop.CollectArangeParams(tpipe, arange_params); | 576 | root_loop.CollectArangeParams(tpipe, arange_params); |
| 577 | + for (auto ¶m : arange_params) { | ||
| 578 | + const auto *tensor = tensor_mgr.GetTensor(param.tensor_id); | ||
| 579 | + GE_ASSERT_NOTNULL(tensor); | ||
| 580 | + for (const auto &stride : tensor->axis_strides_) { | ||
| 581 | + param.logical_strides.push_back(tpipe.tiler.Size(stride)); | ||
| 582 | + } | ||
| 583 | + } | ||
| 569 | for (const auto ¶m : arange_params) { | 584 | for (const auto ¶m : arange_params) { |
| 570 | arange_offsets.emplace(param.tensor_id, "arange_offset_" + std::to_string(param.tensor_id)); | 585 | arange_offsets.emplace(param.tensor_id, "arange_offset_" + std::to_string(param.tensor_id)); |
| 571 | } | 586 | } |
| @@ -580,7 +595,7 @@ Status GenerateVfCallLoopBody(const TPipe &tpipe, const TensorManager &tensor_mg | |||
| 580 | params << std::endl << loop_size << std::endl; | 595 | params << std::endl << loop_size << std::endl; |
| 581 | if (is_double_loop) { // 假如stride_depth为1即两层循环,那实际上loop里递归了三次,分别是0、1、2,在2里单独处理call | 596 | if (is_double_loop) { // 假如stride_depth为1即两层循环,那实际上loop里递归了三次,分别是0、1、2,在2里单独处理call |
| 582 | GenerateStridesEqualCheck(inputs, outputs, merge_info, all_strides, params); | 597 | GenerateStridesEqualCheck(inputs, outputs, merge_info, all_strides, params); |
| 583 | - OptimizeMergeParamsAndLoopSize(loop_size_vec, params); | 598 | + OptimizeMergeParamsAndLoopSize(loop_size_vec, merge_info, params); |
| 584 | GE_ASSERT_SUCCESS(UpdateVectorFuncNodeParams(node, merge_info, all_strides)); | 599 | GE_ASSERT_SUCCESS(UpdateVectorFuncNodeParams(node, merge_info, all_strides)); |
| 585 | } | 600 | } |
| 586 | vf_body << std::endl << loop_body << std::endl; | 601 | vf_body << std::endl << loop_body << std::endl; |
| @@ -605,6 +620,13 @@ af::Status UpdateVectorFuncNodeParams(const af::AscNodePtr &node, const Vectoriz | |||
| 605 | Status VfCall::GenerateFuncDefinition(const TPipe &tpipe, const Tiler &tiler, std::stringstream &ss) const { | 620 | Status VfCall::GenerateFuncDefinition(const TPipe &tpipe, const Tiler &tiler, std::stringstream &ss) const { |
| 606 | std::vector<ArangeParam> arange_params; | 621 | std::vector<ArangeParam> arange_params; |
| 607 | root_loop_.CollectArangeParams(tpipe, arange_params); | 622 | root_loop_.CollectArangeParams(tpipe, arange_params); |
| 623 | + for (auto ¶m : arange_params) { | ||
| 624 | + const auto *tensor = tensor_mgr_.GetTensor(param.tensor_id); | ||
| 625 | + GE_ASSERT_NOTNULL(tensor); | ||
| 626 | + for (const auto &stride : tensor->axis_strides_) { | ||
| 627 | + param.logical_strides.push_back(tpipe.tiler.Size(stride)); | ||
| 628 | + } | ||
| 629 | + } | ||
| 608 | GE_ASSERT_TRUE(tpipe.cv_fusion_type != ascir::CubeTemplateType::kUBFuse || arange_params.empty(), | 630 | GE_ASSERT_TRUE(tpipe.cv_fusion_type != ascir::CubeTemplateType::kUBFuse || arange_params.empty(), |
| 609 | "Arange is not supported in CV UBFuse VectorFunc."); | 631 | "Arange is not supported in CV UBFuse VectorFunc."); |
| 610 | // 收集输入输出信息,由于GenInnerLoopSizeAndActualSize函数中会刷新tiler对象中的actual_sizes字段, | 632 | // 收集输入输出信息,由于GenInnerLoopSizeAndActualSize函数中会刷新tiler对象中的actual_sizes字段, |
| @@ -623,6 +645,12 @@ Status VfCall::GenerateFuncDefinition(const TPipe &tpipe, const Tiler &tiler, st | |||
| 623 | std::stringstream params; | 645 | std::stringstream params; |
| 624 | std::stringstream vf_body; | 646 | std::stringstream vf_body; |
| 625 | int32_t stride_depth = GeOriginLastAxisPos(tiler, axis_ids_, merge_info.merge_axis_ids); | 647 | int32_t stride_depth = GeOriginLastAxisPos(tiler, axis_ids_, merge_info.merge_axis_ids); |
| 648 | + // Loop stride and masks follow the widest compute dtype in the subgraph: | ||
| 649 | + // narrower outputs (e.g. uint8 masks from int32 compares) still use the | ||
| 650 | + // compute register's lane count, and per-tensor Load/Store keep their own | ||
| 651 | + // dtype. Basing the loop on the output dtype shrank EPL below the compute | ||
| 652 | + // register width and produced per-4th-element corruption (device-verified). | ||
| 653 | + std::string loop_dtype = this->max_dtype_size_; | ||
| 626 | 654 | ||
| 627 | // constexpr static uint32_t VECTOR_LENGTH = AscendC::GetVecLen(); | 655 | // constexpr static uint32_t VECTOR_LENGTH = AscendC::GetVecLen(); |
| 628 | // constexpr static uint32_t SIZE_OF_DTYPE = sizeof(half); | 656 | // constexpr static uint32_t SIZE_OF_DTYPE = sizeof(half); |
| @@ -630,9 +658,9 @@ Status VfCall::GenerateFuncDefinition(const TPipe &tpipe, const Tiler &tiler, st | |||
| 630 | // uint32_t element_count = static_cast<uint32_t>(output_dims_0); | 658 | // uint32_t element_count = static_cast<uint32_t>(output_dims_0); |
| 631 | // uint16_t loop_times = static_cast<uint16_t>((element_count + ELEMENT_PER_VECTOR_LENGTH - 1) / | 659 | // uint16_t loop_times = static_cast<uint16_t>((element_count + ELEMENT_PER_VECTOR_LENGTH - 1) / |
| 632 | // ELEMENT_PER_VECTOR_LENGTH); | 660 | // ELEMENT_PER_VECTOR_LENGTH); |
| 633 | - GenerateVfCallLoopParams(tpipe, max_dtype_size_, stride_depth, merge_info, params); | 661 | + GenerateVfCallLoopParams(tpipe, loop_dtype, stride_depth, merge_info, params); |
| 634 | - GenerateVfCallBodyPreamble(tpipe, tensor_mgr_, root_loop_, this->ub_outputs_, this->ub_inputs_, max_dtype_size_, | 662 | + root_loop_.SetLoopDtype(loop_dtype); |
| 635 | - vf_body); | 663 | + GenerateVfCallBodyPreamble(tpipe, tensor_mgr_, root_loop_, this->ub_outputs_, this->ub_inputs_, loop_dtype, vf_body); |
| 636 | GE_ASSERT_SUCCESS(GenerateVfCallLoopBody(tpipe, tensor_mgr_, root_loop_, stride_depth, merge_info, this->ub_inputs_, | 664 | GE_ASSERT_SUCCESS(GenerateVfCallLoopBody(tpipe, tensor_mgr_, root_loop_, stride_depth, merge_info, this->ub_inputs_, |
| 637 | this->ub_outputs_, node, params, vf_body)); | 665 | this->ub_outputs_, node, params, vf_body)); |
| 638 | GetVFCallFuncBody(params.str(), vf_body.str(), ss); | 666 | GetVFCallFuncBody(params.str(), vf_body.str(), ss); |
| @@ -654,6 +682,13 @@ Status VfCall::Generate(const TPipe &tpipe, [[maybe_unused]] const std::vector<a | |||
| 654 | std::stringstream ss; | 682 | std::stringstream ss; |
| 655 | std::vector<ArangeParam> arange_params; | 683 | std::vector<ArangeParam> arange_params; |
| 656 | root_loop_.CollectArangeParams(tpipe, arange_params); | 684 | root_loop_.CollectArangeParams(tpipe, arange_params); |
| 685 | + for (auto ¶m : arange_params) { | ||
| 686 | + const auto *tensor = tensor_mgr_.GetTensor(param.tensor_id); | ||
| 687 | + GE_ASSERT_NOTNULL(tensor); | ||
| 688 | + for (const auto &stride : tensor->axis_strides_) { | ||
| 689 | + param.logical_strides.push_back(tpipe.tiler.Size(stride)); | ||
| 690 | + } | ||
| 691 | + } | ||
| 657 | size_t loop_num = merge_info.merge_repeats_str.size(); | 692 | size_t loop_num = merge_info.merge_repeats_str.size(); |
| 658 | const size_t outer_loop_num = loop_num > kVFMaxLoop ? loop_num - kVFMaxLoop : 0UL; | 693 | const size_t outer_loop_num = loop_num > kVFMaxLoop ? loop_num - kVFMaxLoop : 0UL; |
| 659 | for (auto ¶m : arange_params) { | 694 | for (auto ¶m : arange_params) { |
| @@ -114,7 +114,7 @@ Status GetArangeBlockOffset(const TPipe &tpipe, const MicroApiTensor *reg_tensor | |||
| 114 | GE_ASSERT_TRUE(!logical_axis_indices.empty(), "Arange vectorized axis has no logical source."); | 114 | GE_ASSERT_TRUE(!logical_axis_indices.empty(), "Arange vectorized axis has no logical source."); |
| 115 | const auto axis_index = static_cast<size_t>(logical_axis_indices.back()); | 115 | const auto axis_index = static_cast<size_t>(logical_axis_indices.back()); |
| 116 | GE_ASSERT_TRUE(axis_index < reg_tensor->axis_strides_.size(), "Arange logical axis stride is missing."); | 116 | GE_ASSERT_TRUE(axis_index < reg_tensor->axis_strides_.size(), "Arange logical axis stride is missing."); |
| 117 | - offset << tpipe.tiler.Size(reg_tensor->axis_strides_[axis_index]); | 117 | + offset << "arange_stride_" << reg_tensor->id_ << "_" << axis_index; |
| 118 | has_offset = true; | 118 | has_offset = true; |
| 119 | } | 119 | } |
| 120 | result = has_offset ? offset.str() : "0"; | 120 | result = has_offset ? offset.str() : "0"; |
| @@ -345,6 +345,16 @@ Status VFLoop::ConstructFromNodes(ascir::NodeViewVisitorConst nodes, const ascir | |||
| 345 | 345 | ||
| 346 | void VFLoop::SetMaxDtypeSize(std::string dtype) { | 346 | void VFLoop::SetMaxDtypeSize(std::string dtype) { |
| 347 | this->max_dtype_size_ = dtype; | 347 | this->max_dtype_size_ = dtype; |
| 348 | + this->SetLoopDtype(dtype); | ||
| 349 | +} | ||
| 350 | + | ||
| 351 | +void VFLoop::SetLoopDtype(std::string dtype) const { | ||
| 352 | + this->loop_dtype_ = dtype; | ||
| 353 | + for (const auto &body : this->bodys_) { | ||
| 354 | + if (body.type_ == LoopType::LOOP) { | ||
| 355 | + body.loop_->SetLoopDtype(dtype); | ||
| 356 | + } | ||
| 357 | + } | ||
| 348 | } | 358 | } |
| 349 | 359 | ||
| 350 | void VFLoop::Destruct() { | 360 | void VFLoop::Destruct() { |
| @@ -422,7 +432,7 @@ Status VFLoop::GenerateLoop(const TPipe &tpipe, const TensorManager &tensor_mgr, | |||
| 422 | << axis << "++) " | 432 | << axis << "++) " |
| 423 | << "{" << std::endl; | 433 | << "{" << std::endl; |
| 424 | if (current_depth == depth) { | 434 | if (current_depth == depth) { |
| 425 | - ss << " preg_" << current_depth << " = " << "AscendC::MicroAPI::UpdateMask<" << this->max_dtype_size_ << ">(" | 435 | + ss << " preg_" << current_depth << " = " << "AscendC::MicroAPI::UpdateMask<" << this->loop_dtype_ << ">(" |
| 426 | << "sreg_" << current_depth << ");\n"; | 436 | << "sreg_" << current_depth << ");\n"; |
| 427 | } | 437 | } |
| 428 | GE_CHK_STATUS_RET(this->GenerateBody(tpipe, tensor_mgr, depth, current_axis, ss, loop_size_ss, only_loop_max_depth, | 438 | GE_CHK_STATUS_RET(this->GenerateBody(tpipe, tensor_mgr, depth, current_axis, ss, loop_size_ss, only_loop_max_depth, |
| @@ -498,7 +508,7 @@ void VFLoop::CollectArangeParams(const TPipe &tpipe, std::vector<ArangeParam> &p | |||
| 498 | std::string base; | 508 | std::string base; |
| 499 | std::string step; | 509 | std::string step; |
| 500 | body.call_->GetArangeParams(tpipe, base, step); | 510 | body.call_->GetArangeParams(tpipe, base, step); |
| 501 | - params.push_back({body.call_->GetOutputTensorIdByIndex(0), std::move(base), std::move(step), "0"}); | 511 | + params.push_back({body.call_->GetOutputTensorIdByIndex(0), std::move(base), std::move(step), "0", {}}); |
| 502 | } else if (body.type_ == LoopType::LOOP) { | 512 | } else if (body.type_ == LoopType::LOOP) { |
| 503 | body.loop_->CollectArangeParams(tpipe, params); | 513 | body.loop_->CollectArangeParams(tpipe, params); |
| 504 | } | 514 | } |
| @@ -21,6 +21,7 @@ struct ArangeParam { | |||
| 21 | std::string base; | 21 | std::string base; |
| 22 | std::string step; | 22 | std::string step; |
| 23 | std::string offset; | 23 | std::string offset; |
| 24 | + std::vector<std::string> logical_strides; | ||
| 24 | }; | 25 | }; |
| 25 | 26 | ||
| 26 | using ArangeOffsetMap = std::map<ascir::TensorId, std::string>; | 27 | using ArangeOffsetMap = std::map<ascir::TensorId, std::string>; |
| @@ -59,6 +60,7 @@ class VFLoop { | |||
| 59 | Status GenerateCvUbFuse(const TPipe &tpipe, const TensorManager &tensor_mgr, std::string &result, | 60 | Status GenerateCvUbFuse(const TPipe &tpipe, const TensorManager &tensor_mgr, std::string &result, |
| 60 | std::string &loop_size_result) const; | 61 | std::string &loop_size_result) const; |
| 61 | void SetMaxDtypeSize(std::string dtype); | 62 | void SetMaxDtypeSize(std::string dtype); |
| 63 | + void SetLoopDtype(std::string dtype) const; | ||
| 62 | void CollectMaskRegTempTensors(const TPipe &tpipe, const TensorManager &tensor_mgr, | 64 | void CollectMaskRegTempTensors(const TPipe &tpipe, const TensorManager &tensor_mgr, |
| 63 | std::vector<std::string> &temp_tensors) const; | 65 | std::vector<std::string> &temp_tensors) const; |
| 64 | void CollectArangeParams(const TPipe &tpipe, std::vector<ArangeParam> ¶ms) const; | 66 | void CollectArangeParams(const TPipe &tpipe, std::vector<ArangeParam> ¶ms) const; |
| @@ -68,6 +70,7 @@ class VFLoop { | |||
| 68 | VFLoop *parent_; | 70 | VFLoop *parent_; |
| 69 | std::vector<VFLoopBody> bodys_; | 71 | std::vector<VFLoopBody> bodys_; |
| 70 | std::string max_dtype_size_; | 72 | std::string max_dtype_size_; |
| 73 | + mutable std::string loop_dtype_; | ||
| 71 | 74 | ||
| 72 | Status GenerateLoop(const TPipe &tpipe, const TensorManager &tensor_mgr, int32_t depth, | 75 | Status GenerateLoop(const TPipe &tpipe, const TensorManager &tensor_mgr, int32_t depth, |
| 73 | std::vector<ascir::AxisId> ¤t_axis, std::stringstream &ss, std::stringstream &loop_size_ss, | 76 | std::vector<ascir::AxisId> ¤t_axis, std::stringstream &ss, std::stringstream &loop_size_ss, |
| @@ -38,6 +38,175 @@ af::Status ValidateUniqueNodeNames(const af::AscGraph &graph) { | |||
| 38 | return af::SUCCESS; | 38 | return af::SUCCESS; |
| 39 | } | 39 | } |
| 40 | 40 | ||
| 41 | +// Arange 广播外轴检测: 逻辑 strides 中最后一个非零 stride 为取值轴(lane 轴), | ||
| 42 | +// 其之前或之后尺寸大于 1 的轴若存在零 stride, 表示 Arange 的取值沿该轴广播(值重复), | ||
| 43 | +// 轴合并/flatten 后无法用线性等差序列表达。尺寸为 1 的退化轴 stride 为 0 时不影响取值语义。 | ||
| 44 | +// 使用逻辑 strides 而非 vectorized_strides: 调度器轴合并会重写物理视图, | ||
| 45 | +// 但取值语义始终由逻辑 strides 决定。 | ||
| 46 | +// 同时覆盖 Pattern A(前缀广播, lane 轴在最后)和 Pattern B(尾轴广播, lane 轴在前缀)。 | ||
| 47 | +bool ArangeHasBroadcastOuterAxis(const af::AscNodePtr &node) { | ||
| 48 | + for (const auto &output : node->outputs()) { | ||
| 49 | + const auto &attr = output->attr; | ||
| 50 | + const auto &strides = attr.strides; | ||
| 51 | + const auto &repeats = attr.repeats; | ||
| 52 | + if (strides.empty() || strides.size() != repeats.size()) { | ||
| 53 | + continue; | ||
| 54 | + } | ||
| 55 | + // 找到唯一的非零 stride 轴(lane 轴)。 | ||
| 56 | + // Pattern A: lane 轴在最后(如 strides=[0,...,0,1])。 | ||
| 57 | + // Pattern B: lane 轴在前缀(如 strides=[1,0,...,0])。 | ||
| 58 | + for (size_t i = 0; i < strides.size(); ++i) { | ||
| 59 | + if (af::SymbolicUtils::StaticCheckEq(strides[i], af::sym::kSymbolZero) == af::TriBool::kTrue) { | ||
| 60 | + // 该轴 stride 为 0, 检查是否非退化(尺寸>1)。 | ||
| 61 | + // 非退化的零 stride 轴表示值沿该轴重复, 是广播轴。 | ||
| 62 | + if (af::SymbolicUtils::StaticCheckNe(repeats[i], af::sym::kSymbolOne) == af::TriBool::kTrue) { | ||
| 63 | + return true; | ||
| 64 | + } | ||
| 65 | + } | ||
| 66 | + } | ||
| 67 | + } | ||
| 68 | + return false; | ||
| 69 | +} | ||
| 70 | + | ||
| 71 | +// 广播 Arange 保留在根图物化时, 输出按轴尺寸展开为连续物理布局; | ||
| 72 | +// 逻辑取值语义仍由 strides 决定, 由 ArangeApiCall 的逻辑偏移使用。 | ||
| 73 | +bool MaterializeArangeViewContiguous(const af::AscNodePtr &node) { | ||
| 74 | + for (const auto &output : node->outputs()) { | ||
| 75 | + auto &attr = output->attr; | ||
| 76 | + if (attr.vectorized_axis.empty() || attr.vectorized_strides.empty() || | ||
| 77 | + attr.vectorized_axis.size() != attr.vectorized_strides.size()) { | ||
| 78 | + continue; | ||
| 79 | + } | ||
| 80 | + if (attr.axis.size() != attr.repeats.size()) { | ||
| 81 | + return false; | ||
| 82 | + } | ||
| 83 | + for (const auto &axis_id : attr.vectorized_axis) { | ||
| 84 | + if (std::find(attr.axis.begin(), attr.axis.end(), axis_id) == attr.axis.end()) { | ||
| 85 | + return false; | ||
| 86 | + } | ||
| 87 | + } | ||
| 88 | + std::vector<af::Expression> flat_strides(attr.vectorized_strides.size()); | ||
| 89 | + af::Expression stride = af::ops::One; | ||
| 90 | + for (size_t i = attr.vectorized_strides.size(); i > 0UL; --i) { | ||
| 91 | + flat_strides[i - 1UL] = stride; | ||
| 92 | + const auto axis_iter = std::find(attr.axis.begin(), attr.axis.end(), attr.vectorized_axis[i - 1UL]); | ||
| 93 | + const auto &axis_size = attr.repeats[static_cast<size_t>(std::distance(attr.axis.begin(), axis_iter))]; | ||
| 94 | + stride = (i == attr.vectorized_strides.size()) ? axis_size : af::sym::Mul(stride, axis_size); | ||
| 95 | + } | ||
| 96 | + attr.vectorized_strides = std::move(flat_strides); | ||
| 97 | + } | ||
| 98 | + return true; | ||
| 99 | +} | ||
| 100 | + | ||
| 101 | +// 判断 Arange 输出轴中某个位置的轴是否退化(size==1 且 stride==0)。 | ||
| 102 | +bool IsDegenerateAxis(const af::AscTensorAttr &attr, size_t index) { | ||
| 103 | + return af::SymbolicUtils::StaticCheckEq(attr.repeats[index], af::sym::kSymbolOne) == af::TriBool::kTrue && | ||
| 104 | + af::SymbolicUtils::StaticCheckEq(attr.strides[index], af::sym::kSymbolZero) == af::TriBool::kTrue; | ||
| 105 | +} | ||
| 106 | + | ||
| 107 | +// 前端显式退化轴 Arange -> Broadcast: 前端用多轴退化视图表达通用系数索引 | ||
| 108 | +// (如 c0*p0 + c1*p1 拆成两个退化轴 Arange + Broadcast + Add)。 | ||
| 109 | +// Arange 只负责物化一维等差序列: 退化轴(size 1, stride 0)对物理布局和逻辑取值 | ||
| 110 | +// 均无贡献, 扩维职责由 Broadcast 通用路径(BroadcastExtend)承担, 与退化轴 | ||
| 111 | +// Load -> Broadcast 路径一致。 | ||
| 112 | +// 支持两种模式: | ||
| 113 | +// Pattern A: Arange 前缀轴退化(如 size=[1,8], stride=[0,1]), 值沿最后轴递增。 | ||
| 114 | +// Pattern B: Arange 尾轴退化(如 size=[4,1], stride=[1,0]), 值沿前缀轴递增。 | ||
| 115 | +// 两种模式均保持退化视图原样透传; 仅 1D Arange(轴数少于 Broadcast 输出)在合轴前 | ||
| 116 | +// 补齐为同轴数退化视图, 缺失轴按 Broadcast 输出位置补 size 1, stride 0。 | ||
| 117 | +af::Status NormalizeArangeBroadcastViewsImpl(af::AscGraph &vf_graph) { | ||
| 118 | + for (const auto &broadcast : vf_graph.GetAllNodes()) { | ||
| 119 | + if (!af::ops::IsOps<af::ascir_op::Broadcast>(broadcast) || broadcast->inputs().size() != 1UL) { | ||
| 120 | + continue; | ||
| 121 | + } | ||
| 122 | + const auto arange = std::dynamic_pointer_cast<af::AscNode>(broadcast->inputs[0].anchor.GetOwnerNode()); | ||
| 123 | + if (arange == nullptr || !af::ops::IsOps<af::ascir_op::Arange>(arange) || arange->outputs().size() != 1UL || | ||
| 124 | + arange->GetOutDataNodes().size() != 1UL) { | ||
| 125 | + continue; | ||
| 126 | + } | ||
| 127 | + auto &input = arange->outputs[0].attr; | ||
| 128 | + const auto &output = broadcast->outputs[0].attr; | ||
| 129 | + if (input.axis.empty() || input.axis.size() != input.repeats.size() || input.axis.size() != input.strides.size() || | ||
| 130 | + output.axis.empty() || output.axis.size() != output.repeats.size() || | ||
| 131 | + output.axis.size() != output.strides.size() || input.axis.size() > output.axis.size() || | ||
| 132 | + arange->GetInControlNodesSize() != 0UL || arange->GetOutControlNodesSize() != 0UL || | ||
| 133 | + broadcast->GetInControlNodesSize() != 0UL || broadcast->GetOutControlNodesSize() != 0UL) { | ||
| 134 | + return af::FAILED; | ||
| 135 | + } | ||
| 136 | + // Broadcast 输出的尾轴必须与 Arange 的尾轴一致。 | ||
| 137 | + if (output.axis.back() != input.axis.back()) { | ||
| 138 | + return af::FAILED; | ||
| 139 | + } | ||
| 140 | + // Broadcast 输出的尾 stride 必须为 1。 | ||
| 141 | + if (af::SymbolicUtils::StaticCheckEq(output.strides.back(), af::sym::kSymbolOne) != af::TriBool::kTrue) { | ||
| 142 | + return af::FAILED; | ||
| 143 | + } | ||
| 144 | + | ||
| 145 | + // Arange 输入必须恰好有一个非退化轴(其余全部退化)。 | ||
| 146 | + size_t non_degenerate_count = 0; | ||
| 147 | + size_t varying_index = 0; | ||
| 148 | + for (size_t i = 0; i < input.axis.size(); ++i) { | ||
| 149 | + if (!IsDegenerateAxis(input, i)) { | ||
| 150 | + ++non_degenerate_count; | ||
| 151 | + varying_index = i; | ||
| 152 | + } | ||
| 153 | + } | ||
| 154 | + if (non_degenerate_count != 1UL) { | ||
| 155 | + return af::FAILED; | ||
| 156 | + } | ||
| 157 | + // 保存变化轴 id, 避免后续 input.axis 重写后丢失引用。 | ||
| 158 | + const af::AxisId varying_axis_id = input.axis[varying_index]; | ||
| 159 | + // 非退化轴的 stride 必须为 1。 | ||
| 160 | + if (af::SymbolicUtils::StaticCheckEq(input.strides[varying_index], af::sym::kSymbolOne) != af::TriBool::kTrue) { | ||
| 161 | + return af::FAILED; | ||
| 162 | + } | ||
| 163 | + // Arange 的变化轴必须存在于 Broadcast 输出中。 | ||
| 164 | + if (std::find(output.axis.begin(), output.axis.end(), varying_axis_id) == output.axis.end()) { | ||
| 165 | + return af::FAILED; | ||
| 166 | + } | ||
| 167 | + | ||
| 168 | + if (input.axis.size() == output.axis.size()) { | ||
| 169 | + // 已是同轴数退化视图: 原样透传, Broadcast 走通用扩维路径。 | ||
| 170 | + arange->attr.sched.axis = broadcast->attr.sched.axis; | ||
| 171 | + continue; | ||
| 172 | + } | ||
| 173 | + | ||
| 174 | + // 1D Arange: 按 Broadcast 输出位置补齐退化轴(size 1, stride 0), | ||
| 175 | + // 变化轴保持自身 size 和 stride 1。 | ||
| 176 | + std::vector<af::AxisId> padded_axis; | ||
| 177 | + std::vector<af::Expression> padded_repeats; | ||
| 178 | + std::vector<af::Expression> padded_strides; | ||
| 179 | + const auto &varying_size = input.repeats[varying_index]; | ||
| 180 | + for (size_t i = 0; i < output.axis.size(); ++i) { | ||
| 181 | + padded_axis.push_back(output.axis[i]); | ||
| 182 | + if (output.axis[i] == varying_axis_id) { | ||
| 183 | + padded_repeats.push_back(varying_size); | ||
| 184 | + padded_strides.push_back(af::sym::kSymbolOne); | ||
| 185 | + } else { | ||
| 186 | + padded_repeats.push_back(af::sym::kSymbolOne); | ||
| 187 | + padded_strides.push_back(af::sym::kSymbolZero); | ||
| 188 | + } | ||
| 189 | + } | ||
| 190 | + input.axis = std::move(padded_axis); | ||
| 191 | + input.repeats = std::move(padded_repeats); | ||
| 192 | + input.strides = std::move(padded_strides); | ||
| 193 | + // vectorized 布局按补齐后 repeats 重建为扁平连续布局。 | ||
| 194 | + input.vectorized_axis = output.vectorized_axis; | ||
| 195 | + input.vectorized_strides.assign(output.vectorized_axis.size(), af::sym::kSymbolOne); | ||
| 196 | + af::Expression flat_stride = af::sym::kSymbolOne; | ||
| 197 | + for (size_t i = output.vectorized_axis.size(); i > 0UL; --i) { | ||
| 198 | + input.vectorized_strides[i - 1UL] = flat_stride; | ||
| 199 | + const auto axis_iter = std::find(input.axis.begin(), input.axis.end(), output.vectorized_axis[i - 1UL]); | ||
| 200 | + GE_ASSERT_TRUE(axis_iter != input.axis.end(), "Arange vectorized axis [%ld] not found in padded view", | ||
| 201 | + static_cast<int64_t>(output.vectorized_axis[i - 1UL])); | ||
| 202 | + const auto &axis_size = input.repeats[static_cast<size_t>(std::distance(input.axis.begin(), axis_iter))]; | ||
| 203 | + flat_stride = af::sym::Mul(flat_stride, axis_size); | ||
| 204 | + } | ||
| 205 | + arange->attr.sched.axis = broadcast->attr.sched.axis; | ||
| 206 | + } | ||
| 207 | + return af::SUCCESS; | ||
| 208 | +} | ||
| 209 | + | ||
| 41 | using NodeIndegrees = std::unordered_map<af::Node *, size_t>; | 210 | using NodeIndegrees = std::unordered_map<af::Node *, size_t>; |
| 42 | using NodeSuccessors = std::unordered_map<af::Node *, std::vector<af::NodePtr>>; | 211 | using NodeSuccessors = std::unordered_map<af::Node *, std::vector<af::NodePtr>>; |
| 43 | 212 | ||
| @@ -459,6 +628,10 @@ namespace optimize { | |||
| 459 | const std::string kNamePrefixLoad = "Load_"; | 628 | const std::string kNamePrefixLoad = "Load_"; |
| 460 | const std::string kNamePrefixStore = "Store_"; | 629 | const std::string kNamePrefixStore = "Store_"; |
| 461 | const std::string kNamePrefixData = "Data_"; | 630 | const std::string kNamePrefixData = "Data_"; |
| 631 | + | ||
| 632 | +af::Status NormalizeArangeBroadcastViews(af::AscGraph &graph) { | ||
| 633 | + return NormalizeArangeBroadcastViewsImpl(graph); | ||
| 634 | +} | ||
| 462 | const std::string kNamePrefixScalar = "Scalar_"; | 635 | const std::string kNamePrefixScalar = "Scalar_"; |
| 463 | const std::string kNamePrefixOutput = "Output_"; | 636 | const std::string kNamePrefixOutput = "Output_"; |
| 464 | 637 | ||
| @@ -540,6 +713,29 @@ void VectorFuncPartitioner::RefineEnableVFFlag(const af::AscNodePtr &node, bool | |||
| 540 | return; | 713 | return; |
| 541 | } | 714 | } |
| 542 | 715 | ||
| 716 | + // Arange: 带控制边时 VF 子图无法保留控制依赖; UBFuse 上下文的 VectorFunc codegen 不支持 | ||
| 717 | + // Arange。两类 Arange 均在建簇阶段关闭 VF, 保留在根图走普通 ArangeApiCall。 | ||
| 718 | + if (af::ops::IsOps<af::ascir_op::Arange>(node)) { | ||
| 719 | + // 广播外轴(stride 为 0)的 Arange 融合进 VF 后, 轴合并/flatten 会把广播轴折叠为 | ||
| 720 | + // 线性索引, Reg::Arange 无法表达逐轴重复取值。此类 Arange 保留在根图, 由 | ||
| 721 | + // ArangeApiCall 按逻辑 stride 物化, 物化布局按轴尺寸展开为连续 stride, | ||
| 722 | + // 并在调用内部对非固定广播轴展开逐帧写入以覆盖整个分配范围。 | ||
| 723 | + const bool has_broadcast_outer_axis = ArangeHasBroadcastOuterAxis(node); | ||
| 724 | + if (has_broadcast_outer_axis) { | ||
| 725 | + if (!MaterializeArangeViewContiguous(node)) { | ||
| 726 | + enable_vf = false; | ||
| 727 | + return; | ||
| 728 | + } | ||
| 729 | + } | ||
| 730 | + if (has_broadcast_outer_axis || disable_arange_vf_ || node->GetInControlNodesSize() != 0UL || | ||
| 731 | + node->GetOutControlNodesSize() != 0UL) { | ||
| 732 | + GELOGD("Node [%s] is Arange with broadcast outer axis or control edge or in UBFuse context, disable VF support.", | ||
| 733 | + node->GetNamePtr()); | ||
| 734 | + enable_vf = false; | ||
| 735 | + return; | ||
| 736 | + } | ||
| 737 | + } | ||
| 738 | + | ||
| 543 | // 1. 如果当前图中有reduce节点,cast不参与vf融合 | 739 | // 1. 如果当前图中有reduce节点,cast不参与vf融合 |
| 544 | if (af::ops::IsOps<af::ascir_op::Cast>(node)) { | 740 | if (af::ops::IsOps<af::ascir_op::Cast>(node)) { |
| 545 | if (graph_has_reduce_node_) { | 741 | if (graph_has_reduce_node_) { |
| @@ -19,9 +19,19 @@ | |||
| 19 | 19 | ||
| 20 | 20 | ||
| 21 | namespace optimize { | 21 | namespace optimize { |
| 22 | +// 内部 graph attr: 标记当前 impl graph 属于 kUBFuse 调度上下文。 | ||
| 23 | +// 由 BufQueAllocator::ProcessSingleImplGraph 写入, PlatformV2::PartitionSubFunctions 读取, | ||
| 24 | +// 用于在分区阶段关闭 Arange 的 VF 融合, 避免 UBFuse VectorFunc codegen 路径收到 Arange。 | ||
| 25 | +constexpr char kGraphAttrIsUBFuseContext[] = "af.internal.vf.is_ubfuse_context"; | ||
| 26 | + | ||
| 27 | +// 前端显式 Arange -> Broadcast 视图规范化: 退化轴视图原样透传, 仅 1D Arange | ||
| 28 | +// 在合轴前补齐为 Broadcast 输出的同轴数退化视图, 扩维由 Broadcast 通用路径承担。 | ||
| 29 | +af::Status NormalizeArangeBroadcastViews(af::AscGraph &graph); | ||
| 30 | + | ||
| 22 | class VectorFuncPartitioner { | 31 | class VectorFuncPartitioner { |
| 23 | public: | 32 | public: |
| 24 | - explicit VectorFuncPartitioner(af::AscGraph &impl_graph) : impl_graph_(impl_graph) {}; | 33 | + explicit VectorFuncPartitioner(af::AscGraph &impl_graph, bool disable_arange_vf = false) |
| 34 | + : impl_graph_(impl_graph), disable_arange_vf_(disable_arange_vf) {}; | ||
| 25 | // VF input graph node names must be unique to keep node and boundary-anchor ordering deterministic. | 35 | // VF input graph node names must be unique to keep node and boundary-anchor ordering deterministic. |
| 26 | af::Status Partition(); | 36 | af::Status Partition(); |
| 27 | 37 | ||
| @@ -74,6 +84,8 @@ class VectorFuncPartitioner { | |||
| 74 | ClusterDict cluster_dict_; | 84 | ClusterDict cluster_dict_; |
| 75 | size_t subgraph_id_ = 0UL; | 85 | size_t subgraph_id_ = 0UL; |
| 76 | bool graph_has_reduce_node_ = false; // 缓存图是否有reduce节点 | 86 | bool graph_has_reduce_node_ = false; // 缓存图是否有reduce节点 |
| 87 | + // UBFuse 等上下文中 Arange 不参与 VF 融合, 保留在根图走普通 ArangeApiCall | ||
| 88 | + bool disable_arange_vf_ = false; | ||
| 77 | }; | 89 | }; |
| 78 | } // namespace optimize | 90 | } // namespace optimize |
| 79 | 91 | ||
| @@ -10,6 +10,8 @@ | |||
| 10 | 10 | ||
| 11 | 11 | ||
| 12 | 12 | ||
| 13 | + | ||
| 14 | + | ||
| 13 | 15 | ||
| 14 | 16 | ||
| 15 | 17 | ||
| @@ -34,7 +36,12 @@ PlatformV2::PlatformV2(bool is_default_enabled) { | |||
| 34 | } | 36 | } |
| 35 | 37 | ||
| 36 | af::Status PlatformV2::PartitionSubFunctions(af::AscGraph &impl_graph) { | 38 | af::Status PlatformV2::PartitionSubFunctions(af::AscGraph &impl_graph) { |
| 37 | - VectorFuncPartitioner partitioner(impl_graph); | 39 | + // UBFuse 上下文由 BufQueAllocator 通过内部 graph attr 传入; UBFuse 模板的 VectorFunc |
| 40 | + // codegen 不支持 Arange, 分区阶段关闭 Arange 的 VF 融合。 | ||
| 41 | + bool disable_arange_vf = false; | ||
| 42 | + (void)af::AttrUtils::GetBool(af::AscGraphUtils::GetComputeGraph(impl_graph), kGraphAttrIsUBFuseContext, | ||
| 43 | + disable_arange_vf); | ||
| 44 | + VectorFuncPartitioner partitioner(impl_graph, disable_arange_vf); | ||
| 38 | GE_ASSERT_SUCCESS(partitioner.Partition(), "Failed to partition sub funcs for graph [%s].", | 45 | GE_ASSERT_SUCCESS(partitioner.Partition(), "Failed to partition sub funcs for graph [%s].", |
| 39 | impl_graph.GetName().c_str()); | 46 | impl_graph.GetName().c_str()); |
| 40 | return af::SUCCESS; | 47 | return af::SUCCESS; |
| @@ -19,6 +19,22 @@ | |||
| 19 | 19 | ||
| 20 | namespace optimize { | 20 | namespace optimize { |
| 21 | namespace { | 21 | namespace { |
| 22 | +bool ArangeStoreNeedsDiscontinuousAlignment(const af::AscTensorAttr &attr) { | ||
| 23 | + for (auto id = attr.vectorized_axis.rbegin(); id != attr.vectorized_axis.rend(); ++id) { | ||
| 24 | + const auto iter = std::find(attr.axis.begin(), attr.axis.end(), *id); | ||
| 25 | + if (iter == attr.axis.end()) { | ||
| 26 | + return false; | ||
| 27 | + } | ||
| 28 | + const size_t index = static_cast<size_t>(std::distance(attr.axis.begin(), iter)); | ||
| 29 | + if (af::SymbolicUtils::StaticCheckEq(attr.repeats[index], af::sym::kSymbolOne) == af::TriBool::kTrue || | ||
| 30 | + af::SymbolicUtils::StaticCheckEq(attr.strides[index], af::sym::kSymbolZero) == af::TriBool::kTrue) { | ||
| 31 | + continue; | ||
| 32 | + } | ||
| 33 | + return af::SymbolicUtils::StaticCheckNe(attr.strides[index], af::sym::kSymbolOne) == af::TriBool::kTrue; | ||
| 34 | + } | ||
| 35 | + return false; | ||
| 36 | +} | ||
| 37 | + | ||
| 22 | af::Status NormalizeStoreSingletonStrides(const af::AscNodePtr &node) { | 38 | af::Status NormalizeStoreSingletonStrides(const af::AscNodePtr &node) { |
| 23 | GE_ASSERT_TRUE(node->outputs().size() == 1UL); | 39 | GE_ASSERT_TRUE(node->outputs().size() == 1UL); |
| 24 | auto &attr = node->outputs[0].attr; | 40 | auto &attr = node->outputs[0].attr; |
| @@ -42,7 +58,6 @@ af::Status NormalizeStoreSingletonStrides(const af::AscNodePtr &node) { | |||
| 42 | return af::SUCCESS; | 58 | return af::SUCCESS; |
| 43 | } | 59 | } |
| 44 | } // namespace | 60 | } // namespace |
| 45 | - | ||
| 46 | AlignmentType UnAlignmentStrategy::GetDefaultAlignmentType() { | 61 | AlignmentType UnAlignmentStrategy::GetDefaultAlignmentType() { |
| 47 | return AlignmentType::kNotAligned; | 62 | return AlignmentType::kNotAligned; |
| 48 | } | 63 | } |
| @@ -97,12 +112,16 @@ af::Status UnAlignmentStrategy::StoreAlignmentInferFunc(const af::AscNodePtr &no | |||
| 97 | ++tile_inner_axis_size; | 112 | ++tile_inner_axis_size; |
| 98 | } | 113 | } |
| 99 | } | 114 | } |
| 115 | + const auto input_node = std::dynamic_pointer_cast<af::AscNode>(node->inputs[0].anchor.GetOwnerNode()); | ||
| 116 | + const bool is_arange_input = input_node != nullptr && af::ops::IsOps<af::ascir_op::Arange>(input_node); | ||
| 100 | 117 | ||
| 101 | - if (ScheduleUtils::IsNeedDiscontinuousAligned(output_attr)) { | 118 | + if (ScheduleUtils::IsNeedDiscontinuousAligned(output_attr) || |
| 119 | + (is_arange_input && ArangeStoreNeedsDiscontinuousAlignment(output_attr))) { | ||
| 102 | GELOGD("Node[%s] is last axis discontinuous writing, input tensor needs to be aligned.", node->GetNamePtr()); | 120 | GELOGD("Node[%s] is last axis discontinuous writing, input tensor needs to be aligned.", node->GetNamePtr()); |
| 103 | tensor_to_align_type_[&output_attr] = {AlignmentType::kDiscontinuous}; | 121 | tensor_to_align_type_[&output_attr] = {AlignmentType::kDiscontinuous}; |
| 104 | GE_ASSERT_SUCCESS(BackPropagateAlignment(node, AlignmentType::kDiscontinuous)); | 122 | GE_ASSERT_SUCCESS(BackPropagateAlignment(node, AlignmentType::kDiscontinuous)); |
| 105 | - } else if (!ScheduleUtils::IsVectorizedAxisContinuousInGM(output_attr) && (tile_inner_axis_size > 1UL)) { | 123 | + } else if (!ScheduleUtils::IsVectorizedAxisContinuousInGM(output_attr) && |
| 124 | + (tile_inner_axis_size > 1UL || is_arange_input)) { | ||
| 106 | GELOGD("Node[%s] is discontinuous writing, input tensor needs to be aligned.", node->GetNamePtr()); | 125 | GELOGD("Node[%s] is discontinuous writing, input tensor needs to be aligned.", node->GetNamePtr()); |
| 107 | tensor_to_align_type_[&output_attr] = {AlignmentType::kAligned}; | 126 | tensor_to_align_type_[&output_attr] = {AlignmentType::kAligned}; |
| 108 | GE_ASSERT_SUCCESS(BackPropagateAlignment(node, AlignmentType::kAligned)); | 127 | GE_ASSERT_SUCCESS(BackPropagateAlignment(node, AlignmentType::kAligned)); |