Merged
feat: 恢复 CV Tensor API lowering 与尾部升精度适配 #5760
ling-DTAdd on 5 days ago
feat: 恢复 CV Tensor API lowering 与尾部升精度适配 #5760
Merged
Total: 8 files changed+715-22
| @@ -837,10 +837,7 @@ ge::graphStatus AutofuseNodeWithMatmulTiling(const ge::Operator &op, const fe::P | |||
| 837 | OpRunInfoV2 &run_info, ge::ConstNodePtr node) { | 837 | OpRunInfoV2 &run_info, ge::ConstNodePtr node) { |
| 838 | ge::Operator matmul_op = ge::OpDescUtils::CreateOperatorFromNode(node); | 838 | ge::Operator matmul_op = ge::OpDescUtils::CreateOperatorFromNode(node); |
| 839 | auto op_desc = ge::OpDescUtils::GetOpDescFromOperator(matmul_op); | 839 | auto op_desc = ge::OpDescUtils::GetOpDescFromOperator(matmul_op); |
| 840 | - // CV自动融合场景强制设置offset_x为0x80，触发isForceBasicApi | ||
| 841 | auto node_op_desc = node->GetOpDesc(); | 840 | auto node_op_desc = node->GetOpDesc(); |
 | |||
| 842 | - (void)ge::AttrUtils::SetInt(node_op_desc, "offset_x", 0x80); | ||
| 843 | - (void)ge::AttrUtils::SetInt(op_desc, "offset_x", 0x80); | ||
| 844 | const auto &space_registry = gert::DefaultOpImplSpaceRegistryV2::GetInstance().GetSpaceRegistry( | 841 | const auto &space_registry = gert::DefaultOpImplSpaceRegistryV2::GetInstance().GetSpaceRegistry( |
| 845 | static_cast<gert::OppImplVersionTag>(op_desc->GetOppImplVersion())); | 842 | static_cast<gert::OppImplVersionTag>(op_desc->GetOppImplVersion())); |
| 846 | GE_ASSERT_NOTNULL(space_registry); | 843 | GE_ASSERT_NOTNULL(space_registry); |
| @@ -489,6 +489,57 @@ bool IsStaticShape(std::vector<Expression> output_dims, const ge::OutDataAnchorP | |||
| 489 | return true; | 489 | return true; |
| 490 | } | 490 | } |
| 491 | 491 | ||
| 492 | +bool IsValidMatMulShape(const std::vector<std::vector<Expression>> &input_dims, | ||
| 493 | + const std::vector<Expression> &output_dims, const MatMulAttr &matmul_attr) { | ||
| 494 | + const auto rank = output_dims.size(); | ||
| 495 | + if (input_dims.size() < 2U || rank < 2U || input_dims[0].size() != rank || input_dims[1].size() != rank || | ||
| 496 | + (matmul_attr.is_batch ? rank < 3U : rank != 2U)) { | ||
| 497 | + return false; | ||
| 498 | + } | ||
| 499 | + // Dynamic BMM uses the existing contiguous, equal-batch Cube path. A matching | ||
| 500 | + // batch product alone is insufficient: each batch axis must match the output. | ||
| 501 | + for (size_t i = 0U; i < rank - 2U; ++i) { | ||
| 502 | + if (ge::SymbolicUtils::StaticCheckEq(input_dims[0][i], input_dims[1][i]) != ge::TriBool::kTrue || | ||
| 503 | + ge::SymbolicUtils::StaticCheckEq(output_dims[i], input_dims[0][i]) != ge::TriBool::kTrue) { | ||
| 504 | + return false; | ||
| 505 | + } | ||
| 506 | + } | ||
| 507 | + | ||
| 508 | + const bool transpose_x1 = matmul_attr.adj_x1 || matmul_attr.transpose_x1; | ||
| 509 | + const bool transpose_x2 = matmul_attr.adj_x2 || matmul_attr.transpose_x2; | ||
| 510 | + const auto &x1 = input_dims[0]; | ||
| 511 | + const auto &x2 = input_dims[1]; | ||
| 512 | + const auto &m = transpose_x1 ? x1[rank - 1U] : x1[rank - 2U]; | ||
| 513 | + const auto &k1 = transpose_x1 ? x1[rank - 2U] : x1[rank - 1U]; | ||
| 514 | + const auto &k2 = transpose_x2 ? x2[rank - 1U] : x2[rank - 2U]; | ||
| 515 | + const auto &n = transpose_x2 ? x2[rank - 2U] : x2[rank - 1U]; | ||
| 516 | + if (ge::SymbolicUtils::StaticCheckEq(k1, k2) != ge::TriBool::kTrue || | ||
| 517 | + ge::SymbolicUtils::StaticCheckEq(output_dims[rank - 2U], m) != ge::TriBool::kTrue || | ||
| 518 | + ge::SymbolicUtils::StaticCheckEq(output_dims[rank - 1U], n) != ge::TriBool::kTrue) { | ||
| 519 | + return false; | ||
| 520 | + } | ||
| 521 | + | ||
| 522 | + for (const auto &dim : x1) { | ||
| 523 | + int64_t value = 0; | ||
| 524 | + if (dim.GetConstValue(value) && value <= 0) { | ||
| 525 | + return false; | ||
| 526 | + } | ||
| 527 | + } | ||
| 528 | + for (const auto &dim : x2) { | ||
| 529 | + int64_t value = 0; | ||
| 530 | + if (dim.GetConstValue(value) && value <= 0) { | ||
| 531 | + return false; | ||
| 532 | + } | ||
| 533 | + } | ||
| 534 | + for (const auto &dim : output_dims) { | ||
| 535 | + int64_t value = 0; | ||
| 536 | + if (dim.GetConstValue(value) && value <= 0) { | ||
| 537 | + return false; | ||
| 538 | + } | ||
| 539 | + } | ||
| 540 | + return true; | ||
| 541 | +} | ||
| 542 | + | ||
| 492 | bool IsSameBatchSize(const std::vector<std::vector<Expression>> &input_dims) { | 543 | bool IsSameBatchSize(const std::vector<std::vector<Expression>> &input_dims) { |
| 493 | if (input_dims.size() < 2U) { | 544 | if (input_dims.size() < 2U) { |
| 494 | return false; | 545 | return false; |
| @@ -566,17 +617,16 @@ KernelBox StoreMatMul(const ge::OutDataAnchorPtr &dst, const std::vector<ge::InD | |||
| 566 | if (!ValidateAndLoadInputs(dst, inputs, input_buffers, input_dims, max_dim)) { | 617 | if (!ValidateAndLoadInputs(dst, inputs, input_buffers, input_dims, max_dim)) { |
| 567 | return StoreExtern(dst); | 618 | return StoreExtern(dst); |
| 568 | } | 619 | } |
| 620 | + if (!IsStaticShape(dims, dst, inputs) && !IsValidMatMulShape(input_dims, dims, matmul_attr)) { | ||
| 621 | + GELOGI("Drop lower result of %s as its dynamic matmul shape is unsupported", BufferName(dst).c_str()); | ||
| 622 | + return StoreExtern(dst); | ||
| 623 | + } | ||
🟡 Medium Priority 变更点（loop_api.cpp:620-623）：原逻辑是「非静态 shape 一律回落」，现改为 触发条件：动态 shape 且 x1/x2 带转置属性（adj/transpose）或非连续布局。失败模式：此前该场景走 StoreExtern 的安全回落；现在会生成 Cube kernel entry，而 PR 描述自述「泛化中的动态转置 kernel entry 异常仍在定位」，说明该组合的 kernel entry 尚未验证通过 —— 表现为后端 kernel entry 生成失败/编译报错，或运行期取到不匹配的 tiling 导致精度/性能异常。静态 shape 仍走原路径，故影响仅限动态用例。 建议：在 IsValidMatMulShape 中拒绝动态转置用例（transpose_x1/transpose_x2/adj_x1/adj_x2 为真且对应维度非常量时返回 false），使其继续走 StoreExtern 回落；或在 GAF 侧确认动态转置 kernel entry 可用前加开关隔离该子路径。   Incorrect? | |||
| 569 | PadDimsToMax(input_dims, max_dim); | 624 | PadDimsToMax(input_dims, max_dim); |
| 570 | if (!IsSameBatchSize(input_dims)) { | 625 | if (!IsSameBatchSize(input_dims)) { |
| 571 | GELOGI("Drop lower result of %s as it has different batch size", BufferName(dst).c_str()); | 626 | GELOGI("Drop lower result of %s as it has different batch size", BufferName(dst).c_str()); |
| 572 | return StoreExtern(dst); | 627 | return StoreExtern(dst); |
| 573 | } | 628 | } |
| 574 | 629 | ||
| 575 | - if (!IsStaticShape(dims, dst, inputs)) { | ||
| 576 | - GELOGI("Drop lower result of %s as it is not static shape", BufferName(dst).c_str()); | ||
| 577 | - return StoreExtern(dst); | ||
| 578 | - } | ||
| 579 | - | ||
| 580 | auto loop_var = LoopVar(std::make_shared<StoreMatMulOp>(dst.get(), input_buffers, matmul_attr, dims, input_dims)); | 630 | auto loop_var = LoopVar(std::make_shared<StoreMatMulOp>(dst.get(), input_buffers, matmul_attr, dims, input_dims)); |
| 581 | return SetLoopKernel(dst, loop_var).Realize(); | 631 | return SetLoopKernel(dst, loop_var).Realize(); |
| 582 | } | 632 | } |
| @@ -425,6 +425,14 @@ graphStatus BuildLoopCtx(const std::vector<LoopOpPtr> &inputs, const std::vector | |||
| 425 | } // anonymous namespace | 425 | } // anonymous namespace |
| 426 | 426 | ||
| 427 | graphStatus StoreMatMulOp::RealizeImpl() { | 427 | graphStatus StoreMatMulOp::RealizeImpl() { |
| 428 | + for (auto &dim : dims_) { | ||
| 429 | + dim = dim.Simplify(); | ||
| 430 | + } | ||
| 431 | + for (auto &input_dims : input_dims_) { | ||
| 432 | + for (auto &dim : input_dims) { | ||
| 433 | + dim = dim.Simplify(); | ||
| 434 | + } | ||
| 435 | + } | ||
| 428 | LoopCtx loop_ctx(dims_); | 436 | LoopCtx loop_ctx(dims_); |
| 429 | Ops()->SetLoopAxis(loop_ctx.loop_axis); | 437 | Ops()->SetLoopAxis(loop_ctx.loop_axis); |
| 430 | GE_WARN_ASSERT_GRAPH_SUCCESS(BuildLoopCtx(inputs_, input_dims_, loop_ctx), "Failed build loop ctx for %s", | 438 | GE_WARN_ASSERT_GRAPH_SUCCESS(BuildLoopCtx(inputs_, input_dims_, loop_ctx), "Failed build loop ctx for %s", |
| @@ -9,6 +9,8 @@ | |||
| 9 | */ | 9 | */ |
| 10 | 10 | ||
| 11 | #include "adaption_improve_precision.h" | 11 | #include "adaption_improve_precision.h" |
| 12 | +#include <unordered_set> | ||
| 13 | +#include <vector> | ||
| 12 | #include "common/checker.h" | 14 | #include "common/checker.h" |
| 13 | #include "graph/utils/node_utils.h" | 15 | #include "graph/utils/node_utils.h" |
| 14 | #include "graph/utils/graph_utils.h" | 16 | #include "graph/utils/graph_utils.h" |
| @@ -19,6 +21,8 @@ | |||
| 19 | 21 | ||
| 20 | namespace ge { | 22 | namespace ge { |
| 21 | namespace { | 23 | namespace { |
| 24 | +using PrecisionNodeSet = std::unordered_set<const Node *>; | ||
| 25 | + | ||
| 22 | static const std::unordered_map<std::string, std::string> kTypeToGroup = {{kCastType, kCastType}, | 26 | static const std::unordered_map<std::string, std::string> kTypeToGroup = {{kCastType, kCastType}, |
| 23 | {kLoadType, kLoadType}, | 27 | {kLoadType, kLoadType}, |
| 24 | {kGatherType, kGatherType}, | 28 | {kGatherType, kGatherType}, |
| @@ -95,7 +99,8 @@ bool ShouldChangeDataType(const NodePtr &node, const std::vector<NodePtr> &peer_ | |||
| 95 | } | 99 | } |
| 96 | 100 | ||
| 97 | Status IsNeedInsertCastBeforeOther(const AscGraph &asc_graph, const NodePtr &other_node, bool &need_insert, | 101 | Status IsNeedInsertCastBeforeOther(const AscGraph &asc_graph, const NodePtr &other_node, bool &need_insert, |
| 98 | - std::vector<int32_t> &input_idxs) { | 102 | + std::vector<int32_t> &input_idxs, |
| 103 | + const PrecisionNodeSet *precision_nodes = nullptr) { | ||
| 99 | // 获取前驱节点 | 104 | // 获取前驱节点 |
| 100 | std::vector<NodePtr> peer_out_nodes; | 105 | std::vector<NodePtr> peer_out_nodes; |
| 101 | GE_ASSERT_SUCCESS(asc_adapt::GetPeerOutNodes(other_node, peer_out_nodes)); | 106 | GE_ASSERT_SUCCESS(asc_adapt::GetPeerOutNodes(other_node, peer_out_nodes)); |
| @@ -104,7 +109,8 @@ Status IsNeedInsertCastBeforeOther(const AscGraph &asc_graph, const NodePtr &oth | |||
| 104 | auto peer_out_node = peer_out_nodes[idx]; | 109 | auto peer_out_node = peer_out_nodes[idx]; |
| 105 | GE_ASSERT_SUCCESS(asc_adapt::GetOutputTensorDesc(peer_out_node, peer_output_tensor_desc)); | 110 | GE_ASSERT_SUCCESS(asc_adapt::GetOutputTensorDesc(peer_out_node, peer_output_tensor_desc)); |
| 106 | if (peer_out_node->GetType() == kCastType || peer_out_node->GetType() == kLoadType || | 111 | if (peer_out_node->GetType() == kCastType || peer_out_node->GetType() == kLoadType || |
| 107 | - peer_out_node->GetType() == kGatherType) { | 112 | + peer_out_node->GetType() == kGatherType || |
| 113 | + (precision_nodes != nullptr && precision_nodes->count(peer_out_node.get()) == 0U)) { | ||
| 108 | if (IsLowPrecisionDataType(peer_output_tensor_desc->GetDataType())) { | 114 | if (IsLowPrecisionDataType(peer_output_tensor_desc->GetDataType())) { |
| 109 | need_insert = true; | 115 | need_insert = true; |
| 110 | input_idxs.push_back(idx); | 116 | input_idxs.push_back(idx); |
| @@ -339,7 +345,7 @@ bool IsFloatToUltraLowNeedInsertCast(const NodePtr &peer_out_node, DataType peer | |||
| 339 | return true; | 345 | return true; |
| 340 | } | 346 | } |
| 341 | 347 | ||
| 342 | -Status CastNodeProc(AscGraph &asc_graph, const NodePtr &node) { | 348 | +Status CastNodeProc(AscGraph &asc_graph, const NodePtr &node, bool preserve_control_edges = false) { |
| 343 | // 获取前驱节点 | 349 | // 获取前驱节点 |
| 344 | NodePtr peer_out_node; | 350 | NodePtr peer_out_node; |
| 345 | GE_ASSERT_SUCCESS(asc_adapt::GetPeerOutNode(node, peer_out_node, 0)); | 351 | GE_ASSERT_SUCCESS(asc_adapt::GetPeerOutNode(node, peer_out_node, 0)); |
| @@ -356,8 +362,13 @@ Status CastNodeProc(AscGraph &asc_graph, const NodePtr &node) { | |||
| 356 | GE_ASSERT_SUCCESS(asc_adapt::GetOutputTensorDesc(node, output_tensor_desc)); | 362 | GE_ASSERT_SUCCESS(asc_adapt::GetOutputTensorDesc(node, output_tensor_desc)); |
| 357 | const auto peer_output_dtype = peer_output_tensor_desc->GetDataType(); | 363 | const auto peer_output_dtype = peer_output_tensor_desc->GetDataType(); |
| 358 | const auto output_dtype = output_tensor_desc->GetDataType(); | 364 | const auto output_dtype = output_tensor_desc->GetDataType(); |
| 365 | + // Keep controlled CV casts as ordering boundaries, including data-to-control edges. | ||
| 366 | + const bool keep_control_cast = | ||
| 367 | + preserve_control_edges && (!node->GetInControlAnchor()->GetPeerAnchorsPtr().empty() || | ||
| 368 | + !node->GetOutControlAnchor()->GetPeerAnchorsPtr().empty() || | ||
| 369 | + !node->GetOutDataAnchor(0)->GetPeerInControlAnchors().empty()); | ||
| 359 | // 判断是否需要删除节点 | 370 | // 判断是否需要删除节点 |
| 360 | - if (ShouldDeleteCastNode(node, peer_in_nodes, peer_out_node, peer_output_dtype, output_dtype)) { | 371 | + if (!keep_control_cast && ShouldDeleteCastNode(node, peer_in_nodes, peer_out_node, peer_output_dtype, output_dtype)) { |
| 361 | GE_ASSERT_SUCCESS(asc_adapt::DelNode(asc_graph, node)); | 372 | GE_ASSERT_SUCCESS(asc_adapt::DelNode(asc_graph, node)); |
| 362 | return SUCCESS; | 373 | return SUCCESS; |
| 363 | } | 374 | } |
| @@ -434,7 +445,8 @@ Status CheckNodeDtype(const NodePtr &node) { | |||
| 434 | return SUCCESS; | 445 | return SUCCESS; |
| 435 | } | 446 | } |
| 436 | 447 | ||
| 437 | -Status IsAllNodesInBlacklist(const AscGraph &asc_graph, bool &is_in_blacklist) { | 448 | +Status IsAllNodesInBlacklist(const AscGraph &asc_graph, bool &is_in_blacklist, |
| 449 | + const PrecisionNodeSet *precision_nodes = nullptr) { | ||
| 438 | auto &blacklist2 = autofuse::AutoFuseConfig::Config().GetFusionStrategySolver().improve_precision_blacklist; | 450 | auto &blacklist2 = autofuse::AutoFuseConfig::Config().GetFusionStrategySolver().improve_precision_blacklist; |
| 439 | bool has_all = (blacklist2.find(kAllNodesType) != blacklist2.end()); | 451 | bool has_all = (blacklist2.find(kAllNodesType) != blacklist2.end()); |
| 440 | // 如果有"all"，则对所有节点检查数据类型 | 452 | // 如果有"all"，则对所有节点检查数据类型 |
| @@ -443,7 +455,8 @@ Status IsAllNodesInBlacklist(const AscGraph &asc_graph, bool &is_in_blacklist) { | |||
| 443 | asc_graph.GetName().c_str()); | 455 | asc_graph.GetName().c_str()); |
| 444 | for (const auto &node : AscGraphUtils::GetComputeGraph(asc_graph)->GetAllNodes()) { | 456 | for (const auto &node : AscGraphUtils::GetComputeGraph(asc_graph)->GetAllNodes()) { |
| 445 | // 跳过输入输出节点 | 457 | // 跳过输入输出节点 |
| 446 | - if (BackendUtils::IsOutputNode(node) || BackendUtils::IsInputNode(node)) { | 458 | + if (BackendUtils::IsOutputNode(node) || BackendUtils::IsInputNode(node) || |
| 459 | + (precision_nodes != nullptr && precision_nodes->count(node.get()) == 0U)) { | ||
| 447 | continue; | 460 | continue; |
| 448 | } | 461 | } |
| 449 | // 检查节点数据类型 | 462 | // 检查节点数据类型 |
| @@ -462,6 +475,13 @@ Status IsAllNodesInBlacklist(const AscGraph &asc_graph, bool &is_in_blacklist) { | |||
| 462 | } | 475 | } |
| 463 | // 没有"all"，走按ascir配置黑名单处理的流程 | 476 | // 没有"all"，走按ascir配置黑名单处理的流程 |
| 464 | for (const auto &node : AscGraphUtils::GetComputeGraph(asc_graph)->GetAllNodes()) { | 477 | for (const auto &node : AscGraphUtils::GetComputeGraph(asc_graph)->GetAllNodes()) { |
| 478 | + if (precision_nodes != nullptr && precision_nodes->count(node.get()) == 0U) { | ||
| 479 | + continue; | ||
| 480 | + } | ||
| 481 | + // ReLU and existing casts alone need no promotion; preserve native MM+ReLU fixpipe eligibility. | ||
| 482 | + if (precision_nodes != nullptr && (node->GetType() == kReluType || node->GetType() == kCastType)) { | ||
| 483 | + continue; | ||
| 484 | + } | ||
| 465 | if (IsInBlackList1(node)) { | 485 | if (IsInBlackList1(node)) { |
| 466 | // 在内部写死升精度黑名单, doing nothing | 486 | // 在内部写死升精度黑名单, doing nothing |
| 467 | } else if (IsInBlackList2(node)) { | 487 | } else if (IsInBlackList2(node)) { |
| @@ -477,11 +497,13 @@ Status IsAllNodesInBlacklist(const AscGraph &asc_graph, bool &is_in_blacklist) { | |||
| 477 | } | 497 | } |
| 478 | 498 | ||
| 479 | Status GetTypeToNodesFromGraph(AscGraph &asc_graph, | 499 | Status GetTypeToNodesFromGraph(AscGraph &asc_graph, |
| 480 | - std::unordered_map<std::string, std::vector<NodePtr>> &type_to_nodes) { | 500 | + std::unordered_map<std::string, std::vector<NodePtr>> &type_to_nodes, |
| 501 | + const PrecisionNodeSet *precision_nodes = nullptr) { | ||
| 481 | for (const auto &node : AscGraphUtils::GetComputeGraph(asc_graph)->GetAllNodes()) { | 502 | for (const auto &node : AscGraphUtils::GetComputeGraph(asc_graph)->GetAllNodes()) { |
| 482 | // 输入和Output不需要升精度处理 | 503 | // 输入和Output不需要升精度处理 |
| 483 | auto node_type = node->GetType(); | 504 | auto node_type = node->GetType(); |
| 484 | - if (BackendUtils::IsOutputNode(node) || BackendUtils::IsInputNode(node)) { | 505 | + if (BackendUtils::IsOutputNode(node) || BackendUtils::IsInputNode(node) || |
| 506 | + (precision_nodes != nullptr && precision_nodes->count(node.get()) == 0U)) { | ||
| 485 | continue; | 507 | continue; |
| 486 | } | 508 | } |
| 487 | // 获取当前节点的输出描述符 | 509 | // 获取当前节点的输出描述符 |
| @@ -498,6 +520,45 @@ Status GetTypeToNodesFromGraph(AscGraph &asc_graph, | |||
| 498 | return SUCCESS; | 520 | return SUCCESS; |
| 499 | } | 521 | } |
| 500 | 522 | ||
| 523 | +// Treat each matrix and all its operand producers as an immutable Load boundary. | ||
| 524 | +// Apply ordinary vector precision rules to the remaining graph, including auxiliary branches. | ||
| 525 | +PrecisionNodeSet GetMatmulVectorNodes(const AscGraph &asc_graph) { | ||
| 526 | + static const std::unordered_set<std::string> kMatmulTypes = { | ||
| 527 | + kMatMul, kMatMulBias, kMatMulOffset, kMatMulOffsetBias, | ||
| 528 | + kBatchMatMul, kBatchMatMulBias, kBatchMatMulOffset, kBatchMatMulOffsetBias}; | ||
| 529 | + std::vector<NodePtr> pending; | ||
| 530 | + for (const auto &node : AscGraphUtils::GetComputeGraph(asc_graph)->GetAllNodes()) { | ||
| 531 | + if (!AutofuseUtils::IsCubeNodeType(node)) { | ||
| 532 | + continue; | ||
| 533 | + } | ||
| 534 | + if (kMatmulTypes.count(node->GetType()) == 0U) { | ||
| 535 | + return {}; // Conv precision handling remains unchanged. | ||
| 536 | + } | ||
| 537 | + pending.push_back(node); | ||
| 538 | + } | ||
| 539 | + if (pending.empty()) { | ||
| 540 | + return {}; | ||
| 541 | + } | ||
| 542 | + PrecisionNodeSet protected_nodes; | ||
| 543 | + while (!pending.empty()) { | ||
| 544 | + const auto node = pending.back(); | ||
| 545 | + pending.pop_back(); | ||
| 546 | + if (!protected_nodes.insert(node.get()).second) { | ||
| 547 | + continue; | ||
| 548 | + } | ||
| 549 | + for (const auto &input : node->GetInDataNodes()) { | ||
| 550 | + pending.push_back(input); | ||
| 551 | + } | ||
| 552 | + } | ||
| 553 | + PrecisionNodeSet vector_nodes; | ||
| 554 | + for (const auto &node : AscGraphUtils::GetComputeGraph(asc_graph)->GetAllNodes()) { | ||
| 555 | + if (protected_nodes.count(node.get()) == 0U) { | ||
| 556 | + vector_nodes.insert(node.get()); | ||
| 557 | + } | ||
| 558 | + } | ||
| 559 | + return vector_nodes; | ||
| 560 | +} | ||
| 561 | + | ||
| 501 | Status ImprovePrecision(AscGraph &asc_graph, [[maybe_unused]] const NodePtr &asc_node) { | 562 | Status ImprovePrecision(AscGraph &asc_graph, [[maybe_unused]] const NodePtr &asc_node) { |
| 502 | // 后端不支持concat后有cast的情况，当前分析不出这是concat特例还是搬运类算子的普遍情况，暂时先跳过concat，如果是普遍情况还需要和后端Scheduler讨论处理方案 | 563 | // 后端不支持concat后有cast的情况，当前分析不出这是concat特例还是搬运类算子的普遍情况，暂时先跳过concat，如果是普遍情况还需要和后端Scheduler讨论处理方案 |
| 503 | const auto asc_node_attr = BackendUtils::GetNodeAutoFuseAttr(asc_node); | 564 | const auto asc_node_attr = BackendUtils::GetNodeAutoFuseAttr(asc_node); |
| @@ -506,22 +567,27 @@ Status ImprovePrecision(AscGraph &asc_graph, [[maybe_unused]] const NodePtr &asc | |||
| 506 | GELOGI("graph %s fuse type is concat, don't improve precision.", asc_graph.GetName().c_str()); | 567 | GELOGI("graph %s fuse type is concat, don't improve precision.", asc_graph.GetName().c_str()); |
| 507 | return SUCCESS; | 568 | return SUCCESS; |
| 508 | } | 569 | } |
| 570 | + PrecisionNodeSet vector_nodes; | ||
| 571 | + const PrecisionNodeSet *precision_nodes = nullptr; | ||
| 509 | if (BackendUtils::IsCubeAscNode(asc_node)) { | 572 | if (BackendUtils::IsCubeAscNode(asc_node)) { |
| 510 | - GELOGI("graph %s fuse type is cube, don't improve precision.", asc_graph.GetName().c_str()); | 573 | + vector_nodes = GetMatmulVectorNodes(asc_graph); |
| 511 | - return SUCCESS; | 574 | + if (vector_nodes.empty()) { |
| 575 | + return SUCCESS; | ||
| 576 | + } | ||
| 577 | + precision_nodes = &vector_nodes; | ||
| 512 | } | 578 | } |
| 513 | 579 | ||
| 514 | bool is_in_blacklist = true; | 580 | bool is_in_blacklist = true; |
| 515 | - GE_ASSERT_SUCCESS(IsAllNodesInBlacklist(asc_graph, is_in_blacklist)); | 581 | + GE_ASSERT_SUCCESS(IsAllNodesInBlacklist(asc_graph, is_in_blacklist, precision_nodes)); |
| 516 | if (is_in_blacklist) { | 582 | if (is_in_blacklist) { |
| 517 | GELOGI("graph %s all nodes are in blacklist, don't improve precision.", asc_graph.GetName().c_str()); | 583 | GELOGI("graph %s all nodes are in blacklist, don't improve precision.", asc_graph.GetName().c_str()); |
| 518 | return SUCCESS; | 584 | return SUCCESS; |
| 519 | } | 585 | } |
| 520 | 586 | ||
| 521 | std::unordered_map<std::string, std::vector<NodePtr>> type_to_nodes; | 587 | std::unordered_map<std::string, std::vector<NodePtr>> type_to_nodes; |
| 522 | - GE_ASSERT_SUCCESS(GetTypeToNodesFromGraph(asc_graph, type_to_nodes)); | 588 | + GE_ASSERT_SUCCESS(GetTypeToNodesFromGraph(asc_graph, type_to_nodes, precision_nodes)); |
| 523 | for (const auto &node : type_to_nodes[kCastType]) { | 589 | for (const auto &node : type_to_nodes[kCastType]) { |
| 524 | - GE_ASSERT_SUCCESS(CastNodeProc(asc_graph, node)); | 590 | + GE_ASSERT_SUCCESS(CastNodeProc(asc_graph, node, precision_nodes != nullptr)); |
| 525 | } | 591 | } |
| 526 | for (const auto &node : type_to_nodes[kLoadType]) { | 592 | for (const auto &node : type_to_nodes[kLoadType]) { |
| 527 | bool is_need_insert_cast = false; | 593 | bool is_need_insert_cast = false; |
| @@ -543,7 +609,7 @@ Status ImprovePrecision(AscGraph &asc_graph, [[maybe_unused]] const NodePtr &asc | |||
| 543 | for (const auto &node : type_to_nodes["Other"]) { | 609 | for (const auto &node : type_to_nodes["Other"]) { |
| 544 | bool is_need_insert_cast = false; | 610 | bool is_need_insert_cast = false; |
| 545 | std::vector<int32_t> input_idxs; | 611 | std::vector<int32_t> input_idxs; |
| 546 | - GE_ASSERT_SUCCESS(IsNeedInsertCastBeforeOther(asc_graph, node, is_need_insert_cast, input_idxs)); | 612 | + GE_ASSERT_SUCCESS(IsNeedInsertCastBeforeOther(asc_graph, node, is_need_insert_cast, input_idxs, precision_nodes)); |
| 547 | bool is_increase_precision = true; | 613 | bool is_increase_precision = true; |
| 548 | GE_ASSERT_SUCCESS(InsertCastBeforeNode(asc_graph, node, is_need_insert_cast, is_increase_precision, input_idxs)); | 614 | GE_ASSERT_SUCCESS(InsertCastBeforeNode(asc_graph, node, is_need_insert_cast, is_increase_precision, input_idxs)); |
| 549 | GE_ASSERT_SUCCESS(asc_adapt::FromDtypeToOtherDtype(node, DT_BF16, DT_FLOAT)); | 615 | GE_ASSERT_SUCCESS(asc_adapt::FromDtypeToOtherDtype(node, DT_BF16, DT_FLOAT)); |
| @@ -17,6 +17,7 @@ | |||
| 17 | #include "graph/operator_factory.h" | 17 | #include "graph/operator_factory.h" |
| 18 | #include "graph/utils/op_desc_utils.h" | 18 | #include "graph/utils/op_desc_utils.h" |
| 19 | #include "ascir_ops.h" | 19 | #include "ascir_ops.h" |
| 20 | +#include "ascendc_ir/ascendc_ir_core/asc_graph_ge_bridge.h" | ||
| 20 | #include "utils/autofuse_utils.h" | 21 | #include "utils/autofuse_utils.h" |
| 21 | #include "post_process/scheduler_adapter/adaption_fallback_load.h" | 22 | #include "post_process/scheduler_adapter/adaption_fallback_load.h" |
| 22 | #include "post_process/scheduler_adapter/adaption_fallback_scalar.h" | 23 | #include "post_process/scheduler_adapter/adaption_fallback_scalar.h" |
| @@ -26,6 +27,7 @@ | |||
| 26 | #include "attribute_group/attr_group_shape_env.h" | 27 | #include "attribute_group/attr_group_shape_env.h" |
| 27 | #include "can_fuse/backend/asc_backend_fusion_decider.h" | 28 | #include "can_fuse/backend/asc_backend_fusion_decider.h" |
| 28 | #include "post_process/scheduler_adapter/adaption_complete_node_attrs.h" | 29 | #include "post_process/scheduler_adapter/adaption_complete_node_attrs.h" |
| 30 | +#include "post_process/scheduler_adapter/adaption_improve_precision.h" | ||
| 29 | 31 | ||
| 30 | namespace ge { | 32 | namespace ge { |
| 31 | using namespace autofuse; | 33 | using namespace autofuse; |
| @@ -442,4 +444,319 @@ TEST_F(AdaptionImprovePrecisionTest, ImprovePrecision_BlackListAll_WithSpecific) | |||
| 442 | // 清理黑名单 | 444 | // 清理黑名单 |
| 443 | AutoFuseConfig::MutableConfig().GetMutableFusionStrategySolver().improve_precision_blacklist.clear(); | 445 | AutoFuseConfig::MutableConfig().GetMutableFusionStrategySolver().improve_precision_blacklist.clear(); |
| 444 | } | 446 | } |
| 447 | + | ||
| 448 | +namespace { | ||
| 449 | +template <class Matmul> | ||
| 450 | +std::shared_ptr<AscGraph> CreateMatmulPrecisionGraph(DataType dtype) { | ||
| 451 | + auto graph = std::make_shared<AscGraph>("matmul_precision"); | ||
| 452 | + const auto axes = CreateAscGraphSymbolsAndAxes(*graph); | ||
| 453 | + const auto a = CreateDataOp("a", *graph, axes); | ||
| 454 | + const auto b = CreateDataOp("b", *graph, axes); | ||
| 455 | + a->ir_attr.SetIndex(0); | ||
| 456 | + b->ir_attr.SetIndex(1); | ||
| 457 | + const auto load_a = CreateLoadOp("load_a", a->y, axes); | ||
| 458 | + const auto load_b = CreateLoadOp("load_b", b->y, axes); | ||
| 459 | + af::ascir_op::Cast input_cast("input_cast"); | ||
| 460 | + input_cast.x = load_a->y; | ||
| 461 | + Matmul mm("mm"); | ||
| 462 | + mm.x1 = input_cast.y; | ||
| 463 | + mm.x2 = load_b->y; | ||
| 464 | + af::ascir_op::Gelu gelu("gelu"); | ||
| 465 | + gelu.x = mm.y; | ||
| 466 | + // The side input is also an MM operand: only its vector edge may be cast. | ||
| 467 | + const auto add = CreateAddOp("add", gelu.y, load_b->y, axes); | ||
| 468 | + const auto store = CreateStoreOp("store", add->y, axes); | ||
| 469 | + const auto output = CreateOutputOp("output", store->y, axes); | ||
| 470 | + output->ir_attr.SetIndex(0); | ||
| 471 | + // Preserve the unmodified MM output as a second graph output. | ||
| 472 | + const auto mm_store = CreateStoreOp("mm_store", mm.y, axes); | ||
| 473 | + const auto mm_output = CreateOutputOp("mm_output", mm_store->y, axes); | ||
| 474 | + mm_output->ir_attr.SetIndex(1); | ||
| 475 | + const auto compute_graph = AscGraphUtils::GetComputeGraph(*graph); | ||
| 476 | + for (const auto &node : compute_graph->GetAllNodes()) { | ||
| 477 | + node->GetOpDesc()->GetOrCreateAttrsGroup<AscNodeAttr>()->sched.axis = {axes.a.id, axes.b.id}; | ||
| 478 | + for (auto &desc : node->GetOpDesc()->GetAllOutputsDescPtr()) { | ||
| 479 | + desc->SetDataType(dtype); | ||
| 480 | + auto attr = desc->GetOrCreateAttrsGroup<AscTensorAttr>(); | ||
| 481 | + attr->axis = {axes.a.id, axes.b.id}; | ||
| 482 | + attr->repeats = {axes.A, axes.B}; | ||
| 483 | + attr->strides = {axes.B, axes.ONE}; | ||
| 484 | + } | ||
| 485 | + for (size_t i = 0; i < node->GetOpDesc()->GetInputsSize(); ++i) { | ||
| 486 | + node->GetOpDesc()->MutableInputDesc(i)->SetDataType(dtype); | ||
| 487 | + } | ||
| 488 | + } | ||
| 489 | + (void)GraphUtils::AddEdge(graph->FindNode("input_cast")->GetOutControlAnchor(), | ||
| 490 | + graph->FindNode("gelu")->GetInControlAnchor()); | ||
| 491 | + return graph; | ||
| 492 | +} | ||
| 493 | + | ||
| 494 | +ComputeGraphPtr WrapPrecisionGraph(const std::shared_ptr<AscGraph> &asc_graph, bool cube = true) { | ||
| 495 | + GraphBuilder builder("precision_root"); | ||
| 496 | + const auto node = builder.AddNode("fusion", "AscBackend", 0, 0); | ||
| 497 | + auto attrs = GetOrCreateAutoFuseAttrs(node->GetOpDesc()); | ||
| 498 | + if (cube) { | ||
| 499 | + attrs->SetAscGraph(asc_graph, loop::FuseType::kCube); | ||
| 500 | + } else { | ||
| 501 | + attrs->SetAscGraph(asc_graph); | ||
| 502 | + } | ||
| 503 | + return builder.GetGraph(); | ||
| 504 | +} | ||
| 505 | + | ||
| 506 | +DataType OutputDtype(const NodePtr &node) { | ||
| 507 | + return node->GetOpDesc()->GetOutputDesc(0).GetDataType(); | ||
| 508 | +} | ||
| 509 | + | ||
| 510 | +NodePtr InputProducer(const NodePtr &node, int index) { | ||
| 511 | + return node->GetInDataAnchor(index)->GetPeerOutAnchor()->GetOwnerNode(); | ||
| 512 | +} | ||
| 513 | +} // namespace | ||
| 514 | + | ||
| 515 | +class AdaptionMatmulPrecisionTest : public AdaptionImprovePrecisionTest { | ||
| 516 | + protected: | ||
| 517 | + void SetUp() override { | ||
| 518 | + AdaptionImprovePrecisionTest::SetUp(); | ||
| 519 | + blacklist_ = AutoFuseConfig::Config().GetFusionStrategySolver().improve_precision_blacklist; | ||
| 520 | + AutoFuseConfig::MutableConfig().GetMutableFusionStrategySolver().improve_precision_blacklist.clear(); | ||
| 521 | + } | ||
| 522 | + void TearDown() override { | ||
| 523 | + AutoFuseConfig::MutableConfig().GetMutableFusionStrategySolver().improve_precision_blacklist = blacklist_; | ||
| 524 | + AdaptionImprovePrecisionTest::TearDown(); | ||
| 525 | + } | ||
| 526 | + | ||
| 527 | + template <class Matmul> | ||
| 528 | + void CheckEpilogue(DataType dtype) { | ||
| 529 | + const auto graph = CreateMatmulPrecisionGraph<Matmul>(dtype); | ||
| 530 | + ASSERT_EQ(PrecisionImprover::ImprovePrecisionToFp32(WrapPrecisionGraph(graph)), SUCCESS); | ||
| 531 | + const auto mm = graph->FindNode("mm"); | ||
| 532 | + ASSERT_NE(mm, nullptr); | ||
| 533 | + for (const auto &name : {"a", "b", "load_a", "load_b", "input_cast", "mm", "mm_store", "mm_output"}) { | ||
| 534 | + ASSERT_NE(graph->FindNode(name), nullptr); | ||
| 535 | + EXPECT_EQ(OutputDtype(graph->FindNode(name)), dtype) << name; | ||
| 536 | + } | ||
| 537 | + EXPECT_EQ(InputProducer(mm, 0), graph->FindNode("input_cast")); | ||
| 538 | + EXPECT_EQ(InputProducer(mm, 1), graph->FindNode("load_b")); | ||
| 539 | + const auto gelu = graph->FindNode("gelu"); | ||
| 540 | + const auto mm_cast = InputProducer(gelu, 0); | ||
| 541 | + EXPECT_EQ(mm_cast->GetType(), "Cast"); | ||
| 542 | + EXPECT_EQ(OutputDtype(mm_cast), DT_FLOAT); | ||
| 543 | + EXPECT_EQ(InputProducer(mm_cast, 0), mm); | ||
| 544 | + EXPECT_EQ(OutputDtype(gelu), DT_FLOAT); | ||
| 545 | + const auto add = graph->FindNode("add"); | ||
| 546 | + EXPECT_EQ(OutputDtype(add), DT_FLOAT); | ||
| 547 | + const auto side_cast = InputProducer(add, 1); | ||
| 548 | + EXPECT_EQ(side_cast->GetType(), "Cast"); | ||
| 549 | + EXPECT_EQ(OutputDtype(side_cast), DT_FLOAT); | ||
| 550 | + EXPECT_EQ(InputProducer(side_cast, 0), graph->FindNode("load_b")); | ||
| 551 | + const auto store = graph->FindNode("store"); | ||
| 552 | + EXPECT_EQ(OutputDtype(store), dtype); | ||
| 553 | + const auto output_cast = InputProducer(store, 0); | ||
| 554 | + EXPECT_EQ(output_cast->GetType(), "Cast"); | ||
| 555 | + EXPECT_EQ(OutputDtype(output_cast), dtype); | ||
| 556 | + EXPECT_EQ(InputProducer(output_cast, 0), add); | ||
| 557 | + EXPECT_EQ(InputProducer(graph->FindNode("mm_store"), 0), mm); | ||
| 558 | + EXPECT_EQ(gelu->GetInControlNodes().size(), 1U); | ||
| 559 | + EXPECT_EQ(gelu->GetInControlNodes().at(0), graph->FindNode("input_cast")); | ||
| 560 | + EXPECT_EQ(mm->GetOpDesc()->GetInputDesc(0).GetDataType(), dtype); | ||
| 561 | + EXPECT_EQ(mm->GetOpDesc()->GetInputDesc(1).GetDataType(), dtype); | ||
| 562 | + } | ||
| 563 | + | ||
| 564 | + private: | ||
| 565 | + std::unordered_set<std::string> blacklist_; | ||
| 566 | +}; | ||
| 567 | + | ||
| 568 | +TEST_F(AdaptionMatmulPrecisionTest, Fp16MmPreservesOperandsSharedLoadAndDirectOutput) { | ||
| 569 | + CheckEpilogue<af::ascir_op::MatMul>(DT_FLOAT16); | ||
| 570 | +} | ||
| 571 | + | ||
| 572 | +TEST_F(AdaptionMatmulPrecisionTest, Bf16BmmPreservesOperandsSharedLoadAndDirectOutput) { | ||
| 573 | + CheckEpilogue<af::ascir_op::BatchMatMul>(DT_BF16); | ||
| 574 | +} | ||
| 575 | + | ||
| 576 | +TEST_F(AdaptionMatmulPrecisionTest, ConvStillSkipsPrecisionImprovement) { | ||
| 577 | + const auto graph = CreateMatmulPrecisionGraph<af::ascir_op::MatMul>(DT_FLOAT16); | ||
| 578 | + graph->FindNode("mm")->GetOpDesc()->SetType(kConv2D); | ||
| 579 | + const auto before_count = AscGraphUtils::GetComputeGraph(*graph)->GetAllNodesSize(); | ||
| 580 | + ASSERT_EQ(PrecisionImprover::ImprovePrecisionToFp32(WrapPrecisionGraph(graph)), SUCCESS); | ||
| 581 | + EXPECT_EQ(AscGraphUtils::GetComputeGraph(*graph)->GetAllNodesSize(), before_count); | ||
| 582 | + EXPECT_EQ(OutputDtype(graph->FindNode("gelu")), DT_FLOAT16); | ||
| 583 | +} | ||
| 584 | + | ||
| 585 | +TEST_F(AdaptionMatmulPrecisionTest, AllBlacklistStillDisablesOptionalEpiloguePromotion) { | ||
| 586 | + const auto graph = CreateMatmulPrecisionGraph<af::ascir_op::MatMul>(DT_FLOAT16); | ||
| 587 | + AutoFuseConfig::MutableConfig().GetMutableFusionStrategySolver().improve_precision_blacklist.insert("all"); | ||
| 588 | + const auto before_count = AscGraphUtils::GetComputeGraph(*graph)->GetAllNodesSize(); | ||
| 589 | + ASSERT_EQ(PrecisionImprover::ImprovePrecisionToFp32(WrapPrecisionGraph(graph)), SUCCESS); | ||
| 590 | + EXPECT_EQ(AscGraphUtils::GetComputeGraph(*graph)->GetAllNodesSize(), before_count); | ||
| 591 | + EXPECT_EQ(OutputDtype(graph->FindNode("gelu")), DT_FLOAT16); | ||
| 592 | +} | ||
| 593 | + | ||
| 594 | +TEST_F(AdaptionMatmulPrecisionTest, VectorFeedingAnotherMmKeepsItsOriginalPrecision) { | ||
| 595 | + const auto graph = CreateMatmulPrecisionGraph<af::ascir_op::MatMul>(DT_FLOAT16); | ||
| 596 | + const auto second_node = af::AscGraphAddAscirNodeByType(*graph, kMatMul.c_str(), "second_mm", 0U, 0U); | ||
| 597 | + ASSERT_NE(second_node, nullptr); | ||
| 598 | + ASSERT_EQ(GraphUtils::AddEdge(graph->FindNode("gelu")->GetOutDataAnchor(0), second_node->GetInDataAnchor(0)), | ||
| 599 | + GRAPH_SUCCESS); | ||
| 600 | + ASSERT_EQ(GraphUtils::AddEdge(graph->FindNode("load_b")->GetOutDataAnchor(0), second_node->GetInDataAnchor(1)), | ||
| 601 | + GRAPH_SUCCESS); | ||
| 602 | + second_node->GetOpDesc()->MutableOutputDesc(0)->SetDataType(DT_FLOAT16); | ||
| 603 | + ASSERT_EQ(PrecisionImprover::ImprovePrecisionToFp32(WrapPrecisionGraph(graph)), SUCCESS); | ||
| 604 | + EXPECT_EQ(OutputDtype(graph->FindNode("gelu")), DT_FLOAT16); | ||
| 605 | + EXPECT_EQ(InputProducer(graph->FindNode("gelu"), 0), graph->FindNode("mm")); | ||
| 606 | + EXPECT_EQ(OutputDtype(graph->FindNode("add")), DT_FLOAT); | ||
| 607 | + EXPECT_EQ(OutputDtype(graph->FindNode("second_mm")), DT_FLOAT16); | ||
| 608 | +} | ||
| 609 | + | ||
| 610 | +TEST_F(AdaptionMatmulPrecisionTest, ReluOnlyKeepsNativeFixpipeEligibility) { | ||
| 611 | + const auto graph = CreateMatmulPrecisionGraph<af::ascir_op::MatMul>(DT_FLOAT16); | ||
| 612 | + graph->FindNode("gelu")->GetOpDesc()->SetType(kReluType); | ||
| 613 | + ASSERT_EQ(asc_adapt::DelNode(*graph, graph->FindNode("add")), SUCCESS); | ||
| 614 | + const auto before_count = AscGraphUtils::GetComputeGraph(*graph)->GetAllNodesSize(); | ||
| 615 | + ASSERT_EQ(PrecisionImprover::ImprovePrecisionToFp32(WrapPrecisionGraph(graph)), SUCCESS); | ||
| 616 | + EXPECT_EQ(AscGraphUtils::GetComputeGraph(*graph)->GetAllNodesSize(), before_count); | ||
| 617 | + EXPECT_EQ(OutputDtype(graph->FindNode("gelu")), DT_FLOAT16); | ||
| 618 | + EXPECT_EQ(InputProducer(graph->FindNode("gelu"), 0), graph->FindNode("mm")); | ||
| 619 | +} | ||
| 620 | + | ||
| 621 | +namespace { | ||
| 622 | +NodePtr AddPrecisionVectorNode(const std::shared_ptr<AscGraph> &graph, const char *type, const char *name, | ||
| 623 | + const NodePtr &source) { | ||
| 624 | + const auto node = af::AscGraphAddAscirNodeByType(*graph, type, name, 0U, 0U); | ||
| 625 | + node->GetOpDesc()->UpdateOutputDesc(0, source->GetOpDesc()->GetOutputDesc(0)); | ||
| 626 | + node->GetOpDesc()->GetOrCreateAttrsGroup<AscNodeAttr>()->sched.axis = | ||
| 627 | + source->GetOpDesc()->GetAttrsGroup<AscNodeAttr>()->sched.axis; | ||
| 628 | + (void)GraphUtils::AddEdge(source->GetOutDataAnchor(0), node->GetInDataAnchor(0)); | ||
| 629 | + return node; | ||
| 630 | +} | ||
| 631 | + | ||
| 632 | +bool HasControlPredecessor(const NodePtr &node, const NodePtr &predecessor) { | ||
| 633 | + for (const auto &input : node->GetInControlNodes()) { | ||
| 634 | + if (input == predecessor) { | ||
| 635 | + return true; | ||
| 636 | + } | ||
| 637 | + } | ||
| 638 | + return false; | ||
| 639 | +} | ||
| 640 | + | ||
| 641 | +template <class Matmul> | ||
| 642 | +void CheckAuxiliaryVectorPrecision(DataType dtype) { | ||
| 643 | + const auto graph = CreateMatmulPrecisionGraph<Matmul>(dtype); | ||
| 644 | + const auto load = graph->FindNode("load_b"); | ||
| 645 | + const auto exp = AddPrecisionVectorNode(graph, "Exp", "side_exp", load); | ||
| 646 | + const auto div = AddPrecisionVectorNode(graph, "Div", "side_div", exp); | ||
| 647 | + ASSERT_EQ(GraphUtils::AddEdge(load->GetOutDataAnchor(0), div->GetInDataAnchor(1)), GRAPH_SUCCESS); | ||
| 648 | + const auto add = graph->FindNode("add"); | ||
| 649 | + ASSERT_EQ(GraphUtils::RemoveEdge(load->GetOutDataAnchor(0), add->GetInDataAnchor(1)), GRAPH_SUCCESS); | ||
| 650 | + ASSERT_EQ(GraphUtils::AddEdge(div->GetOutDataAnchor(0), add->GetInDataAnchor(1)), GRAPH_SUCCESS); | ||
| 651 | + ASSERT_EQ(PrecisionImprover::ImprovePrecisionToFp32(WrapPrecisionGraph(graph)), SUCCESS); | ||
| 652 | + EXPECT_EQ(OutputDtype(exp), DT_FLOAT); | ||
| 653 | + EXPECT_EQ(OutputDtype(div), DT_FLOAT); | ||
| 654 | + EXPECT_EQ(OutputDtype(add), DT_FLOAT); | ||
| 655 | + EXPECT_EQ(OutputDtype(load), dtype); | ||
| 656 | + EXPECT_EQ(OutputDtype(graph->FindNode("mm")), dtype); | ||
| 657 | + EXPECT_EQ(InputProducer(graph->FindNode("mm"), 1), load); | ||
| 658 | + EXPECT_EQ(OutputDtype(InputProducer(exp, 0)), DT_FLOAT); | ||
| 659 | + EXPECT_EQ(OutputDtype(InputProducer(div, 1)), DT_FLOAT); | ||
| 660 | + EXPECT_EQ(OutputDtype(graph->FindNode("store")), dtype); | ||
| 661 | +} | ||
| 662 | +} // namespace | ||
| 663 | + | ||
| 664 | +TEST_F(AdaptionMatmulPrecisionTest, Fp16MmAuxiliaryVectorBranchUsesVvPrecision) { | ||
| 665 | + CheckAuxiliaryVectorPrecision<af::ascir_op::MatMul>(DT_FLOAT16); | ||
| 666 | +} | ||
| 667 | + | ||
| 668 | +TEST_F(AdaptionMatmulPrecisionTest, Bf16BmmAuxiliaryVectorBranchUsesVvPrecision) { | ||
| 669 | + CheckAuxiliaryVectorPrecision<af::ascir_op::BatchMatMul>(DT_BF16); | ||
| 670 | +} | ||
| 671 | + | ||
| 672 | +TEST_F(AdaptionMatmulPrecisionTest, ControlledEpilogueCastPreservesAllControlSuccessors) { | ||
| 673 | + const auto graph = CreateMatmulPrecisionGraph<af::ascir_op::MatMul>(DT_FLOAT16); | ||
| 674 | + const auto mm = graph->FindNode("mm"); | ||
| 675 | + const auto gelu = graph->FindNode("gelu"); | ||
| 676 | + const auto cast = AddPrecisionVectorNode(graph, "Cast", "epilogue_cast", mm); | ||
| 677 | + cast->GetOpDesc()->MutableOutputDesc(0)->SetDataType(DT_FLOAT); | ||
| 678 | + ASSERT_EQ(GraphUtils::RemoveEdge(mm->GetOutDataAnchor(0), gelu->GetInDataAnchor(0)), GRAPH_SUCCESS); | ||
| 679 | + ASSERT_EQ(GraphUtils::AddEdge(cast->GetOutDataAnchor(0), gelu->GetInDataAnchor(0)), GRAPH_SUCCESS); | ||
| 680 | + const auto load = graph->FindNode("load_b"); | ||
| 681 | + const auto before = AddPrecisionVectorNode(graph, "Abs", "before", load); | ||
| 682 | + const auto after = AddPrecisionVectorNode(graph, "Abs", "after", load); | ||
| 683 | + const auto data_control = AddPrecisionVectorNode(graph, "Abs", "data_control", load); | ||
| 684 | + ASSERT_EQ(GraphUtils::AddEdge(before->GetOutControlAnchor(), cast->GetInControlAnchor()), GRAPH_SUCCESS); | ||
| 685 | + ASSERT_EQ(GraphUtils::AddEdge(cast->GetOutControlAnchor(), after->GetInControlAnchor()), GRAPH_SUCCESS); | ||
| 686 | + ASSERT_EQ(GraphUtils::AddEdge(cast->GetOutDataAnchor(0), data_control->GetInControlAnchor()), GRAPH_SUCCESS); | ||
| 687 | + ASSERT_EQ(PrecisionImprover::ImprovePrecisionToFp32(WrapPrecisionGraph(graph)), SUCCESS); | ||
| 688 | + EXPECT_EQ(graph->FindNode("epilogue_cast"), cast); | ||
| 689 | + EXPECT_EQ(InputProducer(gelu, 0), cast); | ||
| 690 | + EXPECT_EQ(OutputDtype(cast), DT_FLOAT); | ||
| 691 | + EXPECT_TRUE(HasControlPredecessor(cast, before)); | ||
| 692 | + EXPECT_TRUE(HasControlPredecessor(after, cast)); | ||
| 693 | + EXPECT_TRUE(data_control->GetInControlAnchor()->IsLinkedWith(cast->GetOutDataAnchor(0))); | ||
| 694 | +} | ||
| 695 | + | ||
| 696 | +TEST_F(AdaptionMatmulPrecisionTest, ControlledEpilogueCastKeepsHighFanoutWithoutNoOp) { | ||
| 697 | + const auto graph = CreateMatmulPrecisionGraph<af::ascir_op::MatMul>(DT_FLOAT16); | ||
| 698 | + const auto mm = graph->FindNode("mm"); | ||
| 699 | + const auto gelu = graph->FindNode("gelu"); | ||
| 700 | + const auto cast = AddPrecisionVectorNode(graph, "Cast", "epilogue_cast", mm); | ||
| 701 | + cast->GetOpDesc()->MutableOutputDesc(0)->SetDataType(DT_FLOAT); | ||
| 702 | + ASSERT_EQ(GraphUtils::RemoveEdge(mm->GetOutDataAnchor(0), gelu->GetInDataAnchor(0)), GRAPH_SUCCESS); | ||
| 703 | + ASSERT_EQ(GraphUtils::AddEdge(cast->GetOutDataAnchor(0), gelu->GetInDataAnchor(0)), GRAPH_SUCCESS); | ||
| 704 | + std::vector<NodePtr> predecessors; | ||
| 705 | + std::vector<NodePtr> successors; | ||
| 706 | + for (size_t i = 0; i < 32U; ++i) { | ||
| 707 | + const auto before_name = "before_" + std::to_string(i); | ||
| 708 | + const auto after_name = "after_" + std::to_string(i); | ||
| 709 | + predecessors.push_back(AddPrecisionVectorNode(graph, "Abs", before_name.c_str(), graph->FindNode("load_b"))); | ||
| 710 | + successors.push_back(AddPrecisionVectorNode(graph, "Abs", after_name.c_str(), graph->FindNode("load_b"))); | ||
| 711 | + ASSERT_EQ(GraphUtils::AddEdge(predecessors.back()->GetOutControlAnchor(), cast->GetInControlAnchor()), | ||
| 712 | + GRAPH_SUCCESS); | ||
| 713 | + ASSERT_EQ(GraphUtils::AddEdge(cast->GetOutControlAnchor(), successors.back()->GetInControlAnchor()), GRAPH_SUCCESS); | ||
| 714 | + } | ||
| 715 | + ASSERT_EQ(PrecisionImprover::ImprovePrecisionToFp32(WrapPrecisionGraph(graph)), SUCCESS); | ||
| 716 | + EXPECT_EQ(graph->FindNode("epilogue_cast"), cast); | ||
| 717 | + EXPECT_EQ(InputProducer(gelu, 0), cast); | ||
| 718 | + EXPECT_EQ(cast->GetInControlNodes().size(), predecessors.size()); | ||
| 719 | + EXPECT_EQ(cast->GetOutControlNodes().size(), successors.size()); | ||
| 720 | + for (const auto &predecessor : predecessors) { | ||
| 721 | + EXPECT_TRUE(HasControlPredecessor(cast, predecessor)); | ||
| 722 | + } | ||
| 723 | + for (const auto &successor : successors) { | ||
| 724 | + EXPECT_TRUE(HasControlPredecessor(successor, cast)); | ||
| 725 | + } | ||
| 726 | + for (const auto &node : graph->GetAllNodes()) { | ||
| 727 | + EXPECT_NE(node->GetType(), "NoOp"); | ||
| 728 | + } | ||
| 729 | +} | ||
| 730 | + | ||
| 731 | +TEST_F(AdaptionMatmulPrecisionTest, EpilogueCastWithOnlyDataControlSuccessorIsRetained) { | ||
| 732 | + const auto graph = CreateMatmulPrecisionGraph<af::ascir_op::MatMul>(DT_FLOAT16); | ||
| 733 | + const auto mm = graph->FindNode("mm"); | ||
| 734 | + const auto gelu = graph->FindNode("gelu"); | ||
| 735 | + const auto cast = AddPrecisionVectorNode(graph, "Cast", "epilogue_cast", mm); | ||
| 736 | + cast->GetOpDesc()->MutableOutputDesc(0)->SetDataType(DT_FLOAT); | ||
| 737 | + ASSERT_EQ(GraphUtils::RemoveEdge(mm->GetOutDataAnchor(0), gelu->GetInDataAnchor(0)), GRAPH_SUCCESS); | ||
| 738 | + ASSERT_EQ(GraphUtils::AddEdge(cast->GetOutDataAnchor(0), gelu->GetInDataAnchor(0)), GRAPH_SUCCESS); | ||
| 739 | + const auto after = AddPrecisionVectorNode(graph, "Abs", "after", graph->FindNode("load_b")); | ||
| 740 | + ASSERT_EQ(GraphUtils::AddEdge(cast->GetOutDataAnchor(0), after->GetInControlAnchor()), GRAPH_SUCCESS); | ||
| 741 | + ASSERT_EQ(PrecisionImprover::ImprovePrecisionToFp32(WrapPrecisionGraph(graph)), SUCCESS); | ||
| 742 | + EXPECT_EQ(graph->FindNode("epilogue_cast"), cast); | ||
| 743 | + EXPECT_EQ(InputProducer(gelu, 0), cast); | ||
| 744 | + EXPECT_TRUE(after->GetInControlAnchor()->IsLinkedWith(cast->GetOutDataAnchor(0))); | ||
| 745 | +} | ||
| 746 | + | ||
| 747 | +TEST_F(AdaptionMatmulPrecisionTest, UncontrolledEpilogueCastStillUsesOrdinaryElimination) { | ||
| 748 | + const auto graph = CreateMatmulPrecisionGraph<af::ascir_op::MatMul>(DT_FLOAT16); | ||
| 749 | + const auto mm = graph->FindNode("mm"); | ||
| 750 | + const auto gelu = graph->FindNode("gelu"); | ||
| 751 | + const auto cast = AddPrecisionVectorNode(graph, "Cast", "epilogue_cast", mm); | ||
| 752 | + cast->GetOpDesc()->MutableOutputDesc(0)->SetDataType(DT_FLOAT); | ||
| 753 | + ASSERT_EQ(GraphUtils::RemoveEdge(mm->GetOutDataAnchor(0), gelu->GetInDataAnchor(0)), GRAPH_SUCCESS); | ||
| 754 | + ASSERT_EQ(GraphUtils::AddEdge(cast->GetOutDataAnchor(0), gelu->GetInDataAnchor(0)), GRAPH_SUCCESS); | ||
| 755 | + ASSERT_EQ(PrecisionImprover::ImprovePrecisionToFp32(WrapPrecisionGraph(graph)), SUCCESS); | ||
| 756 | + EXPECT_EQ(graph->FindNode("epilogue_cast"), nullptr); | ||
| 757 | + const auto promoted_input = InputProducer(gelu, 0); | ||
| 758 | + EXPECT_EQ(promoted_input->GetType(), "Cast"); | ||
| 759 | + EXPECT_EQ(OutputDtype(promoted_input), DT_FLOAT); | ||
| 760 | + EXPECT_EQ(InputProducer(promoted_input, 0), mm); | ||
| 761 | +} | ||
| 445 | } // namespace ge | 762 | } // namespace ge |
| @@ -824,6 +824,117 @@ TEST_F(LoopApiUT, StoreMatmulInputOverflowWithCubeWithOutMatmulEnv) { | |||
| 824 | EXPECT_TRUE(kernel_box.IsExternKernel()); | 824 | EXPECT_TRUE(kernel_box.IsExternKernel()); |
| 825 | } | 825 | } |
| 826 | 826 | ||
| 827 | +TEST_F(LoopApiUT, StoreMatmulDynamicShapeUsesCubeLowering) { | ||
| 828 | + [this]() { | ||
| 829 | + auto data0 = es_graph_->CreateInput(0, "data0", nullptr); | ||
| 830 | + data0.SetSymbolShape({"s_m", "s_k"}); | ||
| 831 | + auto data1 = es_graph_->CreateInput(1, "data1", nullptr); | ||
| 832 | + data1.SetSymbolShape({"s_k", "s_n"}); | ||
| 833 | + auto mm = es::MatMul(data0, data1); | ||
| 834 | + mm.SetSymbolShape({"s_m", "s_n"}); | ||
| 835 | + es_graph_->SetOutput(mm, 0); | ||
| 836 | + }(); | ||
| 837 | + | ||
| 838 | + auto graph = es_graph_->Build(); | ||
| 839 | + auto cg = GraphUtilsEx::GetComputeGraph(*graph); | ||
| 840 | + auto mm = cg->FindNode("MatMul_0"); | ||
| 841 | + ASSERT_NE(mm, nullptr); | ||
| 842 | + | ||
| 843 | + AutoFuseConfig::MutableLoweringConfig().experimental_lowering_matmul = true; | ||
| 844 | + std::vector<ge::InDataAnchorPtr> inputs = {mm->GetInDataAnchor(0), mm->GetInDataAnchor(1)}; | ||
| 845 | + MatMulAttr matmul_attr; | ||
| 846 | + auto kernel_box = loop::StoreMatMul(mm->GetOutDataAnchor(0), inputs, matmul_attr); | ||
| 847 | + | ||
| 848 | + EXPECT_FALSE(kernel_box.IsExternKernel()); | ||
| 849 | +} | ||
| 850 | + | ||
| 851 | +TEST_F(LoopApiUT, StoreMatmulDynamicShapeWithMismatchedKFallsBack) { | ||
| 852 | + [this]() { | ||
| 853 | + auto data0 = es_graph_->CreateInput(0, "data0", nullptr); | ||
| 854 | + data0.SetSymbolShape({"s_m", "s_k0"}); | ||
| 855 | + auto data1 = es_graph_->CreateInput(1, "data1", nullptr); | ||
| 856 | + data1.SetSymbolShape({"s_k1", "s_n"}); | ||
| 857 | + auto mm = es::MatMul(data0, data1); | ||
| 858 | + mm.SetSymbolShape({"s_m", "s_n"}); | ||
| 859 | + es_graph_->SetOutput(mm, 0); | ||
| 860 | + }(); | ||
| 861 | + | ||
| 862 | + auto graph = es_graph_->Build(); | ||
| 863 | + auto cg = GraphUtilsEx::GetComputeGraph(*graph); | ||
| 864 | + auto mm = cg->FindNode("MatMul_0"); | ||
| 865 | + ASSERT_NE(mm, nullptr); | ||
| 866 | + | ||
| 867 | + AutoFuseConfig::MutableLoweringConfig().experimental_lowering_matmul = true; | ||
| 868 | + std::vector<ge::InDataAnchorPtr> inputs = {mm->GetInDataAnchor(0), mm->GetInDataAnchor(1)}; | ||
| 869 | + MatMulAttr matmul_attr; | ||
| 870 | + auto kernel_box = loop::StoreMatMul(mm->GetOutDataAnchor(0), inputs, matmul_attr); | ||
| 871 | + | ||
| 872 | + EXPECT_TRUE(kernel_box.IsExternKernel()); | ||
| 873 | +} | ||
| 874 | + | ||
| 875 | +TEST_F(LoopApiUT, StoreBatchMatmulDynamicShapeUsesCubeLowering) { | ||
| 876 | + [this]() { | ||
| 877 | + auto data0 = es_graph_->CreateInput(0, "data0", nullptr); | ||
| 878 | + data0.SetSymbolShape({"s_batch", "s_m", "s_k"}); | ||
| 879 | + auto data1 = es_graph_->CreateInput(1, "data1", nullptr); | ||
| 880 | + data1.SetSymbolShape({"s_batch", "s_k", "s_n"}); | ||
| 881 | + auto mm = es::BatchMatMul(data0, data1); | ||
| 882 | + mm.SetSymbolShape({"s_batch", "s_m", "s_n"}); | ||
| 883 | + es_graph_->SetOutput(mm, 0); | ||
| 884 | + }(); | ||
| 885 | + | ||
| 886 | + auto graph = es_graph_->Build(); | ||
| 887 | + auto cg = GraphUtilsEx::GetComputeGraph(*graph); | ||
| 888 | + auto mm = cg->FindNode("BatchMatMul_0"); | ||
| 889 | + ASSERT_NE(mm, nullptr); | ||
| 890 | + | ||
| 891 | + AutoFuseConfig::MutableLoweringConfig().experimental_lowering_matmul = true; | ||
| 892 | + std::vector<ge::InDataAnchorPtr> inputs = {mm->GetInDataAnchor(0), mm->GetInDataAnchor(1)}; | ||
| 893 | + MatMulAttr matmul_attr; | ||
| 894 | + matmul_attr.is_batch = true; | ||
| 895 | + auto kernel_box = loop::StoreMatMul(mm->GetOutDataAnchor(0), inputs, matmul_attr); | ||
| 896 | + | ||
| 897 | + EXPECT_FALSE(kernel_box.IsExternKernel()); | ||
| 898 | +} | ||
| 899 | + | ||
| 900 | +TEST_F(LoopApiUT, StoreBatchMatmulUnsupportedDynamicShapesFallBack) { | ||
| 901 | + struct ShapeCase { | ||
| 902 | + std::vector<const char *> x1; | ||
| 903 | + std::vector<const char *> x2; | ||
| 904 | + std::vector<const char *> y; | ||
| 905 | + }; | ||
| 906 | + const std::vector<ShapeCase> shape_cases = { | ||
| 907 | + // Batch broadcasting remains outside the dynamic Cube path. | ||
| 908 | + {{"1", "s_m", "s_k"}, {"s_batch", "s_k", "s_n"}, {"s_batch", "s_m", "s_n"}}, | ||
| 909 | + // Equal input batch products do not imply matching batch axes. | ||
| 910 | + {{"s_batch", "1", "s_m", "s_k"}, {"1", "s_batch", "s_k", "s_n"}, {"s_batch", "s_batch", "s_m", "s_n"}}, | ||
| 911 | + // K dimensions must be provably equal. | ||
| 912 | + {{"s_batch", "s_m", "s_k0"}, {"s_batch", "s_k1", "s_n"}, {"s_batch", "s_m", "s_n"}}, | ||
| 913 | + }; | ||
| 914 | + AutoFuseConfig::MutableLoweringConfig().experimental_lowering_matmul = true; | ||
| 915 | + for (size_t i = 0U; i < shape_cases.size(); ++i) { | ||
| 916 | + SCOPED_TRACE(i); | ||
| 917 | + const auto &shape = shape_cases[i]; | ||
| 918 | + es::Graph builder("dynamic_bmm_fallback"); | ||
| 919 | + auto data0 = builder.CreateInput(0, "data0", nullptr); | ||
| 920 | + auto data1 = builder.CreateInput(1, "data1", nullptr); | ||
| 921 | + data0.SetSymbolShape(shape.x1); | ||
| 922 | + data1.SetSymbolShape(shape.x2); | ||
| 923 | + auto output = es::BatchMatMul(data0, data1); | ||
| 924 | + output.SetSymbolShape(shape.y); | ||
| 925 | + ASSERT_EQ(builder.SetOutput(output, 0), SUCCESS); | ||
| 926 | + const auto graph = builder.Build(); | ||
| 927 | + const auto cg = GraphUtilsEx::GetComputeGraph(*graph); | ||
| 928 | + const auto mm = cg->FindNode("BatchMatMul_0"); | ||
| 929 | + ASSERT_NE(mm, nullptr); | ||
| 930 | + const std::vector<ge::InDataAnchorPtr> inputs = {mm->GetInDataAnchor(0), mm->GetInDataAnchor(1)}; | ||
| 931 | + MatMulAttr matmul_attr; | ||
| 932 | + matmul_attr.is_batch = true; | ||
| 933 | + const auto kernel_box = loop::StoreMatMul(mm->GetOutDataAnchor(0), inputs, matmul_attr); | ||
| 934 | + EXPECT_TRUE(kernel_box.IsExternKernel()); | ||
| 935 | + } | ||
| 936 | +} | ||
| 937 | + | ||
| 827 | TEST_F(LoopApiUT, StoreMatmulInputOverflow) { | 938 | TEST_F(LoopApiUT, StoreMatmulInputOverflow) { |
| 828 | [this]() { | 939 | [this]() { |
| 829 | auto data0 = es_graph_->CreateInput(0, "data0", nullptr); | 940 | auto data0 = es_graph_->CreateInput(0, "data0", nullptr); |
| @@ -2767,6 +2767,34 @@ tmp6 = ascir.Output(Output_0, [tmp5]) | |||
| 2767 | )"); | 2767 | )"); |
| 2768 | } | 2768 | } |
| 2769 | 2769 | ||
| 2770 | +TEST_F(LoopLoweringToAscBackendUT, LoweringDynamicMatmulRankOneBias) { | ||
| 2771 | + auto data0 = es_graph_->CreateInput(0, "data0", nullptr); | ||
| 2772 | + auto data1 = es_graph_->CreateInput(1, "data1", nullptr); | ||
| 2773 | + auto bias = es_graph_->CreateInput(2, "bias", nullptr); | ||
| 2774 | + data0.SetSymbolShape({"s_m", "s_k"}); | ||
| 2775 | + data1.SetSymbolShape({"s_k", "s_n"}); | ||
| 2776 | + bias.SetSymbolShape({"s_n"}); | ||
| 2777 | + auto mm = es::MatMul(data0, data1, bias); | ||
| 2778 | + mm.SetSymbolShape({"s_m", "s_n"}); | ||
| 2779 | + es_graph_->SetOutput(mm, 0); | ||
| 2780 | + | ||
| 2781 | + auto graph = es_graph_->Build(); | ||
| 2782 | + auto cg = GraphUtilsEx::GetComputeGraph(*graph); | ||
| 2783 | + AutoFuseConfig::MutableLoweringConfig().experimental_lowering_matmul = true; | ||
| 2784 | + ASSERT_EQ(LoweringManager::LoweringGraph(cg), GRAPH_SUCCESS); | ||
| 2785 | + ASSERT_EQ(LoweringManager::FusedLoopToAscBackendOp(cg), GRAPH_SUCCESS); | ||
| 2786 | + | ||
| 2787 | + auto asc_node = cg->FindFirstNodeMatchType("AscBackend"); | ||
| 2788 | + ASSERT_NE(asc_node, nullptr); | ||
| 2789 | + auto fused_attrs = asc_node->GetOpDesc()->GetAttrsGroup<AutoFuseAttrs>(); | ||
| 2790 | + ASSERT_NE(fused_attrs, nullptr); | ||
| 2791 | + ASSERT_NE(fused_attrs->GetAscGraph(), nullptr); | ||
| 2792 | + EXPECT_EQ(fused_attrs->GetFuseType(), loop::FuseType::kCube); | ||
| 2793 | + const auto readable = ReadableAscGraph(*fused_attrs->GetAscGraph()); | ||
| 2794 | + EXPECT_NE(readable.find("ascir.MatMulBias("), std::string::npos); | ||
| 2795 | + EXPECT_NE(readable.find("repeats = [1, s_n]"), std::string::npos); | ||
| 2796 | +} | ||
| 2797 | + | ||
| 2770 | TEST_F(LoopLoweringToAscBackendUT, LoweringMatmulBias) { | 2798 | TEST_F(LoopLoweringToAscBackendUT, LoweringMatmulBias) { |
| 2771 | [this]() { | 2799 | [this]() { |
| 2772 | auto data0 = es_graph_->CreateInput(0, "data0", nullptr); | 2800 | auto data0 = es_graph_->CreateInput(0, "data0", nullptr); |
| @@ -9,6 +9,8 @@ | |||
| 9 | * See LICENSE in the root of the software repository for the full text of the License. | 9 | * See LICENSE in the root of the software repository for the full text of the License. |
| 10 | */ | 10 | */ |
| 11 | #include <gtest/gtest.h> | 11 | #include <gtest/gtest.h> |
| 12 | +#include "depends/runtime/src/runtime_stub.h" | ||
| 13 | +#include "common/autofuse_platform_api.h" | ||
| 12 | 14 | ||
| 13 | #include "graph/attribute_group/attr_group_symbolic_desc.h" | 15 | #include "graph/attribute_group/attr_group_symbolic_desc.h" |
| 14 | #include "graph/attribute_group/attr_group_shape_env.h" | 16 | #include "graph/attribute_group/attr_group_shape_env.h" |
| @@ -2152,4 +2154,118 @@ TEST_F(LoweringAndCanfuseUT, ReshapePaddingAnchorAlignmentFallbackWhenNoAnchorMa | |||
| 2152 | EXPECT_EQ(insert_indexes[0], 1U); | 2154 | EXPECT_EQ(insert_indexes[0], 1U); |
| 2153 | } | 2155 | } |
| 2154 | 2156 | ||
| 2157 | +class MatmulPrecisionLoweringUT : public LoweringAndCanfuseUT { | ||
| 2158 | + protected: | ||
| 2159 | + void SetUp() override { | ||
| 2160 | + LoweringAndCanfuseUT::SetUp(); | ||
| 2161 | + old_lower_mm_ = AutoFuseConfig::LoweringConfig().experimental_lowering_matmul; | ||
| 2162 | + const auto value = std::getenv("ENABLE_LOWER_MATMUL"); | ||
| 2163 | + had_lower_mm_env_ = value != nullptr; | ||
| 2164 | + old_lower_mm_env_ = value == nullptr ? "" : value; | ||
| 2165 | + setenv("ENABLE_LOWER_MATMUL", "true", 1); | ||
| 2166 | + RuntimeStub::SetInstance(std::make_shared<RuntimeStubV2Common>()); | ||
| 2167 | + ResetAutofusePlatform(); | ||
| 2168 | + } | ||
| 2169 | + | ||
| 2170 | + void TearDown() override { | ||
| 2171 | + SetCurShapeEnvContext(nullptr); | ||
| 2172 | + AutoFuseConfig::MutableLoweringConfig().experimental_lowering_matmul = old_lower_mm_; | ||
| 2173 | + if (had_lower_mm_env_) { | ||
| 2174 | + setenv("ENABLE_LOWER_MATMUL", old_lower_mm_env_.c_str(), 1); | ||
| 2175 | + } else { | ||
| 2176 | + unsetenv("ENABLE_LOWER_MATMUL"); | ||
| 2177 | + } | ||
| 2178 | + RuntimeStub::Reset(); | ||
| 2179 | + ResetAutofusePlatform(); | ||
| 2180 | + LoweringAndCanfuseUT::TearDown(); | ||
| 2181 | + } | ||
| 2182 | + | ||
| 2183 | + ComputeGraphPtr BuildAuxiliaryBranch(bool dynamic) { | ||
| 2184 | + const std::vector<const char *> shape = | ||
| 2185 | + dynamic ? std::vector<const char *>{"s0", "s1", "s1"} : std::vector<const char *>{"2", "16", "16"}; | ||
| 2186 | + auto a = es_graph_->CreateInput(0, "a", nullptr); | ||
| 2187 | + auto b = es_graph_->CreateInput(1, "b", nullptr); | ||
| 2188 | + a.SetSymbolShape(shape); | ||
| 2189 | + b.SetSymbolShape(shape); | ||
| 2190 | + auto mm = es::BatchMatMul(a, b); | ||
| 2191 | + mm.SetSymbolShape(shape); | ||
| 2192 | + auto exp = es::Exp(b); | ||
| 2193 | + exp.SetSymbolShape(shape); | ||
| 2194 | + auto div = es::Div(exp, b); | ||
| 2195 | + div.SetSymbolShape(shape); | ||
| 2196 | + auto add = es::Add(mm, div); | ||
| 2197 | + add.SetSymbolShape(shape); | ||
| 2198 | + (void)es_graph_->SetOutput(add, 0); | ||
| 2199 | + shape_env_ = | ||
| 2200 | + std::make_unique<ShapeEnvAttr>(ShapeEnvSetting(false, dynamic ? DynamicMode::kDynamic : DynamicMode::kStatic)); | ||
| 2201 | + SetCurShapeEnvContext(shape_env_.get()); | ||
| 2202 | + if (dynamic) { | ||
| 2203 | + shape_env_->CreateSymbol(2, MakeShared<GraphInputShapeSourceStub>(0, 0)); | ||
| 2204 | + shape_env_->CreateSymbol(16, MakeShared<GraphInputShapeSourceStub>(0, 1)); | ||
| 2205 | + } | ||
| 2206 | + const auto graph = es_graph_->Build(); | ||
| 2207 | + const auto cg = GraphUtilsEx::GetComputeGraph(*graph); | ||
| 2208 | + for (const auto &node : cg->GetAllNodes()) { | ||
| 2209 | + for (size_t i = 0; i < node->GetOpDesc()->GetOutputsSize(); ++i) { | ||
| 2210 | + node->GetOpDesc()->MutableOutputDesc(i)->SetDataType(DT_FLOAT16); | ||
| 2211 | + } | ||
| 2212 | + for (size_t i = 0; i < node->GetOpDesc()->GetInputsSize(); ++i) { | ||
| 2213 | + node->GetOpDesc()->MutableInputDesc(i)->SetDataType(DT_FLOAT16); | ||
| 2214 | + } | ||
| 2215 | + } | ||
| 2216 | + return cg; | ||
| 2217 | + } | ||
| 2218 | + | ||
| 2219 | + void CheckAuxiliaryBranch(bool dynamic, bool mm_enabled) { | ||
| 2220 | + AutoFuseConfig::MutableLoweringConfig().experimental_lowering_matmul = mm_enabled; | ||
| 2221 | + const auto cg = BuildAuxiliaryBranch(dynamic); | ||
| 2222 | + AscIrLowerer lowerer; | ||
| 2223 | + ASSERT_EQ(lowerer.Lowering(cg), GRAPH_SUCCESS); | ||
| 2224 | + ASSERT_EQ(asc_adapt::GeFallback(cg), GRAPH_SUCCESS); | ||
| 2225 | + FusionStrategySolver solver; | ||
| 2226 | + FusionDeciderRegistry::Instance().Register(std::unique_ptr<FusionDecider>(new AscBackendFusionDecider())); | ||
| 2227 | + ASSERT_EQ(solver.Fuse(cg), SUCCESS); | ||
| 2228 | + ASSERT_EQ(lowerer.Lifting(cg), GRAPH_SUCCESS); | ||
| 2229 | + ASSERT_EQ(AscBackendPostProcessor().Do(cg), SUCCESS); | ||
| 2230 | + size_t cube_groups = 0; | ||
| 2231 | + size_t side_nodes = 0; | ||
| 2232 | + for (const auto &node : cg->GetAllNodes()) { | ||
| 2233 | + const auto attr = node->GetOpDesc()->GetAttrsGroup<AutoFuseAttrs>(); | ||
| 2234 | + if (attr == nullptr || attr->GetAscGraph() == nullptr) { | ||
| 2235 | + continue; | ||
| 2236 | + } | ||
| 2237 | + cube_groups += attr->HasFuseType(loop::FuseType::kCube) ? 1U : 0U; | ||
| 2238 | + for (const auto &inner : attr->GetAscGraph()->GetAllNodes()) { | ||
| 2239 | + if (inner->GetType() == "Exp" || inner->GetType() == "Div") { | ||
| 2240 | + ++side_nodes; | ||
| 2241 | + EXPECT_EQ(inner->GetOpDesc()->GetOutputDesc(0).GetDataType(), DT_FLOAT); | ||
| 2242 | + } | ||
| 2243 | + } | ||
| 2244 | + } | ||
| 2245 | + EXPECT_EQ(cube_groups, mm_enabled ? 1U : 0U); | ||
| 2246 | + EXPECT_EQ(side_nodes, 2U); | ||
| 2247 | + } | ||
| 2248 | + | ||
| 2249 | + private: | ||
| 2250 | + bool old_lower_mm_ = false; | ||
| 2251 | + bool had_lower_mm_env_ = false; | ||
| 2252 | + std::string old_lower_mm_env_; | ||
| 2253 | + std::unique_ptr<ShapeEnvAttr> shape_env_; | ||
| 2254 | +}; | ||
| 2255 | + | ||
| 2256 | +TEST_F(MatmulPrecisionLoweringUT, StaticMmOnPromotesAuxiliaryVector) { | ||
| 2257 | + CheckAuxiliaryBranch(false, true); | ||
| 2258 | +} | ||
| 2259 | + | ||
| 2260 | +TEST_F(MatmulPrecisionLoweringUT, StaticMmOffPromotesAuxiliaryVector) { | ||
| 2261 | + CheckAuxiliaryBranch(false, false); | ||
| 2262 | +} | ||
| 2263 | + | ||
| 2264 | +TEST_F(MatmulPrecisionLoweringUT, DynamicMmOnPromotesAuxiliaryVector) { | ||
| 2265 | + CheckAuxiliaryBranch(true, true); | ||
| 2266 | +} | ||
| 2267 | + | ||
| 2268 | +TEST_F(MatmulPrecisionLoweringUT, DynamicMmOffPromotesAuxiliaryVector) { | ||
| 2269 | + CheckAuxiliaryBranch(true, false); | ||
| 2270 | +} | ||
| 2155 | } // namespace ge | 2271 | } // namespace ge |
🟡 Medium Priority
改动删除了
(void)ge::AttrUtils::SetInt(node_op_desc, "offset_x", 0x80);与(void)ge::AttrUtils::SetInt(op_desc, "offset_x", 0x80);两行，但保留了上一行的auto node_op_desc = node->GetOpDesc();（本文件第 840 行）。在AutofuseNodeWithMatmulTiling余下代码（841-861 行）中只使用op_desc、space_registry、run_info等，node_op_desc再无任何引用。 影响：该局部变量成为纯粹的死代码；在开启-Wunused-variable（CANN 构建常用-Wall -Wextra，部分目标叠加-Werror）时会退化编译告警甚至直接编译失败，属于本次仅删两行引入的残留。