已合并
feat: 完善 IndirectLoad 广播融合、Reduce 后融合及 SIMT 地址处理 #1783
xiebangrui2025创建于 2 天前
feat: 完善 IndirectLoad 广播融合、Reduce 后融合及 SIMT 地址处理 #1783
已合并
共 32 个文件变更+3686-3259
| @@ -765,7 +765,9 @@ af::Status AscendGraphParser::GetNodeFromData(const af::AscNodePtr &ge_node, Nod | |||
| 765 | af::Status AscendGraphParser::ConvertNodeInfos(const af::AscNodePtr &ge_node, const ScheduleAttr &attrs, | 765 | af::Status AscendGraphParser::ConvertNodeInfos(const af::AscNodePtr &ge_node, const ScheduleAttr &attrs, |
| 766 | const af::AscGraph &graph, const bool use_cache_flag) { | 766 | const af::AscGraph &graph, const bool use_cache_flag) { |
| 767 | if (ascgen_utils::indirect_load::GetTemplateBehavior(ge_node).uses_direct_gm_pipeline) { | 767 | if (ascgen_utils::indirect_load::GetTemplateBehavior(ge_node).uses_direct_gm_pipeline) { |
| 768 | - if (ascgen_utils::indirect_load::IsPostReduceInputProducer(ge_node)) { | 768 | + const auto post_reduce_consumer = ascgen_utils::indirect_load::GetOnlyOutputConsumer(ge_node); |
| 769 | + if (post_reduce_consumer != nullptr && | ||
| 770 | + post_reduce_consumer->attr.api.compute_type == af::ComputeType::kComputeReduce) { | ||
| 769 | NodeInfo reduce_input_tensor_info; | 771 | NodeInfo reduce_input_tensor_info; |
| 770 | reduce_input_tensor_info.name = ge_node->GetName(); | 772 | reduce_input_tensor_info.name = ge_node->GetName(); |
| 771 | GE_ASSERT_SUCCESS(ParserNodeOutputInfos(ge_node, graph, reduce_input_tensor_info)); | 773 | GE_ASSERT_SUCCESS(ParserNodeOutputInfos(ge_node, graph, reduce_input_tensor_info)); |
| @@ -632,12 +632,12 @@ static std::string GetCacheGuardCondition(const ApiCall &call, bool is_enable_ca | |||
| 632 | 632 | ||
| 633 | Status Loop::GenerateBody(const Tiler &tiler, const TPipe &tpipe, std::vector<ascir::AxisId> ¤t_axis, | 633 | Status Loop::GenerateBody(const Tiler &tiler, const TPipe &tpipe, std::vector<ascir::AxisId> ¤t_axis, |
| 634 | std::stringstream &ss) { | 634 | std::stringstream &ss) { |
| 635 | - bool need_collect = this->bodys.size() > 1; | ||
| 636 | std::map<ascir::AxisId, std::map<Loop *, std::vector<ApiCall *>>> api_calls_cross_loop; | 635 | std::map<ascir::AxisId, std::map<Loop *, std::vector<ApiCall *>>> api_calls_cross_loop; |
| 637 | - if (need_collect) { | 636 | + if (this->bodys.size() > 1) { |
| 638 | CollectTensorCrossLoop(api_calls_cross_loop); | 637 | CollectTensorCrossLoop(api_calls_cross_loop); |
| 639 | } | 638 | } |
| 640 | auto &cross_loop_map = api_calls_cross_loop[this->axis_id]; | 639 | auto &cross_loop_map = api_calls_cross_loop[this->axis_id]; |
| 640 | + static const std::vector<ApiCall *> kEmptyCrossLoopCalls; | ||
| 641 | 641 | ||
| 642 | for (const auto &body : this->bodys) { | 642 | for (const auto &body : this->bodys) { |
| 643 | if ((body.type == LoopType::CALL) && (body.call->api_call_context.scene == ApiScene::kCVFuseUBLoad || | 643 | if ((body.type == LoopType::CALL) && (body.call->api_call_context.scene == ApiScene::kCVFuseUBLoad || |
| @@ -646,29 +646,28 @@ Status Loop::GenerateBody(const Tiler &tiler, const TPipe &tpipe, std::vector<as | |||
| 646 | } | 646 | } |
| 647 | if (body.type == LoopType::LOOP) { | 647 | if (body.type == LoopType::LOOP) { |
| 648 | auto it = cross_loop_map.find(body.loop); | 648 | auto it = cross_loop_map.find(body.loop); |
| 649 | - if (it != cross_loop_map.end()) { | 649 | + const auto &cross_loop_calls = it == cross_loop_map.end() ? kEmptyCrossLoopCalls : it->second; |
| 650 | - for (auto call : it->second) { | 650 | + for (auto call : cross_loop_calls) { |
| 651 | - GE_CHK_STATUS_RET(call->AllocOutputs(tpipe, ss), "Codegen alloc outputs failed"); | 651 | + GE_CHK_STATUS_RET(call->AllocOutputs(tpipe, ss), "Codegen alloc outputs failed"); |
| 652 | - used_calls.insert(call); | 652 | + used_calls.insert(call); |
| 653 | - } | ||
| 654 | } | 653 | } |
| 655 | body.loop->compute_stage = this->compute_stage; | 654 | body.loop->compute_stage = this->compute_stage; |
| 656 | GE_CHK_STATUS_RET(body.loop->GenerateLoop(tiler, tpipe, current_axis, ss), "Generate loop for body failed"); | 655 | GE_CHK_STATUS_RET(body.loop->GenerateLoop(tiler, tpipe, current_axis, ss), "Generate loop for body failed"); |
| 657 | - if (it != cross_loop_map.end()) { | 656 | + for (auto call : cross_loop_calls) { |
| 658 | - for (auto call : it->second) { | 657 | + GE_CHK_BOOL_RET_STATUS(call->SyncOutputs(tpipe, ss), af::FAILED, "Func SyncOutputs return false"); |
| 659 | - GE_CHK_BOOL_RET_STATUS(call->SyncOutputs(tpipe, ss), af::FAILED, "Func SyncOutputs return false"); | ||
| 660 | - } | ||
| 661 | } | 658 | } |
| 662 | used_calls.clear(); | 659 | used_calls.clear(); |
| 663 | } else { | 660 | } else { |
| 664 | if (body.call->unit == af::ComputeUnit::kUnitNone || body.call->skip_api_emit) { | 661 | if (body.call->unit == af::ComputeUnit::kUnitNone || body.call->skip_api_emit) { |
| 665 | continue; | 662 | continue; |
| 666 | } | 663 | } |
| 667 | - const bool skips_ub_lifecycle = | 664 | + const auto behavior = |
| 668 | - ascgen_utils::indirect_load::GetTemplateBehavior(std::dynamic_pointer_cast<af::AscNode>(body.call->node)) | 665 | + ascgen_utils::indirect_load::GetTemplateBehavior(std::dynamic_pointer_cast<af::AscNode>(body.call->node)); |
| 669 | - .skips_ub_lifecycle; | 666 | + // SIMT IndirectLoad 的 input0/input1 不进入通用 Wait/Free；输出生命周期仍由 skips_ub_lifecycle 单独控制。 |
| 670 | - GE_CHK_BOOL_RET_STATUS(body.call->WaitInputs(tpipe, ss), af::FAILED, "Func WaitInputs return false"); | 667 | + if (!behavior.skips_input_lifecycle) { |
| 671 | - if (!IsFindInUsedCalls(body.call) && !skips_ub_lifecycle) { | 668 | + GE_CHK_BOOL_RET_STATUS(body.call->WaitInputs(tpipe, ss), af::FAILED, "Func WaitInputs return false"); |
| 669 | + } | ||
| 670 | + if (!IsFindInUsedCalls(body.call) && !behavior.skips_ub_lifecycle) { | ||
| 672 | GE_CHK_STATUS_RET(body.call->AllocOutputs(tpipe, ss), "Codegen alloc outputs failed"); | 671 | GE_CHK_STATUS_RET(body.call->AllocOutputs(tpipe, ss), "Codegen alloc outputs failed"); |
| 673 | } | 672 | } |
| 674 | std::string call; | 673 | std::string call; |
| @@ -700,13 +699,15 @@ Status Loop::GenerateBody(const Tiler &tiler, const TPipe &tpipe, std::vector<as | |||
| 700 | ss << "}" << std::endl; | 699 | ss << "}" << std::endl; |
| 701 | } | 700 | } |
| 702 | 701 | ||
| 703 | - if (!skips_ub_lifecycle) { | 702 | + // post-reduce SIMT 需要保留输出 Alloc/Sync/Free，因此不能与输入生命周期共用一个开关。 |
| 703 | + if (!behavior.skips_ub_lifecycle) { | ||
| 704 | if (!IsFindInUsedCalls(body.call)) { | 704 | if (!IsFindInUsedCalls(body.call)) { |
| 705 | GE_CHK_BOOL_RET_STATUS(body.call->SyncOutputs(tpipe, ss), af::FAILED, "Func SyncOutputs return false"); | 705 | GE_CHK_BOOL_RET_STATUS(body.call->SyncOutputs(tpipe, ss), af::FAILED, "Func SyncOutputs return false"); |
| 706 | } | 706 | } |
| 707 | - GE_CHK_BOOL_RET_STATUS(body.call->FreeInputs(tpipe, ss), af::FAILED, "Func FreeInputs return false"); | 707 | + if (!behavior.skips_input_lifecycle) { |
| 708 | - GE_CHK_BOOL_RET_STATUS(body.call->FreeUnusedOutputs(tpipe, ss), af::FAILED, | 708 | + GE_CHK_BOOL_RET_STATUS(body.call->FreeInputs(tpipe, ss), af::FAILED, "Func FreeInputs return false"); |
| 709 | - "Func FreeUnusedOutputs return false"); | 709 | + } |
| 710 | + GE_CHK_BOOL_RET_STATUS(body.call->FreeUnusedOutputs(tpipe, ss), af::FAILED, "Func FreeUnusedOutputs fails"); | ||
| 710 | } | 711 | } |
| 711 | ss << std::endl; | 712 | ss << std::endl; |
| 712 | } | 713 | } |
| @@ -34,6 +34,37 @@ bool IsValidLogicalTensorView(const LogicalTensorView &view) { | |||
| 34 | view.axis_ids.size() == view.strides.size(); | 34 | view.axis_ids.size() == view.strides.size(); |
| 35 | } | 35 | } |
| 36 | 36 | ||
| 37 | +enum class TensorDimKind : int64_t { kIllegal, kZeroStride, kRegular }; | ||
| 38 | +TensorDimKind ClassifyTensorDim(const af::Expression &size, const af::Expression &stride) { | ||
| 39 | + if (af::SymbolicUtils::StaticCheckLe(size, af::sym::kSymbolZero) == af::TriBool::kTrue || | ||
| 40 | + af::SymbolicUtils::StaticCheckLt(stride, af::sym::kSymbolZero) == af::TriBool::kTrue) { | ||
| 41 | + return TensorDimKind::kIllegal; | ||
| 42 | + } | ||
| 43 | + if (af::SymbolicUtils::StaticCheckEq(stride, af::sym::kSymbolZero) == af::TriBool::kTrue) { | ||
| 44 | + return TensorDimKind::kZeroStride; | ||
| 45 | + } | ||
| 46 | + return TensorDimKind::kRegular; | ||
| 47 | +} | ||
| 48 | + | ||
| 49 | +bool TryClassifyDynamicShapeLayout(const LogicalTensorView &logical, IndirectLoadTensorLayout &layout) { | ||
| 50 | + const bool has_dynamic_shape = std::any_of(logical.sizes.begin(), logical.sizes.end(), | ||
| 51 | + [](const af::Expression &size) { return !size.IsConstExpr(); }); | ||
| 52 | + if (!has_dynamic_shape) { | ||
| 53 | + return false; | ||
| 54 | + } | ||
| 55 | + for (size_t dim = 0UL; dim < logical.sizes.size(); ++dim) { | ||
| 56 | + const auto dim_kind = ClassifyTensorDim(logical.sizes[dim], logical.strides[dim]); | ||
| 57 | + if (dim_kind == TensorDimKind::kIllegal) { | ||
| 58 | + return false; | ||
| 59 | + } | ||
| 60 | + if (dim_kind == TensorDimKind::kZeroStride) { | ||
| 61 | + layout.physical_repeats[dim] = af::sym::kSymbolOne; | ||
| 62 | + } | ||
| 63 | + } | ||
| 64 | + layout.kind = IndirectLoadLayoutKind::kStrided; | ||
| 65 | + return true; | ||
| 66 | +} | ||
| 67 | + | ||
| 37 | bool IsValidTensorLayout(const IndirectLoadTensorLayout &layout) { | 68 | bool IsValidTensorLayout(const IndirectLoadTensorLayout &layout) { |
| 38 | return IsValidLogicalTensorView(layout) && layout.kind != IndirectLoadLayoutKind::kUnsupported && | 69 | return IsValidLogicalTensorView(layout) && layout.kind != IndirectLoadLayoutKind::kUnsupported && |
| 39 | layout.physical_repeats.size() == layout.sizes.size(); | 70 | layout.physical_repeats.size() == layout.sizes.size(); |
| @@ -68,19 +99,11 @@ TemplateBehavior GetBehavior(TemplateRole role) { | |||
| 68 | switch (role) { | 99 | switch (role) { |
| 69 | case TemplateRole::kSimdInputPre: | 100 | case TemplateRole::kSimdInputPre: |
| 70 | case TemplateRole::kSimdInputPreStridedUbPath: | 101 | case TemplateRole::kSimdInputPreStridedUbPath: |
| 71 | - behavior.excludes_tiling_group = true; | ||
| 72 | - behavior.preserves_vectorized_axis = true; | ||
| 73 | - break; | ||
| 74 | case TemplateRole::kSimdIndexPre: | 102 | case TemplateRole::kSimdIndexPre: |
| 75 | behavior.excludes_tiling_group = true; | 103 | behavior.excludes_tiling_group = true; |
| 76 | behavior.preserves_vectorized_axis = true; | 104 | behavior.preserves_vectorized_axis = true; |
| 77 | break; | 105 | break; |
| 78 | case TemplateRole::kSimtInputBoundary: | 106 | case TemplateRole::kSimtInputBoundary: |
| 79 | - behavior.skips_main_schedule_tiling = true; | ||
| 80 | - behavior.skips_api_emit = true; | ||
| 81 | - behavior.uses_direct_gm_pipeline = true; | ||
| 82 | - behavior.preserves_vectorized_axis = true; | ||
| 83 | - break; | ||
| 84 | case TemplateRole::kSimtDirectGmBoundary: | 107 | case TemplateRole::kSimtDirectGmBoundary: |
| 85 | case TemplateRole::kSimtInlineTransform: | 108 | case TemplateRole::kSimtInlineTransform: |
| 86 | behavior.skips_main_schedule_tiling = true; | 109 | behavior.skips_main_schedule_tiling = true; |
| @@ -98,10 +121,7 @@ TemplateBehavior GetBehavior(TemplateRole role) { | |||
| 98 | behavior.skips_api_emit = true; | 121 | behavior.skips_api_emit = true; |
| 99 | behavior.preserves_vectorized_axis = true; | 122 | behavior.preserves_vectorized_axis = true; |
| 100 | break; | 123 | break; |
| 101 | - case TemplateRole::kSkOp: | 124 | + default: |
| 102 | - case TemplateRole::kStridedUbPath: | ||
| 103 | - break; | ||
| 104 | - case TemplateRole::kNone: | ||
| 105 | break; | 125 | break; |
| 106 | } | 126 | } |
| 107 | return behavior; | 127 | return behavior; |
| @@ -112,58 +132,61 @@ TemplateRole GetTemplateRole(const af::AscNodePtr &node) { | |||
| 112 | return GetAnnotatedTemplateRole(node); | 132 | return GetAnnotatedTemplateRole(node); |
| 113 | } | 133 | } |
| 114 | 134 | ||
| 115 | -bool IsSimtInlineTransform(const af::AscNodePtr &node) { | ||
| 116 | - return GetTemplateRole(node) == TemplateRole::kSimtInlineTransform; | ||
| 117 | -} | ||
| 118 | - | ||
| 119 | TemplateBehavior GetTemplateBehavior(const af::AscNodePtr &node) { | 135 | TemplateBehavior GetTemplateBehavior(const af::AscNodePtr &node) { |
| 120 | const TemplateRole role = GetTemplateRole(node); | 136 | const TemplateRole role = GetTemplateRole(node); |
| 121 | TemplateBehavior behavior = GetBehavior(role); | 137 | TemplateBehavior behavior = GetBehavior(role); |
| 122 | - if (role == TemplateRole::kSimtOp && HasPostReduceConsumer(node)) { | 138 | + if (role == TemplateRole::kSimtOp && GetPostReduceConsumer(node) != nullptr) { |
| 123 | behavior = {}; | 139 | behavior = {}; |
| 124 | behavior.excludes_tiling_group = true; | 140 | behavior.excludes_tiling_group = true; |
| 125 | } | 141 | } |
| 142 | + behavior.skips_input_lifecycle = node != nullptr && af::ops::IsOps<af::ascir_op::IndirectLoad>(node) && | ||
| 143 | + ::ascir::GetTemplateIdOrDefault(*node) == ::ascir::TemplateId::kIndirectLoadSimt; | ||
| 126 | return behavior; | 144 | return behavior; |
| 127 | } | 145 | } |
| 128 | 146 | ||
| 129 | -bool HasPostReduceConsumer(const af::AscNodePtr &node) { | 147 | +af::AscNodePtr GetOnlyOutputConsumer(const af::AscNodePtr &node) { |
| 130 | - return GetPostReduceConsumer(node) != nullptr; | 148 | + if (node == nullptr || node->GetOutDataNodesSize() != 1UL) { |
| 131 | -} | 149 | + return nullptr; |
| 132 | - | ||
| 133 | -af::AscNodePtr GetPostReduceConsumer(const af::AscNodePtr &node) { | ||
| 134 | - for (af::AscNodePtr consumer = GetOnlyOutputConsumer(node); consumer != nullptr; | ||
| 135 | - consumer = GetOnlyOutputConsumer(consumer)) { | ||
| 136 | - if (consumer->attr.api.compute_type == af::ComputeType::kComputeReduce) { | ||
| 137 | - return consumer; | ||
| 138 | - } | ||
| 139 | } | 150 | } |
| 140 | - return nullptr; | 151 | + return std::dynamic_pointer_cast<af::AscNode>(*node->GetOutDataNodes().begin()); |
| 141 | } | 152 | } |
| 142 | 153 | ||
| 143 | -af::AscNodePtr GetPostReduceInputProducer(const af::AscNodePtr &node) { | 154 | +namespace { |
| 155 | +struct PostReduceChain { | ||
| 156 | + af::AscNodePtr input_producer; | ||
| 157 | + af::AscNodePtr reduce; | ||
| 158 | +}; | ||
| 159 | + | ||
| 160 | +PostReduceChain FindPostReduceChain(const af::AscNodePtr &node) { | ||
| 144 | af::AscNodePtr producer = node; | 161 | af::AscNodePtr producer = node; |
| 145 | while (producer != nullptr) { | 162 | while (producer != nullptr) { |
| 146 | const af::AscNodePtr consumer = GetOnlyOutputConsumer(producer); | 163 | const af::AscNodePtr consumer = GetOnlyOutputConsumer(producer); |
| 147 | if (consumer == nullptr) { | 164 | if (consumer == nullptr) { |
| 148 | - return nullptr; | 165 | + return {}; |
| 149 | } | 166 | } |
| 150 | if (consumer->attr.api.compute_type == af::ComputeType::kComputeReduce) { | 167 | if (consumer->attr.api.compute_type == af::ComputeType::kComputeReduce) { |
| 151 | - return producer; | 168 | + return {producer, consumer}; |
| 152 | } | 169 | } |
| 153 | producer = consumer; | 170 | producer = consumer; |
| 154 | } | 171 | } |
| 155 | - return nullptr; | 172 | + return {}; |
| 173 | +} | ||
| 174 | +} // namespace | ||
| 175 | + | ||
| 176 | +af::AscNodePtr GetPostReduceConsumer(const af::AscNodePtr &node) { | ||
| 177 | + return FindPostReduceChain(node).reduce; | ||
| 156 | } | 178 | } |
| 157 | 179 | ||
| 158 | -bool IsPostReduceInputProducer(const af::AscNodePtr &node) { | 180 | +af::AscNodePtr GetPostReduceInputProducer(const af::AscNodePtr &node) { |
| 159 | - const af::AscNodePtr consumer = GetOnlyOutputConsumer(node); | 181 | + return FindPostReduceChain(node).input_producer; |
| 160 | - return consumer != nullptr && consumer->attr.api.compute_type == af::ComputeType::kComputeReduce; | ||
| 161 | } | 182 | } |
| 162 | 183 | ||
| 163 | bool ShouldSkipTpipeTensorCollection(const af::AscNodePtr &node) { | 184 | bool ShouldSkipTpipeTensorCollection(const af::AscNodePtr &node) { |
| 164 | const TemplateBehavior behavior = GetTemplateBehavior(node); | 185 | const TemplateBehavior behavior = GetTemplateBehavior(node); |
| 186 | + const af::AscNodePtr consumer = GetOnlyOutputConsumer(node); | ||
| 165 | return (behavior.skips_api_emit || behavior.skips_ub_lifecycle) && | 187 | return (behavior.skips_api_emit || behavior.skips_ub_lifecycle) && |
| 166 | - !(IsSimtInlineTransform(node) && IsPostReduceInputProducer(node)); | 188 | + !(GetTemplateRole(node) == TemplateRole::kSimtInlineTransform && consumer != nullptr && |
| 189 | + consumer->attr.api.compute_type == af::ComputeType::kComputeReduce); | ||
| 167 | } | 190 | } |
| 168 | 191 | ||
| 169 | af::Status InheritTemplateRoleIfIL(af::AscGraph &graph, const std::string &vf_node_name, const af::AscNodePtr &src) { | 192 | af::Status InheritTemplateRoleIfIL(af::AscGraph &graph, const std::string &vf_node_name, const af::AscNodePtr &src) { |
| @@ -241,20 +264,7 @@ af::Status ClassifyIndirectLoadLayout(const LogicalTensorView &logical, Indirect | |||
| 241 | layout.kind = IndirectLoadLayoutKind::kUnsupported; | 264 | layout.kind = IndirectLoadLayoutKind::kUnsupported; |
| 242 | layout.physical_repeats = logical.sizes; | 265 | layout.physical_repeats = logical.sizes; |
| 243 | 266 | ||
| 244 | - const bool has_dynamic_shape = std::any_of(logical.sizes.begin(), logical.sizes.end(), | 267 | + if (TryClassifyDynamicShapeLayout(logical, layout)) { |
| 245 | - [](const af::Expression &size) { return !size.IsConstExpr(); }); | ||
| 246 | - if (has_dynamic_shape) { | ||
| 247 | - for (size_t dim = 0UL; dim < logical.sizes.size(); ++dim) { | ||
| 248 | - if (af::SymbolicUtils::StaticCheckLe(logical.sizes[dim], af::sym::kSymbolZero) == af::TriBool::kTrue || | ||
| 249 | - af::SymbolicUtils::StaticCheckLt(logical.strides[dim], af::sym::kSymbolZero) == af::TriBool::kTrue) { | ||
| 250 | - return af::SUCCESS; | ||
| 251 | - } | ||
| 252 | - if (af::SymbolicUtils::StaticCheckEq(logical.strides[dim], af::sym::kSymbolZero) == af::TriBool::kTrue) { | ||
| 253 | - layout.physical_repeats[dim] = af::sym::kSymbolOne; | ||
| 254 | - } | ||
| 255 | - } | ||
| 256 | - // Dynamic shapes cannot use the dense fast path. Reuse the stride-aware path so codegen derives offsets at runtime. | ||
| 257 | - layout.kind = IndirectLoadLayoutKind::kStrided; | ||
| 258 | return af::SUCCESS; | 268 | return af::SUCCESS; |
| 259 | } | 269 | } |
| 260 | af::Expression physical_span = af::sym::kSymbolOne; | 270 | af::Expression physical_span = af::sym::kSymbolOne; |
| @@ -262,12 +272,11 @@ af::Status ClassifyIndirectLoadLayout(const LogicalTensorView &logical, Indirect | |||
| 262 | bool has_physical_gap = false; | 272 | bool has_physical_gap = false; |
| 263 | for (size_t index = logical.sizes.size(); index > 0UL; --index) { | 273 | for (size_t index = logical.sizes.size(); index > 0UL; --index) { |
| 264 | const size_t dim = index - 1UL; | 274 | const size_t dim = index - 1UL; |
| 265 | - if (af::SymbolicUtils::StaticCheckLe(logical.sizes[dim], af::sym::kSymbolZero) == af::TriBool::kTrue || | 275 | + const auto dim_kind = ClassifyTensorDim(logical.sizes[dim], logical.strides[dim]); |
| 266 | - af::SymbolicUtils::StaticCheckLt(logical.strides[dim], af::sym::kSymbolZero) == af::TriBool::kTrue) { | 276 | + if (dim_kind == TensorDimKind::kIllegal) { |
| 267 | return af::SUCCESS; | 277 | return af::SUCCESS; |
| 268 | } | 278 | } |
| 269 | - const af::TriBool is_zero = af::SymbolicUtils::StaticCheckEq(logical.strides[dim], af::sym::kSymbolZero); | 279 | + if (dim_kind == TensorDimKind::kZeroStride) { |
| 270 | - if (is_zero == af::TriBool::kTrue) { | ||
| 271 | layout.physical_repeats[dim] = af::sym::kSymbolOne; | 280 | layout.physical_repeats[dim] = af::sym::kSymbolOne; |
| 272 | has_zero_stride = true; | 281 | has_zero_stride = true; |
| 273 | continue; | 282 | continue; |
| @@ -293,10 +302,6 @@ af::Status ValidateIndirectLoadOutputLayout(const LogicalTensorView &output) { | |||
| 293 | af::Expression expected_stride = af::sym::kSymbolOne; | 302 | af::Expression expected_stride = af::sym::kSymbolOne; |
| 294 | for (size_t index = output.sizes.size(); index > 0UL; --index) { | 303 | for (size_t index = output.sizes.size(); index > 0UL; --index) { |
| 295 | const size_t dim = index - 1UL; | 304 | const size_t dim = index - 1UL; |
| 296 | - GE_ASSERT_TRUE( | ||
| 297 | - af::SymbolicUtils::StaticCheckLe(output.sizes[dim], af::sym::kSymbolZero) != af::TriBool::kTrue && | ||
| 298 | - af::SymbolicUtils::StaticCheckLt(output.strides[dim], af::sym::kSymbolZero) != af::TriBool::kTrue, | ||
| 299 | - "IndirectLoad output must use a dense contiguous layout."); | ||
| 300 | if (af::SymbolicUtils::StaticCheckEq(output.sizes[dim], af::sym::kSymbolOne) == af::TriBool::kTrue) { | 305 | if (af::SymbolicUtils::StaticCheckEq(output.sizes[dim], af::sym::kSymbolOne) == af::TriBool::kTrue) { |
| 301 | continue; | 306 | continue; |
| 302 | } | 307 | } |
| @@ -326,24 +331,12 @@ af::Status GetImplementation(const af::AscNodePtr &node, Implementation &impleme | |||
| 326 | return af::SUCCESS; | 331 | return af::SUCCESS; |
| 327 | } | 332 | } |
| 328 | 333 | ||
| 329 | -bool ShouldSkipMainScheduleTiling(const af::AscNodePtr &node) { | ||
| 330 | - return GetTemplateBehavior(node).skips_main_schedule_tiling; | ||
| 331 | -} | ||
| 332 | - | ||
| 333 | -bool ShouldPreserveVectorizedAxis(const af::AscNodePtr &node) { | ||
| 334 | - return GetTemplateBehavior(node).preserves_vectorized_axis; | ||
| 335 | -} | ||
| 336 | - | ||
| 337 | bool ShouldApplyInputInnerVectorization(const af::AscNodePtr &node) { | 334 | bool ShouldApplyInputInnerVectorization(const af::AscNodePtr &node) { |
| 338 | const TemplateRole role = GetTemplateRole(node); | 335 | const TemplateRole role = GetTemplateRole(node); |
| 339 | return role == TemplateRole::kSimdInputPre || role == TemplateRole::kSimdInputPreStridedUbPath || | 336 | return role == TemplateRole::kSimdInputPre || role == TemplateRole::kSimdInputPreStridedUbPath || |
| 340 | role == TemplateRole::kSkInputBoundary; | 337 | role == TemplateRole::kSkInputBoundary; |
| 341 | } | 338 | } |
| 342 | 339 | ||
| 343 | -bool ShouldDisableRegularVectorFunc(const af::AscNodePtr &node) { | ||
| 344 | - return GetTemplateBehavior(node).uses_direct_gm_pipeline; | ||
| 345 | -} | ||
| 346 | - | ||
| 347 | af::AscNodePtr GetInputProducer(const af::AscNodePtr &node, size_t input_index) { | 340 | af::AscNodePtr GetInputProducer(const af::AscNodePtr &node, size_t input_index) { |
| 348 | auto input_anchor = node == nullptr ? nullptr : node->GetInDataAnchor(input_index); | 341 | auto input_anchor = node == nullptr ? nullptr : node->GetInDataAnchor(input_index); |
| 349 | if (input_anchor == nullptr || input_anchor->GetPeerOutAnchor() == nullptr) { | 342 | if (input_anchor == nullptr || input_anchor->GetPeerOutAnchor() == nullptr) { |
| @@ -352,33 +345,31 @@ af::AscNodePtr GetInputProducer(const af::AscNodePtr &node, size_t input_index) | |||
| 352 | return std::dynamic_pointer_cast<af::AscNode>(input_anchor->GetPeerOutAnchor()->GetOwnerNode()); | 345 | return std::dynamic_pointer_cast<af::AscNode>(input_anchor->GetPeerOutAnchor()->GetOwnerNode()); |
| 353 | } | 346 | } |
| 354 | 347 | ||
| 355 | -af::AscNodePtr GetOnlyOutputConsumer(const af::AscNodePtr &node) { | 348 | +namespace { |
| 356 | - if (node == nullptr || node->GetOutDataNodesSize() != 1UL) { | 349 | +std::vector<af::AscNodePtr> CollectIndirectLoadNodes(const af::AscGraph &graph) { |
| 357 | - return nullptr; | 350 | + std::vector<af::AscNodePtr> nodes; |
| 358 | - } | ||
| 359 | - return std::dynamic_pointer_cast<af::AscNode>(*node->GetOutDataNodes().begin()); | ||
| 360 | -} | ||
| 361 | - | ||
| 362 | -af::AscNodePtr FindIndirectLoadNode(const af::AscGraph &graph) { | ||
| 363 | for (const af::AscNodePtr &node : graph.GetAllNodes()) { | 351 | for (const af::AscNodePtr &node : graph.GetAllNodes()) { |
| 364 | if (af::ops::IsOps<af::ascir_op::IndirectLoad>(node)) { | 352 | if (af::ops::IsOps<af::ascir_op::IndirectLoad>(node)) { |
| 365 | - return node; | 353 | + nodes.push_back(node); |
| 366 | } | 354 | } |
| 367 | } | 355 | } |
| 368 | - return nullptr; | 356 | + return nodes; |
| 357 | +} | ||
| 358 | +} // namespace | ||
| 359 | + | ||
| 360 | +af::AscNodePtr FindIndirectLoadNode(const af::AscGraph &graph) { | ||
| 361 | + const auto nodes = CollectIndirectLoadNodes(graph); | ||
| 362 | + return nodes.empty() ? nullptr : nodes.front(); | ||
| 369 | } | 363 | } |
| 370 | 364 | ||
| 371 | af::Status ValidateSingleIndirectLoadNode(const af::AscGraph &graph, af::AscNodePtr &node) { | 365 | af::Status ValidateSingleIndirectLoadNode(const af::AscGraph &graph, af::AscNodePtr &node) { |
| 372 | node = nullptr; | 366 | node = nullptr; |
| 373 | - for (const af::AscNodePtr &candidate : graph.GetAllNodes()) { | 367 | + const auto nodes = CollectIndirectLoadNodes(graph); |
| 374 | - if (!af::ops::IsOps<af::ascir_op::IndirectLoad>(candidate)) { | 368 | + GE_ASSERT_TRUE(nodes.size() <= 1UL, |
| 375 | - continue; | 369 | + "[IndirectLoad] Graph[%s] contains multiple IndirectLoad nodes, first[%s], next[%s].", |
| 376 | - } | 370 | + graph.GetName().c_str(), nodes.empty() ? "<null>" : nodes[0]->GetNamePtr(), |
| 377 | - GE_ASSERT_TRUE(node == nullptr, | 371 | + nodes.size() < 2UL ? "<null>" : nodes[1]->GetNamePtr()); |
| 378 | - "[IndirectLoad] Graph[%s] contains multiple IndirectLoad nodes, first[%s], next[%s].", | 372 | + node = nodes.empty() ? nullptr : nodes.front(); |
| 379 | - graph.GetName().c_str(), node->GetNamePtr(), candidate->GetNamePtr()); | ||
| 380 | - node = candidate; | ||
| 381 | - } | ||
| 382 | if (node != nullptr) { | 373 | if (node != nullptr) { |
| 383 | GELOGD("[IndirectLoad] Graph[%s] found IndirectLoad node[%s].", graph.GetName().c_str(), node->GetNamePtr()); | 374 | GELOGD("[IndirectLoad] Graph[%s] found IndirectLoad node[%s].", graph.GetName().c_str(), node->GetNamePtr()); |
| 384 | } | 375 | } |
| @@ -33,7 +33,6 @@ enum class TemplateRole : int64_t { | |||
| 33 | kSimtInlineTransform, | 33 | kSimtInlineTransform, |
| 34 | kSimtOp, | 34 | kSimtOp, |
| 35 | kSkInputBoundary, | 35 | kSkInputBoundary, |
| 36 | - kSkOp, | ||
| 37 | kStridedUbPath, | 36 | kStridedUbPath, |
| 38 | }; | 37 | }; |
| 39 | 38 | ||
| @@ -49,6 +48,7 @@ struct TemplateBehavior { | |||
| 49 | bool skips_api_emit = false; | 48 | bool skips_api_emit = false; |
| 50 | bool uses_direct_gm_pipeline = false; | 49 | bool uses_direct_gm_pipeline = false; |
| 51 | bool skips_ub_lifecycle = false; | 50 | bool skips_ub_lifecycle = false; |
| 51 | + bool skips_input_lifecycle = false; | ||
| 52 | bool preserves_vectorized_axis = false; | 52 | bool preserves_vectorized_axis = false; |
| 53 | }; | 53 | }; |
| 54 | 54 | ||
| @@ -89,11 +89,8 @@ struct TemplateLogicalView { | |||
| 89 | 89 | ||
| 90 | TemplateBehavior GetTemplateBehavior(const af::AscNodePtr &node); | 90 | TemplateBehavior GetTemplateBehavior(const af::AscNodePtr &node); |
| 91 | TemplateRole GetTemplateRole(const af::AscNodePtr &node); | 91 | TemplateRole GetTemplateRole(const af::AscNodePtr &node); |
| 92 | -bool IsSimtInlineTransform(const af::AscNodePtr &node); | ||
| 93 | -bool HasPostReduceConsumer(const af::AscNodePtr &node); | ||
| 94 | af::AscNodePtr GetPostReduceConsumer(const af::AscNodePtr &node); | 92 | af::AscNodePtr GetPostReduceConsumer(const af::AscNodePtr &node); |
| 95 | af::AscNodePtr GetPostReduceInputProducer(const af::AscNodePtr &node); | 93 | af::AscNodePtr GetPostReduceInputProducer(const af::AscNodePtr &node); |
| 96 | -bool IsPostReduceInputProducer(const af::AscNodePtr &node); | ||
| 97 | bool ShouldSkipTpipeTensorCollection(const af::AscNodePtr &node); | 94 | bool ShouldSkipTpipeTensorCollection(const af::AscNodePtr &node); |
| 98 | af::Status InheritTemplateRoleIfIL(af::AscGraph &graph, const std::string &vf_node_name, const af::AscNodePtr &src); | 95 | af::Status InheritTemplateRoleIfIL(af::AscGraph &graph, const std::string &vf_node_name, const af::AscNodePtr &src); |
| 99 | af::Status SetTemplateRole(const af::AscNodePtr &node, TemplateRole role); | 96 | af::Status SetTemplateRole(const af::AscNodePtr &node, TemplateRole role); |
| @@ -105,10 +102,7 @@ af::Status SetImplementation(const af::AscNodePtr &node, Implementation implemen | |||
| 105 | af::Status GetImplementation(const af::AscNodePtr &node, Implementation &implementation); | 102 | af::Status GetImplementation(const af::AscNodePtr &node, Implementation &implementation); |
| 106 | af::Status ClassifyIndirectLoadLayout(const LogicalTensorView &logical, IndirectLoadTensorLayout &layout); | 103 | af::Status ClassifyIndirectLoadLayout(const LogicalTensorView &logical, IndirectLoadTensorLayout &layout); |
| 107 | af::Status ValidateIndirectLoadOutputLayout(const LogicalTensorView &output); | 104 | af::Status ValidateIndirectLoadOutputLayout(const LogicalTensorView &output); |
| 108 | -bool ShouldSkipMainScheduleTiling(const af::AscNodePtr &node); | ||
| 109 | -bool ShouldPreserveVectorizedAxis(const af::AscNodePtr &node); | ||
| 110 | bool ShouldApplyInputInnerVectorization(const af::AscNodePtr &node); | 105 | bool ShouldApplyInputInnerVectorization(const af::AscNodePtr &node); |
| 111 | -bool ShouldDisableRegularVectorFunc(const af::AscNodePtr &node); | ||
| 112 | af::AscNodePtr GetInputProducer(const af::AscNodePtr &node, size_t input_index); | 106 | af::AscNodePtr GetInputProducer(const af::AscNodePtr &node, size_t input_index); |
| 113 | af::AscNodePtr GetOnlyOutputConsumer(const af::AscNodePtr &node); | 107 | af::AscNodePtr GetOnlyOutputConsumer(const af::AscNodePtr &node); |
| 114 | af::AscNodePtr FindIndirectLoadNode(const af::AscGraph &graph); | 108 | af::AscNodePtr FindIndirectLoadNode(const af::AscGraph &graph); |
| @@ -659,8 +659,8 @@ Status Scheduler::InitIndirectLoadScheduleCase() { | |||
| 659 | } | 659 | } |
| 660 | GE_ASSERT_NOTNULL(tiling_case_.ub_tiling_y.first); | 660 | GE_ASSERT_NOTNULL(tiling_case_.ub_tiling_y.first); |
| 661 | GE_ASSERT_SUCCESS(ascgen_utils::indirect_load::GetTemplateAxes(indirect_load, indirect_load_info_.axes)); | 661 | GE_ASSERT_SUCCESS(ascgen_utils::indirect_load::GetTemplateAxes(indirect_load, indirect_load_info_.axes)); |
| 662 | - if (ascgen_utils::indirect_load::HasPostReduceConsumer(indirect_load)) { | 662 | + indirect_load_info_.reduce = ascgen_utils::indirect_load::GetPostReduceConsumer(indirect_load); |
| 663 | - indirect_load_info_.reduce = ascgen_utils::indirect_load::GetPostReduceConsumer(indirect_load); | 663 | + if (indirect_load_info_.reduce != nullptr) { |
| 664 | indirect_load_info_.reduce_input = ascgen_utils::indirect_load::GetPostReduceInputProducer(indirect_load); | 664 | indirect_load_info_.reduce_input = ascgen_utils::indirect_load::GetPostReduceInputProducer(indirect_load); |
| 665 | GE_ASSERT_NOTNULL(indirect_load_info_.reduce_input, "IndirectLoad post Reduce input producer is missing."); | 665 | GE_ASSERT_NOTNULL(indirect_load_info_.reduce_input, "IndirectLoad post Reduce input producer is missing."); |
| 666 | } | 666 | } |
| @@ -682,7 +682,7 @@ Status Scheduler::ApplyIndirectLoadNodeAxes(const af::AscNodePtr &node, bool &sk | |||
| 682 | if (is_index_pre) { | 682 | if (is_index_pre) { |
| 683 | GE_ASSERT_SUCCESS(ApplyInputInnerVectorizedAxis(graph_, node, indirect_load_info_.axes.index_inner_axis)); | 683 | GE_ASSERT_SUCCESS(ApplyInputInnerVectorizedAxis(graph_, node, indirect_load_info_.axes.index_inner_axis)); |
| 684 | } | 684 | } |
| 685 | - skip_main_tiling = ascgen_utils::indirect_load::ShouldSkipMainScheduleTiling(node); | 685 | + skip_main_tiling = ascgen_utils::indirect_load::GetTemplateBehavior(node).skips_main_schedule_tiling; |
| 686 | if (skip_main_tiling) { | 686 | if (skip_main_tiling) { |
| 687 | return af::SUCCESS; | 687 | return af::SUCCESS; |
| 688 | } | 688 | } |
| @@ -759,7 +759,8 @@ Status Scheduler::TileSplit() { | |||
| 759 | // 非reduce场景应该将向量化轴调整为tensor中的相对顺序, 带reduce场景由于tiling策略已经做了特别的reorder,需要跳过 | 759 | // 非reduce场景应该将向量化轴调整为tensor中的相对顺序, 带reduce场景由于tiling策略已经做了特别的reorder,需要跳过 |
| 760 | // tiling策略暂时无法支持具有reduce和transpose融合的场景 | 760 | // tiling策略暂时无法支持具有reduce和transpose融合的场景 |
| 761 | for (auto &output : node->outputs()) { | 761 | for (auto &output : node->outputs()) { |
| 762 | - if (indirect_load_info_.active && ascgen_utils::indirect_load::ShouldPreserveVectorizedAxis(node)) { | 762 | + if (indirect_load_info_.active && |
| 763 | + ascgen_utils::indirect_load::GetTemplateBehavior(node).preserves_vectorized_axis) { | ||
| 763 | continue; | 764 | continue; |
| 764 | } | 765 | } |
| 765 | output->attr.vectorized_axis = node_vectorized_axes; | 766 | output->attr.vectorized_axis = node_vectorized_axes; |
| @@ -817,7 +818,8 @@ Status Scheduler::ApplyBlockSplit(const std::vector<ascir::AxisId> &new_sched_ax | |||
| 817 | bool is_reduce_after = false; | 818 | bool is_reduce_after = false; |
| 818 | for (auto node : graph_.GetAllNodes()) { | 819 | for (auto node : graph_.GetAllNodes()) { |
| 819 | if (ScheduleUtils::IsBuffer(node) || | 820 | if (ScheduleUtils::IsBuffer(node) || |
| 820 | - (indirect_load_info_.active && ascgen_utils::indirect_load::ShouldSkipMainScheduleTiling(node))) { | 821 | + (indirect_load_info_.active && |
| 822 | + ascgen_utils::indirect_load::GetTemplateBehavior(node).skips_main_schedule_tiling)) { | ||
| 821 | continue; | 823 | continue; |
| 822 | } | 824 | } |
| 823 | if ((!is_reduce_after) && ScheduleUtils::IsReduce(node)) { | 825 | if ((!is_reduce_after) && ScheduleUtils::IsReduce(node)) { |
| @@ -105,21 +105,11 @@ bool IsSupportedBroadcastPath(const NodePath &path, size_t broadcast_index, asci | |||
| 105 | return false; | 105 | return false; |
| 106 | } | 106 | } |
| 107 | const af::AscNodePtr &broadcast = path[broadcast_index]; | 107 | const af::AscNodePtr &broadcast = path[broadcast_index]; |
| 108 | - const auto broadcast_source = ascgen_utils::indirect_load::GetInputProducer(broadcast, 0UL); | ||
| 109 | if (!IsSingleConsumerWithoutControlEdge(broadcast)) { | 108 | if (!IsSingleConsumerWithoutControlEdge(broadcast)) { |
| 110 | GELOGI("[IndirectLoad] Reject candidate[%d]: Broadcast path node[%s] is not safely foldable.", | 109 | GELOGI("[IndirectLoad] Reject candidate[%d]: Broadcast path node[%s] is not safely foldable.", |
| 111 | static_cast<int32_t>(template_id), broadcast->GetNamePtr()); | 110 | static_cast<int32_t>(template_id), broadcast->GetNamePtr()); |
| 112 | return false; | 111 | return false; |
| 113 | } | 112 | } |
| 114 | - if (template_id == ascir::TemplateId::kIndirectLoadSimd) { | ||
| 115 | - if (!IsSingleConsumerWithoutControlEdge(broadcast_source)) { | ||
| 116 | - GELOGI("[IndirectLoad] Reject candidate[%d]: Broadcast source node[%s] is not safely foldable.", | ||
| 117 | - static_cast<int32_t>(template_id), | ||
| 118 | - broadcast_source == nullptr ? "<null>" : broadcast_source->GetNamePtr()); | ||
| 119 | - return false; | ||
| 120 | - } | ||
| 121 | - return true; | ||
| 122 | - } | ||
| 123 | // CollectInputPaths 只有 SK 会继续回溯 Broadcast 前的单输入链。 | 113 | // CollectInputPaths 只有 SK 会继续回溯 Broadcast 前的单输入链。 |
| 124 | for (size_t i = 0UL; i < broadcast_index; ++i) { | 114 | for (size_t i = 0UL; i < broadcast_index; ++i) { |
| 125 | const af::AscNodePtr &element = path[i]; | 115 | const af::AscNodePtr &element = path[i]; |
| @@ -207,50 +197,6 @@ af::Status BuildBroadcastLogicalView(const af::AscTensorAttr &logical_attr, cons | |||
| 207 | return af::SUCCESS; | 197 | return af::SUCCESS; |
| 208 | } | 198 | } |
| 209 | 199 | ||
| 210 | -bool HasMixedInnerBroadcast(const ascgen_utils::indirect_load::LogicalTensorView &logical_view, | ||
| 211 | - const af::AscTensorAttr &physical_attr, size_t axis_index) { | ||
| 212 | - bool has_broadcast_axis = false; | ||
| 213 | - bool has_regular_axis = false; | ||
| 214 | - for (size_t dim = axis_index; dim < logical_view.sizes.size(); ++dim) { | ||
| 215 | - const bool is_broadcast = | ||
| 216 | - af::SymbolicUtils::StaticCheckEq(physical_attr.repeats[dim], af::sym::kSymbolOne) == af::TriBool::kTrue && | ||
| 217 | - af::SymbolicUtils::StaticCheckEq(logical_view.sizes[dim], af::sym::kSymbolOne) != af::TriBool::kTrue; | ||
| 218 | - has_broadcast_axis = has_broadcast_axis || is_broadcast; | ||
| 219 | - has_regular_axis = has_regular_axis || !is_broadcast; | ||
| 220 | - } | ||
| 221 | - return has_broadcast_axis && has_regular_axis; | ||
| 222 | -} | ||
| 223 | - | ||
| 224 | -bool HasDirectOuterBroadcast(const ascgen_utils::indirect_load::LogicalTensorView &logical_view, | ||
| 225 | - const af::AscTensorAttr &physical_attr, size_t axis_index) { | ||
| 226 | - for (size_t dim = 0UL; dim < axis_index; ++dim) { | ||
| 227 | - if (af::SymbolicUtils::StaticCheckEq(physical_attr.repeats[dim], af::sym::kSymbolOne) == af::TriBool::kTrue && | ||
| 228 | - af::SymbolicUtils::StaticCheckEq(logical_view.sizes[dim], af::sym::kSymbolOne) != af::TriBool::kTrue) { | ||
| 229 | - return !HasMixedInnerBroadcast(logical_view, physical_attr, axis_index); | ||
| 230 | - } | ||
| 231 | - } | ||
| 232 | - return false; | ||
| 233 | -} | ||
| 234 | - | ||
| 235 | -bool IsSupportedDirectOuterBroadcast(const af::AscNodePtr &broadcast, | ||
| 236 | - const ascgen_utils::indirect_load::LogicalTensorView &logical_view, | ||
| 237 | - const af::AscTensorAttr &physical_attr, size_t axis_index, | ||
| 238 | - ascir::TemplateId template_id) { | ||
| 239 | - if (template_id != ascir::TemplateId::kIndirectLoadSimd) { | ||
| 240 | - return true; | ||
| 241 | - } | ||
| 242 | - const auto broadcast_producer = ascgen_utils::indirect_load::GetInputProducer(broadcast, 0UL); | ||
| 243 | - if (broadcast_producer == nullptr || broadcast_producer->inputs.Size() != 1UL) { | ||
| 244 | - return true; | ||
| 245 | - } | ||
| 246 | - if (!HasDirectOuterBroadcast(logical_view, physical_attr, axis_index)) { | ||
| 247 | - return true; | ||
| 248 | - } | ||
| 249 | - GELOGI("[IndirectLoad] Reject candidate[%d]: direct outer Broadcast in the input/index path.", | ||
| 250 | - static_cast<int32_t>(template_id)); | ||
| 251 | - return false; | ||
| 252 | -} | ||
| 253 | - | ||
| 254 | af::Status ApplyZeroStrideCompactView(const NodePath &path, | 200 | af::Status ApplyZeroStrideCompactView(const NodePath &path, |
| 255 | const ascgen_utils::indirect_load::IndirectLoadTensorLayout &layout) { | 201 | const ascgen_utils::indirect_load::IndirectLoadTensorLayout &layout) { |
| 256 | GE_ASSERT_TRUE( | 202 | GE_ASSERT_TRUE( |
| @@ -570,9 +516,8 @@ bool IsSkTemplateCandidateLegal(const af::AscNodePtr &indirect_load) { | |||
| 570 | 516 | ||
| 571 | af::Status ValidateIndirectLoadNode(const af::AscNodePtr &indirect_load) { | 517 | af::Status ValidateIndirectLoadNode(const af::AscNodePtr &indirect_load) { |
| 572 | const auto outputs = indirect_load->outputs(); | 518 | const auto outputs = indirect_load->outputs(); |
| 573 | - GE_ASSERT_TRUE(outputs.size() == kIndirectLoadOutputCount, | 519 | + GE_ASSERT_TRUE(outputs.size() == kIndirectLoadOutputCount, "Invalid IndirectLoad output number:%zu, node[%s].", |
| 574 | - "IndirectLoad node[%s] output count is invalid, actual:%zu.", indirect_load->GetNamePtr(), | 520 | + outputs.size(), indirect_load->GetNamePtr()); |
| 575 | - outputs.size()); | ||
| 576 | const auto output = outputs[0]; | 521 | const auto output = outputs[0]; |
| 577 | GE_ASSERT_NOTNULL(output, "IndirectLoad output tensor is null."); | 522 | GE_ASSERT_NOTNULL(output, "IndirectLoad output tensor is null."); |
| 578 | const size_t output_rank = output->attr.axis.size(); | 523 | const size_t output_rank = output->attr.axis.size(); |
| @@ -583,8 +528,8 @@ af::Status ValidateIndirectLoadNode(const af::AscNodePtr &indirect_load) { | |||
| 583 | GE_ASSERT_TRUE(axis_index != kIndirectLoadInvalidAxisIndex, "IndirectLoad axis index of node[%s] is invalid.", | 528 | GE_ASSERT_TRUE(axis_index != kIndirectLoadInvalidAxisIndex, "IndirectLoad axis index of node[%s] is invalid.", |
| 584 | indirect_load->GetNamePtr()); | 529 | indirect_load->GetNamePtr()); |
| 585 | const auto inputs = indirect_load->inputs(); | 530 | const auto inputs = indirect_load->inputs(); |
| 586 | - GE_ASSERT_TRUE(inputs.size() == kIndirectLoadInputCount, "IndirectLoad node[%s] input count is invalid, actual:%zu.", | 531 | + GE_ASSERT_TRUE(inputs.size() == kIndirectLoadInputCount, "Invalid IndirectLoad input number:%zu, node[%s].", |
| 587 | - indirect_load->GetNamePtr(), inputs.size()); | 532 | + inputs.size(), indirect_load->GetNamePtr()); |
| 588 | const auto input = inputs[ascgen_utils::indirect_load::kInputTensorIndex]; | 533 | const auto input = inputs[ascgen_utils::indirect_load::kInputTensorIndex]; |
| 589 | const auto index = inputs[ascgen_utils::indirect_load::kIndexTensorIndex]; | 534 | const auto index = inputs[ascgen_utils::indirect_load::kIndexTensorIndex]; |
| 590 | GE_ASSERT_NOTNULL(input, "IndirectLoad input tensor is null."); | 535 | GE_ASSERT_NOTNULL(input, "IndirectLoad input tensor is null."); |
| @@ -1031,6 +976,9 @@ af::Status MoveInputPreNode(const af::AscNodePtr &node, const af::AscNodePtr &in | |||
| 1031 | 976 | ||
| 1032 | af::Status RewriteInputPreNodes(af::AscGraph &graph, const af::AscNodePtr &indirect_load, | 977 | af::Status RewriteInputPreNodes(af::AscGraph &graph, const af::AscNodePtr &indirect_load, |
| 1033 | ascir::TemplateId template_id) { | 978 | ascir::TemplateId template_id) { |
| 979 | + if (template_id == ascir::TemplateId::kIndirectLoadSK) { | ||
| 980 | + return af::SUCCESS; | ||
| 981 | + } | ||
| 1034 | if (HasControlEdge(indirect_load)) { | 982 | if (HasControlEdge(indirect_load)) { |
| 1035 | GELOGI("[IndirectLoad] Skip moving input-pre nodes for node[%s].", indirect_load->GetNamePtr()); | 983 | GELOGI("[IndirectLoad] Skip moving input-pre nodes for node[%s].", indirect_load->GetNamePtr()); |
| 1036 | return af::SUCCESS; | 984 | return af::SUCCESS; |
| @@ -1170,27 +1118,23 @@ af::Status AnalyzeInputPath(const af::AscNodePtr &indirect_load, size_t input_id | |||
| 1170 | } else { | 1118 | } else { |
| 1171 | const size_t broadcast_index = static_cast<size_t>(plan.path_broadcast_index); | 1119 | const size_t broadcast_index = static_cast<size_t>(plan.path_broadcast_index); |
| 1172 | const af::AscNodePtr &broadcast = plan.path[broadcast_index]; | 1120 | const af::AscNodePtr &broadcast = plan.path[broadcast_index]; |
| 1173 | - // 取源物理属性前保留源存在性检查；SIMD/SK 的路径折叠安全性在 FoldBroadcastPath 阶段统一校验。 | 1121 | + // 取源物理属性前保留源存在性检查。 |
| 1174 | if (ascgen_utils::indirect_load::GetInputProducer(broadcast, 0UL) == nullptr) { | 1122 | if (ascgen_utils::indirect_load::GetInputProducer(broadcast, 0UL) == nullptr) { |
| 1175 | GELOGI("[IndirectLoad] Reject candidate[%d]: Broadcast node[%s] source is invalid.", | 1123 | GELOGI("[IndirectLoad] Reject candidate[%d]: Broadcast node[%s] source is invalid.", |
| 1176 | static_cast<int32_t>(template_id), broadcast->GetNamePtr()); | 1124 | static_cast<int32_t>(template_id), broadcast->GetNamePtr()); |
| 1177 | is_path_supported = false; | 1125 | is_path_supported = false; |
| 1178 | return af::SUCCESS; | 1126 | return af::SUCCESS; |
| 1179 | } | 1127 | } |
| 1180 | - // 折叠源物理属性：以广播源的物理属性重写逻辑视图的 strides，源形状为 1 而逻辑视图非 1 的维度置 stride 0。 | 1128 | + // 基于广播源物理属性构造执行视图：源形状为 1 而逻辑视图非 1 的维度置 stride 0。 |
| 1181 | af::AscTensorAttr physical_attr; | 1129 | af::AscTensorAttr physical_attr; |
| 1182 | GE_ASSERT_SUCCESS(GetBroadcastPhysicalAttr(broadcast, template_id, physical_attr)); | 1130 | GE_ASSERT_SUCCESS(GetBroadcastPhysicalAttr(broadcast, template_id, physical_attr)); |
| 1183 | GE_ASSERT_SUCCESS(BuildBroadcastLogicalView(logical_attr, physical_attr, template_id, view)); | 1131 | GE_ASSERT_SUCCESS(BuildBroadcastLogicalView(logical_attr, physical_attr, template_id, view)); |
| 1184 | - if (!IsSupportedDirectOuterBroadcast(broadcast, view, physical_attr, axis_index, template_id)) { | ||
| 1185 | - is_path_supported = false; | ||
| 1186 | - return af::SUCCESS; | ||
| 1187 | - } | ||
| 1188 | } | 1132 | } |
| 1189 | 1133 | ||
| 1190 | GE_ASSERT_SUCCESS(ascgen_utils::indirect_load::ClassifyIndirectLoadLayout(view, plan.layout)); | 1134 | GE_ASSERT_SUCCESS(ascgen_utils::indirect_load::ClassifyIndirectLoadLayout(view, plan.layout)); |
| 1191 | if (plan.layout.kind == ascgen_utils::indirect_load::IndirectLoadLayoutKind::kUnsupported) { | 1135 | if (plan.layout.kind == ascgen_utils::indirect_load::IndirectLoadLayoutKind::kUnsupported) { |
| 1192 | GELOGI("[IndirectLoad] Reject candidate[%d]: input path layout%s is unsupported.", | 1136 | GELOGI("[IndirectLoad] Reject candidate[%d]: input path layout%s is unsupported.", |
| 1193 | - static_cast<int32_t>(template_id), has_broadcast ? " after Broadcast folding" : ""); | 1137 | + static_cast<int32_t>(template_id), has_broadcast ? " with Broadcast source view" : ""); |
| 1194 | is_path_supported = false; | 1138 | is_path_supported = false; |
| 1195 | } | 1139 | } |
| 1196 | plan.simd_index_uses_output_inner_axis = template_id == ascir::TemplateId::kIndirectLoadSimd && | 1140 | plan.simd_index_uses_output_inner_axis = template_id == ascir::TemplateId::kIndirectLoadSimd && |
| @@ -1198,22 +1142,27 @@ af::Status AnalyzeInputPath(const af::AscNodePtr &indirect_load, size_t input_id | |||
| 1198 | return af::SUCCESS; | 1142 | return af::SUCCESS; |
| 1199 | } | 1143 | } |
| 1200 | 1144 | ||
| 1201 | -af::Status RewriteInputPathsForTemplate(ascir::TemplateId template_id, PhysicalViewPreparation &preparation, | 1145 | +af::Status RewriteSkInputPaths(PhysicalViewPreparation &preparation, RewrittenGraphAnalysis &analysis, |
| 1202 | - RewrittenGraphAnalysis &analysis, bool &is_candidate_legal) { | 1146 | + bool &is_candidate_legal) { |
| 1203 | - if (template_id == ascir::TemplateId::kIndirectLoadSimt) { | ||
| 1204 | - // SIMT 先完成 input-pre 搬移，再统一处理当前直连的 input Broadcast。 | ||
| 1205 | - return af::SUCCESS; | ||
| 1206 | - } | ||
| 1207 | for (InputViewPlan *const plan : {&preparation.input, &preparation.index}) { | 1147 | for (InputViewPlan *const plan : {&preparation.input, &preparation.index}) { |
| 1208 | - GE_ASSERT_SUCCESS(FoldBroadcastPath(*plan, template_id, is_candidate_legal)); | 1148 | + GE_ASSERT_SUCCESS(FoldBroadcastPath(*plan, ascir::TemplateId::kIndirectLoadSK, is_candidate_legal)); |
| 1209 | if (!is_candidate_legal) { | 1149 | if (!is_candidate_legal) { |
| 1210 | return af::SUCCESS; | 1150 | return af::SUCCESS; |
| 1211 | } | 1151 | } |
| 1212 | } | 1152 | } |
| 1153 | + analysis.input_path = std::move(preparation.input.path); | ||
| 1154 | + analysis.index_path = std::move(preparation.index.path); | ||
| 1155 | + return af::SUCCESS; | ||
| 1156 | +} | ||
| 1157 | + | ||
| 1158 | +af::Status RewriteBroadcastPaths(const af::AscNodePtr &indirect_load, ascir::TemplateId template_id, | ||
| 1159 | + PhysicalViewPreparation &preparation, RewrittenGraphAnalysis &analysis, | ||
| 1160 | + bool &is_candidate_legal) { | ||
| 1213 | if (template_id == ascir::TemplateId::kIndirectLoadSK) { | 1161 | if (template_id == ascir::TemplateId::kIndirectLoadSK) { |
| 1214 | - // SK 需要保留完整路径，供调用方统一写回物理视图。 | 1162 | + return RewriteSkInputPaths(preparation, analysis, is_candidate_legal); |
| 1215 | - analysis.input_path = std::move(preparation.input.path); | 1163 | + } |
| 1216 | - analysis.index_path = std::move(preparation.index.path); | 1164 | + if (template_id == ascir::TemplateId::kIndirectLoadSimt) { |
| 1165 | + return RewriteSimtInputBroadcast(indirect_load, preparation.input, is_candidate_legal); | ||
| 1217 | } | 1166 | } |
| 1218 | return af::SUCCESS; | 1167 | return af::SUCCESS; |
| 1219 | } | 1168 | } |
| @@ -1314,28 +1263,18 @@ af::Status AnalyzeRewrittenGraph(af::AscGraph &graph, const af::AscNodePtr &indi | |||
| 1314 | NeedsAlignedUbWindow(preparation.logical_view.input, axis_index); | 1263 | NeedsAlignedUbWindow(preparation.logical_view.input, axis_index); |
| 1315 | analysis.simd_index_uses_output_inner_axis = preparation.index.simd_index_uses_output_inner_axis; | 1264 | analysis.simd_index_uses_output_inner_axis = preparation.index.simd_index_uses_output_inner_axis; |
| 1316 | 1265 | ||
| 1317 | - // 改写阶段：全部拓扑改动一次完成（Broadcast 删除/折叠、input-pre 搬移） | 1266 | + // SK 跳过 input-pre 搬移；SIMD/SIMT 将 input-pre 单目元素链搬到 IndirectLoad 之后。 |
| 1318 | - GE_ASSERT_SUCCESS(RewriteInputPathsForTemplate(template_id, preparation, analysis, is_candidate_legal)); | 1267 | + GE_ASSERT_SUCCESS(RewriteInputPreNodes(graph, indirect_load, template_id)); |
| 1268 | + GE_ASSERT_SUCCESS(RewriteBroadcastPaths(indirect_load, template_id, preparation, analysis, is_candidate_legal)); | ||
| 1319 | if (!is_candidate_legal) { | 1269 | if (!is_candidate_legal) { |
| 1320 | return af::SUCCESS; | 1270 | return af::SUCCESS; |
| 1321 | } | 1271 | } |
| 1322 | - if (template_id == ascir::TemplateId::kIndirectLoadSK) { | ||
| 1323 | - // SK 不参与 region 收集，直接处理完整路径并结束。 | ||
| 1324 | - GE_ASSERT_SUCCESS(ApplyTemplatePathLayouts(indirect_load, template_id, preparation, analysis)); | ||
| 1325 | - return af::SUCCESS; | ||
| 1326 | - } | ||
| 1327 | - // 将 input-pre 单目元素链搬到 IndirectLoad 之后 | ||
| 1328 | - GE_ASSERT_SUCCESS(RewriteInputPreNodes(graph, indirect_load, template_id)); | ||
| 1329 | - if (template_id == ascir::TemplateId::kIndirectLoadSimt) { | ||
| 1330 | - GE_ASSERT_SUCCESS(RewriteSimtInputBroadcast(indirect_load, preparation.input, is_candidate_legal)); | ||
| 1331 | - if (!is_candidate_legal) { | ||
| 1332 | - return af::SUCCESS; | ||
| 1333 | - } | ||
| 1334 | - } | ||
| 1335 | 1272 | ||
| 1336 | - // 收集阶段：一次遍历收集全部状态（改写定稿后无需重收） | 1273 | + if (template_id != ascir::TemplateId::kIndirectLoadSK) { |
| 1337 | - GE_ASSERT_SUCCESS(CollectRewrittenBoundaries(indirect_load, analysis)); | 1274 | + // 收集阶段：一次遍历收集全部状态（改写定稿后无需重收） |
| 1338 | - GE_ASSERT_SUCCESS(CollectRewrittenRegion(graph, indirect_load, template_id, analysis)); | 1275 | + GE_ASSERT_SUCCESS(CollectRewrittenBoundaries(indirect_load, analysis)); |
| 1276 | + GE_ASSERT_SUCCESS(CollectRewrittenRegion(graph, indirect_load, template_id, analysis)); | ||
| 1277 | + } | ||
| 1339 | 1278 | ||
| 1340 | // 路径布局处理：紧凑零 stride 写回物理视图，需对齐的 strided 路径标注 UB role。 | 1279 | // 路径布局处理：紧凑零 stride 写回物理视图，需对齐的 strided 路径标注 UB role。 |
| 1341 | GE_ASSERT_SUCCESS(ApplyTemplatePathLayouts(indirect_load, template_id, preparation, analysis)); | 1280 | GE_ASSERT_SUCCESS(ApplyTemplatePathLayouts(indirect_load, template_id, preparation, analysis)); |
| @@ -1463,8 +1402,6 @@ af::Status ApplySkGraphPass(af::AscGraph &graph, const af::AscNodePtr &indirect_ | |||
| 1463 | ascir::AxisId input_inner_axis = af::kIdNone; | 1402 | ascir::AxisId input_inner_axis = af::kIdNone; |
| 1464 | GE_ASSERT_SUCCESS(BuildSkInputInnerAxis(graph, indirect_load, axis_index, input_inner_axis)); | 1403 | GE_ASSERT_SUCCESS(BuildSkInputInnerAxis(graph, indirect_load, axis_index, input_inner_axis)); |
| 1465 | GE_ASSERT_SUCCESS(NormalizeAxesForTemplate(graph, indirect_load, axis_index, input_inner_axis, af::kIdNone)); | 1404 | GE_ASSERT_SUCCESS(NormalizeAxesForTemplate(graph, indirect_load, axis_index, input_inner_axis, af::kIdNone)); |
| 1466 | - GE_ASSERT_SUCCESS( | ||
| 1467 | - ascgen_utils::indirect_load::SetTemplateRole(indirect_load, ascgen_utils::indirect_load::TemplateRole::kSkOp)); | ||
| 1468 | const auto input_boundary = ascgen_utils::indirect_load::GetInputProducer(indirect_load, 0UL); | 1405 | const auto input_boundary = ascgen_utils::indirect_load::GetInputProducer(indirect_load, 0UL); |
| 1469 | GE_ASSERT_TRUE(input_boundary != nullptr && af::ops::IsOps<af::ascir_op::Load>(input_boundary), | 1406 | GE_ASSERT_TRUE(input_boundary != nullptr && af::ops::IsOps<af::ascir_op::Load>(input_boundary), |
| 1470 | "IndirectLoad SK input boundary must be a Load node, node[%s].", indirect_load->GetNamePtr()); | 1407 | "IndirectLoad SK input boundary must be a Load node, node[%s].", indirect_load->GetNamePtr()); |
| @@ -296,7 +296,7 @@ Status ReducePartitionCaseGenerator::GeneratorTask(ascir::HintGraph &optimize_gr | |||
| 296 | const OptimizerOptions &options) { | 296 | const OptimizerOptions &options) { |
| 297 | (void)options; | 297 | (void)options; |
| 298 | const af::AscNodePtr indirect_load = ascgen_utils::indirect_load::FindIndirectLoadNode(optimize_graph); | 298 | const af::AscNodePtr indirect_load = ascgen_utils::indirect_load::FindIndirectLoadNode(optimize_graph); |
| 299 | - if (indirect_load != nullptr && ascgen_utils::indirect_load::HasPostReduceConsumer(indirect_load)) { | 299 | + if (indirect_load != nullptr && ascgen_utils::indirect_load::GetPostReduceConsumer(indirect_load) != nullptr) { |
| 300 | GELOGI("Graph %s has indirect load with post reduce consumer, skip reduce task generation.", | 300 | GELOGI("Graph %s has indirect load with post reduce consumer, skip reduce task generation.", |
| 301 | optimize_graph.GetName().c_str()); | 301 | optimize_graph.GetName().c_str()); |
| 302 | return ge::GRAPH_SUCCESS; | 302 | return ge::GRAPH_SUCCESS; |
| @@ -891,9 +891,6 @@ TEST(IndirectLoadScheduleCaseGeneratorTest, SimtSetsDcacheAndUsesUnifiedVectoriz | |||
| 891 | EXPECT_TRUE(behavior.skips_ub_lifecycle); | 891 | EXPECT_TRUE(behavior.skips_ub_lifecycle); |
| 892 | EXPECT_TRUE(behavior.preserves_vectorized_axis); | 892 | EXPECT_TRUE(behavior.preserves_vectorized_axis); |
| 893 | EXPECT_FALSE(ascgen_utils::indirect_load::ShouldApplyInputInnerVectorization(indirect_load)); | 893 | EXPECT_FALSE(ascgen_utils::indirect_load::ShouldApplyInputInnerVectorization(indirect_load)); |
| 894 | - EXPECT_FALSE(ascgen_utils::indirect_load::ShouldSkipMainScheduleTiling(indirect_load)); | ||
| 895 | - EXPECT_TRUE(ascgen_utils::indirect_load::ShouldPreserveVectorizedAxis(indirect_load)); | ||
| 896 | - EXPECT_TRUE(ascgen_utils::indirect_load::ShouldDisableRegularVectorFunc(indirect_load)); | ||
| 897 | } | 894 | } |
| 898 | 895 | ||
| 899 | TEST(IndirectLoadScheduleCaseGeneratorTest, ClassifiesDenseLayout) { | 896 | TEST(IndirectLoadScheduleCaseGeneratorTest, ClassifiesDenseLayout) { |
| @@ -993,24 +990,28 @@ TEST(IndirectLoadScheduleCaseGeneratorTest, KeepsBroadcastElementPathForGeneralI | |||
| 993 | } | 990 | } |
| 994 | } | 991 | } |
| 995 | 992 | ||
| 996 | -TEST(IndirectLoadScheduleCaseGeneratorTest, RejectsOuterInputBroadcastForSimd) { | 993 | +TEST(IndirectLoadScheduleCaseGeneratorTest, AcceptsOuterInputBroadcastForSimd) { |
| 997 | auto graph = BuildIndirectLoadBroadcastGraph(false, false, 2L); | 994 | auto graph = BuildIndirectLoadBroadcastGraph(false, false, 2L); |
| 998 | optimize::IndirectLoadScheduleCaseGenerator generator; | 995 | optimize::IndirectLoadScheduleCaseGenerator generator; |
| 999 | std::vector<af::AscGraph> graphs; | 996 | std::vector<af::AscGraph> graphs; |
| 1000 | std::vector<std::string> score_functions; | 997 | std::vector<std::string> score_functions; |
| 1001 | ASSERT_EQ(generator.Generate(graph, graphs, score_functions), af::SUCCESS); | 998 | ASSERT_EQ(generator.Generate(graph, graphs, score_functions), af::SUCCESS); |
| 1002 | 999 | ||
| 1003 | - EXPECT_EQ(FindGeneratedGraphByTemplate(graphs, ascir::TemplateId::kIndirectLoadSimd), graphs.end()); | 1000 | + const auto simd = FindGeneratedGraphByTemplate(graphs, ascir::TemplateId::kIndirectLoadSimd); |
| 1001 | + ASSERT_NE(simd, graphs.end()); | ||
| 1002 | + EXPECT_NE(simd->FindNode("input_broadcast"), nullptr); | ||
| 1004 | } | 1003 | } |
| 1005 | 1004 | ||
| 1006 | -TEST(IndirectLoadScheduleCaseGeneratorTest, RejectsDirectOuterIndexBroadcastForSimd) { | 1005 | +TEST(IndirectLoadScheduleCaseGeneratorTest, AcceptsOuterIndexBroadcastForSimd) { |
| 1007 | auto graph = BuildIndirectLoadBroadcastGraph(false, true, 2L); | 1006 | auto graph = BuildIndirectLoadBroadcastGraph(false, true, 2L); |
| 1008 | optimize::IndirectLoadScheduleCaseGenerator generator; | 1007 | optimize::IndirectLoadScheduleCaseGenerator generator; |
| 1009 | std::vector<af::AscGraph> graphs; | 1008 | std::vector<af::AscGraph> graphs; |
| 1010 | std::vector<std::string> score_functions; | 1009 | std::vector<std::string> score_functions; |
| 1011 | ASSERT_EQ(generator.Generate(graph, graphs, score_functions), af::SUCCESS); | 1010 | ASSERT_EQ(generator.Generate(graph, graphs, score_functions), af::SUCCESS); |
| 1012 | 1011 | ||
| 1013 | - EXPECT_EQ(FindGeneratedGraphByTemplate(graphs, ascir::TemplateId::kIndirectLoadSimd), graphs.end()); | 1012 | + const auto simd = FindGeneratedGraphByTemplate(graphs, ascir::TemplateId::kIndirectLoadSimd); |
| 1013 | + ASSERT_NE(simd, graphs.end()); | ||
| 1014 | + EXPECT_NE(simd->FindNode("index_outer_broadcast"), nullptr); | ||
| 1014 | } | 1015 | } |
| 1015 | 1016 | ||
| 1016 | TEST(IndirectLoadScheduleCaseGeneratorTest, SimtRewritesInputBroadcastAndSetsTemplateMetadata) { | 1017 | TEST(IndirectLoadScheduleCaseGeneratorTest, SimtRewritesInputBroadcastAndSetsTemplateMetadata) { |
| @@ -1053,10 +1054,15 @@ TEST(IndirectLoadScheduleCaseGeneratorTest, BroadcastDirectPathUsesPhysicalViewF | |||
| 1053 | EXPECT_EQ(graphs.size(), 4UL); | 1054 | EXPECT_EQ(graphs.size(), 4UL); |
| 1054 | ASSERT_EQ(score_functions.size(), graphs.size()); | 1055 | ASSERT_EQ(score_functions.size(), graphs.size()); |
| 1055 | for (auto &candidate : graphs) { | 1056 | for (auto &candidate : graphs) { |
| 1056 | - EXPECT_EQ(candidate.FindNode("input_broadcast"), nullptr); | ||
| 1057 | - EXPECT_EQ(candidate.FindNode("broadcast_input_abs"), nullptr); | ||
| 1058 | const auto indirect_load = candidate.FindNode("indirect_load"); | 1057 | const auto indirect_load = candidate.FindNode("indirect_load"); |
| 1059 | ASSERT_NE(indirect_load, nullptr); | 1058 | ASSERT_NE(indirect_load, nullptr); |
| 1059 | + const auto template_id = ascir::GetTemplateIdOrDefault(*indirect_load); | ||
| 1060 | + if (template_id == ascir::TemplateId::kIndirectLoadSimd) { | ||
| 1061 | + EXPECT_NE(candidate.FindNode("input_broadcast"), nullptr); | ||
| 1062 | + } else { | ||
| 1063 | + EXPECT_EQ(candidate.FindNode("input_broadcast"), nullptr); | ||
| 1064 | + } | ||
| 1065 | + EXPECT_EQ(candidate.FindNode("broadcast_input_abs"), nullptr); | ||
| 1060 | ascgen_utils::indirect_load::TemplateLogicalView logical_view; | 1066 | ascgen_utils::indirect_load::TemplateLogicalView logical_view; |
| 1061 | ASSERT_EQ(ascgen_utils::indirect_load::GetTemplateLogicalView(indirect_load, logical_view), af::SUCCESS); | 1067 | ASSERT_EQ(ascgen_utils::indirect_load::GetTemplateLogicalView(indirect_load, logical_view), af::SUCCESS); |
| 1062 | EXPECT_EQ(logical_view.input.kind, ascgen_utils::indirect_load::IndirectLoadLayoutKind::kZeroStrideCompact); | 1068 | EXPECT_EQ(logical_view.input.kind, ascgen_utils::indirect_load::IndirectLoadLayoutKind::kZeroStrideCompact); |
| @@ -1068,7 +1074,7 @@ TEST(IndirectLoadScheduleCaseGeneratorTest, BroadcastDirectPathUsesPhysicalViewF | |||
| 1068 | ASSERT_EQ(generator.Generate(inner_graph, inner_graphs, inner_score_functions), af::SUCCESS); | 1074 | ASSERT_EQ(generator.Generate(inner_graph, inner_graphs, inner_score_functions), af::SUCCESS); |
| 1069 | const auto inner_simd = FindGeneratedGraphByTemplate(inner_graphs, ascir::TemplateId::kIndirectLoadSimd); | 1075 | const auto inner_simd = FindGeneratedGraphByTemplate(inner_graphs, ascir::TemplateId::kIndirectLoadSimd); |
| 1070 | ASSERT_NE(inner_simd, inner_graphs.end()); | 1076 | ASSERT_NE(inner_simd, inner_graphs.end()); |
| 1071 | - EXPECT_EQ(inner_simd->FindNode("input_broadcast"), nullptr); | 1077 | + EXPECT_NE(inner_simd->FindNode("input_broadcast"), nullptr); |
| 1072 | } | 1078 | } |
| 1073 | 1079 | ||
| 1074 | TEST(IndirectLoadScheduleCaseGeneratorTest, SimdDirectBroadcastUsesUnserializedTileSplit) { | 1080 | TEST(IndirectLoadScheduleCaseGeneratorTest, SimdDirectBroadcastUsesUnserializedTileSplit) { |
| @@ -1136,8 +1142,8 @@ TEST(IndirectLoadScheduleCaseGeneratorTest, CompletesMissingDataViewAfterBroadca | |||
| 1136 | EXPECT_EQ(completed_input->outputs()[0]->attr.strides, input_load->outputs()[0]->attr.strides); | 1142 | EXPECT_EQ(completed_input->outputs()[0]->attr.strides, input_load->outputs()[0]->attr.strides); |
| 1137 | } | 1143 | } |
| 1138 | 1144 | ||
| 1139 | -TEST(IndirectLoadScheduleCaseGeneratorTest, RejectsBranchedBroadcastPaths) { | 1145 | +TEST(IndirectLoadScheduleCaseGeneratorTest, BranchedBroadcastPathsKeepSimdCandidate) { |
| 1140 | - // 直连广播形态：广播本身与源的旁路消费者不能安全折叠，淘汰所有相关 candidate。 | 1146 | + // SIMD 不折叠 Broadcast，因此 Broadcast 或其源节点存在旁路消费者时仍可保留 SIMD candidate。 |
| 1141 | for (const char *producer_name : {"input_broadcast", "broadcast_input_load"}) { | 1147 | for (const char *producer_name : {"input_broadcast", "broadcast_input_load"}) { |
| 1142 | auto graph = BuildIndirectLoadBroadcastGraph(false); | 1148 | auto graph = BuildIndirectLoadBroadcastGraph(false); |
| 1143 | ASSERT_TRUE(AddSideConsumer(graph, producer_name)); | 1149 | ASSERT_TRUE(AddSideConsumer(graph, producer_name)); |
| @@ -1145,8 +1151,9 @@ TEST(IndirectLoadScheduleCaseGeneratorTest, RejectsBranchedBroadcastPaths) { | |||
| 1145 | std::vector<af::AscGraph> graphs; | 1151 | std::vector<af::AscGraph> graphs; |
| 1146 | std::vector<std::string> score_functions; | 1152 | std::vector<std::string> score_functions; |
| 1147 | EXPECT_EQ(generator.Generate(graph, graphs, score_functions), af::SUCCESS) << producer_name; | 1153 | EXPECT_EQ(generator.Generate(graph, graphs, score_functions), af::SUCCESS) << producer_name; |
| 1148 | - EXPECT_TRUE(graphs.empty()) << producer_name; | 1154 | + EXPECT_NE(FindGeneratedGraphByTemplate(graphs, ascir::TemplateId::kIndirectLoadSimd), graphs.end()) |
| 1149 | - EXPECT_TRUE(score_functions.empty()) << producer_name; | 1155 | + << producer_name; |
| 1156 | + EXPECT_EQ(score_functions.size(), graphs.size()) << producer_name; | ||
| 1150 | } | 1157 | } |
| 1151 | } | 1158 | } |
| 1152 | 1159 | ||
| @@ -1684,7 +1691,6 @@ TEST(IndirectLoadScheduleCaseGeneratorTest, GeneratedSkCandidateUsesSkBehavior) | |||
| 1684 | EXPECT_FALSE(behavior.uses_direct_gm_pipeline); | 1691 | EXPECT_FALSE(behavior.uses_direct_gm_pipeline); |
| 1685 | EXPECT_FALSE(behavior.skips_ub_lifecycle); | 1692 | EXPECT_FALSE(behavior.skips_ub_lifecycle); |
| 1686 | EXPECT_FALSE(behavior.preserves_vectorized_axis); | 1693 | EXPECT_FALSE(behavior.preserves_vectorized_axis); |
| 1687 | - EXPECT_FALSE(ascgen_utils::indirect_load::ShouldDisableRegularVectorFunc(indirect_load)); | ||
| 1688 | 1694 | ||
| 1689 | const auto input_boundary = ascgen_utils::indirect_load::GetInputProducer(indirect_load, 0UL); | 1695 | const auto input_boundary = ascgen_utils::indirect_load::GetInputProducer(indirect_load, 0UL); |
| 1690 | ASSERT_NE(input_boundary, nullptr); | 1696 | ASSERT_NE(input_boundary, nullptr); |
| @@ -1,3 +1,12 @@ | |||
| 1 | +function(mark_indirect_load_codegen_and_e2e test_name) | ||
| 2 | + target_compile_definitions(${test_name}_codegen_v2 PRIVATE ${ARGN}) | ||
| 3 | + target_compile_definitions(${test_name}_e2e_v2 PRIVATE ${ARGN}) | ||
| 4 | +endfunction() | ||
| 5 | + | ||
| 6 | +function(mark_indirect_load_codegen test_name) | ||
| 7 | + target_compile_definitions(${test_name}_codegen_v2 PRIVATE ${ARGN}) | ||
| 8 | +endfunction() | ||
| 9 | + | ||
| 1 | function(add_indirect_load_e2e_case test_name rank axis input_pre_type use_exp2 tiling_key selected_template expected_templates x0 x1 x2 x3 index0 index1 index2 index3) | 10 | function(add_indirect_load_e2e_case test_name rank axis input_pre_type use_exp2 tiling_key selected_template expected_templates x0 x1 x2 x3 index0 index1 index2 index3) |
| 2 | set(case_workdir ${CMAKE_CURRENT_BINARY_DIR}/${test_name}) | 11 | set(case_workdir ${CMAKE_CURRENT_BINARY_DIR}/${test_name}) |
| 3 | file(MAKE_DIRECTORY ${case_workdir}) | 12 | file(MAKE_DIRECTORY ${case_workdir}) |
| @@ -13,6 +22,7 @@ function(add_indirect_load_e2e_case test_name rank axis input_pre_type use_exp2 | |||
| 13 | TEST_SRC test_e2e_indirect_load_store_kernel.cpp) | 22 | TEST_SRC test_e2e_indirect_load_store_kernel.cpp) |
| 14 | 23 | ||
| 15 | set(case_definitions | 24 | set(case_definitions |
| 25 | + IL_CASE_STORE | ||
| 16 | IL_RANK=${rank} | 26 | IL_RANK=${rank} |
| 17 | IL_AXIS=${axis} | 27 | IL_AXIS=${axis} |
| 18 | IL_INPUT_PRE_TYPE=${input_pre_type} | 28 | IL_INPUT_PRE_TYPE=${input_pre_type} |
| @@ -28,60 +38,50 @@ function(add_indirect_load_e2e_case test_name rank axis input_pre_type use_exp2 | |||
| 28 | IL_INDEX_S1=${index1} | 38 | IL_INDEX_S1=${index1} |
| 29 | IL_INDEX_S2=${index2} | 39 | IL_INDEX_S2=${index2} |
| 30 | IL_INDEX_S3=${index3}) | 40 | IL_INDEX_S3=${index3}) |
| 31 | - target_compile_definitions(${test_name}_codegen_v2 PRIVATE ${case_definitions}) | 41 | + mark_indirect_load_codegen_and_e2e(${test_name} ${case_definitions}) |
| 32 | - target_compile_definitions(${test_name}_e2e_v2 PRIVATE ${case_definitions}) | ||
| 33 | endfunction() | 42 | endfunction() |
| 34 | 43 | ||
| 35 | function(mark_indirect_load_static_shape test_name) | 44 | function(mark_indirect_load_static_shape test_name) |
| 36 | - target_compile_definitions(${test_name}_codegen_v2 PRIVATE IL_STATIC_SHAPE) | 45 | + mark_indirect_load_codegen_and_e2e(${test_name} IL_STATIC_SHAPE) |
| 37 | - target_compile_definitions(${test_name}_e2e_v2 PRIVATE IL_STATIC_SHAPE) | ||
| 38 | endfunction() | 46 | endfunction() |
| 39 | 47 | ||
| 40 | function(mark_indirect_load_sk test_name) | 48 | function(mark_indirect_load_sk test_name) |
| 41 | - target_compile_definitions(${test_name}_codegen_v2 PRIVATE IL_EXPECT_SK) | 49 | + mark_indirect_load_codegen_and_e2e(${test_name} IL_EXPECT_SK) |
| 42 | - target_compile_definitions(${test_name}_e2e_v2 PRIVATE IL_EXPECT_SK) | ||
| 43 | endfunction() | 50 | endfunction() |
| 44 | 51 | ||
| 45 | function(mark_indirect_load_wide_types test_name) | 52 | function(mark_indirect_load_wide_types test_name) |
| 46 | - target_compile_definitions(${test_name}_codegen_v2 PRIVATE IL_DATA_FLOAT IL_INDEX_INT64) | 53 | + mark_indirect_load_codegen_and_e2e(${test_name} IL_DATA_FLOAT IL_INDEX_INT64) |
| 47 | - target_compile_definitions(${test_name}_e2e_v2 PRIVATE IL_DATA_FLOAT IL_INDEX_INT64) | ||
| 48 | endfunction() | 54 | endfunction() |
| 49 | 55 | ||
| 50 | function(mark_indirect_load_input_outer_stride test_name stride) | 56 | function(mark_indirect_load_input_outer_stride test_name stride) |
| 51 | - target_compile_definitions(${test_name}_codegen_v2 PRIVATE IL_INPUT_OUTER_STRIDE=${stride}) | 57 | + mark_indirect_load_codegen_and_e2e(${test_name} IL_INPUT_OUTER_STRIDE=${stride}) |
| 52 | - target_compile_definitions(${test_name}_e2e_v2 PRIVATE IL_INPUT_OUTER_STRIDE=${stride}) | ||
| 53 | endfunction() | 58 | endfunction() |
| 54 | function(mark_indirect_load_large_simt test_name) | 59 | function(mark_indirect_load_large_simt test_name) |
| 55 | target_compile_definitions(${test_name}_e2e_v2 PRIVATE IL_EXPECT_SIMT_MULTI_ROUND) | 60 | target_compile_definitions(${test_name}_e2e_v2 PRIVATE IL_EXPECT_SIMT_MULTI_ROUND) |
| 56 | endfunction() | 61 | endfunction() |
| 57 | 62 | ||
| 58 | function(mark_indirect_load_mixed_index_pre test_name) | 63 | function(mark_indirect_load_mixed_index_pre test_name) |
| 59 | - target_compile_definitions(${test_name}_codegen_v2 PRIVATE IL_MIXED_INDEX_PRE) | 64 | + mark_indirect_load_codegen_and_e2e(${test_name} IL_MIXED_INDEX_PRE) |
| 60 | - target_compile_definitions(${test_name}_e2e_v2 PRIVATE IL_MIXED_INDEX_PRE) | ||
| 61 | endfunction() | 65 | endfunction() |
| 62 | 66 | ||
| 63 | function(mark_indirect_load_output_post test_name output_post_type expect_simt) | 67 | function(mark_indirect_load_output_post test_name output_post_type expect_simt) |
| 64 | - target_compile_definitions(${test_name}_codegen_v2 PRIVATE | 68 | + mark_indirect_load_codegen_and_e2e(${test_name} |
| 65 | - IL_OUTPUT_POST_TYPE=${output_post_type} | ||
| 66 | - IL_EXPECT_SIMT=${expect_simt}) | ||
| 67 | - target_compile_definitions(${test_name}_e2e_v2 PRIVATE | ||
| 68 | IL_OUTPUT_POST_TYPE=${output_post_type} | 69 | IL_OUTPUT_POST_TYPE=${output_post_type} |
| 69 | IL_EXPECT_SIMT=${expect_simt}) | 70 | IL_EXPECT_SIMT=${expect_simt}) |
| 70 | endfunction() | 71 | endfunction() |
| 71 | 72 | ||
| 72 | function(mark_indirect_load_simt_selected test_name) | 73 | function(mark_indirect_load_simt_selected test_name) |
| 73 | - target_compile_definitions(${test_name}_codegen_v2 PRIVATE IL_EXPECT_SIMT_SELECTED) | 74 | + mark_indirect_load_codegen_and_e2e(${test_name} IL_EXPECT_SIMT_SELECTED) |
| 74 | - target_compile_definitions(${test_name}_e2e_v2 PRIVATE IL_EXPECT_SIMT_SELECTED) | ||
| 75 | endfunction() | 75 | endfunction() |
| 76 | 76 | ||
| 77 | function(mark_indirect_load_simt_codegen test_name policy offset_bits) | 77 | function(mark_indirect_load_simt_codegen test_name policy offset_bits) |
| 78 | - target_compile_definitions(${test_name}_codegen_v2 PRIVATE | 78 | + mark_indirect_load_codegen(${test_name} |
| 79 | IL_EXPECT_SIMT_POLICY=${policy} | 79 | IL_EXPECT_SIMT_POLICY=${policy} |
| 80 | IL_EXPECT_SIMT_OFFSET_BITS=${offset_bits}) | 80 | IL_EXPECT_SIMT_OFFSET_BITS=${offset_bits}) |
| 81 | endfunction() | 81 | endfunction() |
| 82 | 82 | ||
| 83 | function(mark_indirect_load_implementation test_name implementation) | 83 | function(mark_indirect_load_implementation test_name implementation) |
| 84 | - target_compile_definitions(${test_name}_codegen_v2 PRIVATE | 84 | + mark_indirect_load_codegen(${test_name} |
| 85 | IL_SELECTED_IMPLEMENTATION=${implementation}) | 85 | IL_SELECTED_IMPLEMENTATION=${implementation}) |
| 86 | endfunction() | 86 | endfunction() |
| 87 | 87 | ||
| @@ -103,6 +103,8 @@ function(add_indirect_load_mixed_e2e_case test_name rank axis is_simt selected_t | |||
| 103 | autofuse_tiling_data.h | 103 | autofuse_tiling_data.h |
| 104 | TEST_SRC test_e2e_indirect_load_store_kernel.cpp) | 104 | TEST_SRC test_e2e_indirect_load_store_kernel.cpp) |
| 105 | set(case_definitions | 105 | set(case_definitions |
| 106 | + IL_CASE_STORE | ||
| 107 | + IL_CASE_MIXED | ||
| 106 | IL_MIXED_ELEMENTWISE | 108 | IL_MIXED_ELEMENTWISE |
| 107 | IL_EXPECT_SIMT=${is_simt} | 109 | IL_EXPECT_SIMT=${is_simt} |
| 108 | IL_RANK=${rank} | 110 | IL_RANK=${rank} |
| @@ -123,13 +125,11 @@ function(add_indirect_load_mixed_e2e_case test_name rank axis is_simt selected_t | |||
| 123 | if(static_shape) | 125 | if(static_shape) |
| 124 | list(APPEND case_definitions IL_STATIC_SHAPE) | 126 | list(APPEND case_definitions IL_STATIC_SHAPE) |
| 125 | endif() | 127 | endif() |
| 126 | - target_compile_definitions(${test_name}_codegen_v2 PRIVATE ${case_definitions}) | 128 | + mark_indirect_load_codegen_and_e2e(${test_name} ${case_definitions}) |
| 127 | - target_compile_definitions(${test_name}_e2e_v2 PRIVATE ${case_definitions}) | ||
| 128 | endfunction() | 129 | endfunction() |
| 129 | 130 | ||
| 130 | function(mark_indirect_load_simt_elementwise_coverage test_name) | 131 | function(mark_indirect_load_simt_elementwise_coverage test_name) |
| 131 | - target_compile_definitions(${test_name}_codegen_v2 PRIVATE IL_SIMT_ELEMENTWISE_COVERAGE) | 132 | + mark_indirect_load_codegen_and_e2e(${test_name} IL_SIMT_ELEMENTWISE_COVERAGE) |
| 132 | - target_compile_definitions(${test_name}_e2e_v2 PRIVATE IL_SIMT_ELEMENTWISE_COVERAGE) | ||
| 133 | endfunction() | 133 | endfunction() |
| 134 | 134 | ||
| 135 | add_indirect_load_e2e_case(indirect_load_rank2_axis1_simd 2 1 1 1 0 0 3 5 8 1 1 4 6 1 1) | 135 | add_indirect_load_e2e_case(indirect_load_rank2_axis1_simd 2 1 1 1 0 0 3 5 8 1 1 4 6 1 1) |
| @@ -137,10 +137,8 @@ mark_indirect_load_static_shape(indirect_load_rank2_axis1_simd) | |||
| 137 | 137 | ||
| 138 | add_indirect_load_e2e_case(indirect_load_rank2_axis1_half_int64_simd 2 1 1 1 0 0 3 5 160 1 1 4 137 1 1) | 138 | add_indirect_load_e2e_case(indirect_load_rank2_axis1_half_int64_simd 2 1 1 1 0 0 3 5 160 1 1 4 137 1 1) |
| 139 | mark_indirect_load_static_shape(indirect_load_rank2_axis1_half_int64_simd) | 139 | mark_indirect_load_static_shape(indirect_load_rank2_axis1_half_int64_simd) |
| 140 | -target_compile_definitions(indirect_load_rank2_axis1_half_int64_simd_codegen_v2 PRIVATE | 140 | +mark_indirect_load_codegen_and_e2e(indirect_load_rank2_axis1_half_int64_simd IL_INDEX_INT64) |
| 141 | - IL_INDEX_INT64 IL_FUNCTIONAL_ONLY) | 141 | +target_compile_definitions(indirect_load_rank2_axis1_half_int64_simd_codegen_v2 PRIVATE IL_FUNCTIONAL_ONLY) |
| 142 | -target_compile_definitions(indirect_load_rank2_axis1_half_int64_simd_e2e_v2 PRIVATE | ||
| 143 | - IL_INDEX_INT64) | ||
| 144 | 142 | ||
| 145 | add_indirect_load_e2e_case(indirect_load_rank2_axis1_b16_gather 2 1 1 1 1 0 3 5 8 1 1 4 6 1 1) | 143 | add_indirect_load_e2e_case(indirect_load_rank2_axis1_b16_gather 2 1 1 1 1 0 3 5 8 1 1 4 6 1 1) |
| 146 | mark_indirect_load_static_shape(indirect_load_rank2_axis1_b16_gather) | 144 | mark_indirect_load_static_shape(indirect_load_rank2_axis1_b16_gather) |
| @@ -148,74 +146,46 @@ mark_indirect_load_implementation(indirect_load_rank2_axis1_b16_gather 1) | |||
| 148 | 146 | ||
| 149 | add_indirect_load_e2e_case(indirect_load_rank3_axis1_float_int64_static 3 1 0 0 0 0 7 16 32 10 1 16 32 10 1) | 147 | add_indirect_load_e2e_case(indirect_load_rank3_axis1_float_int64_static 3 1 0 0 0 0 7 16 32 10 1 16 32 10 1) |
| 150 | mark_indirect_load_static_shape(indirect_load_rank3_axis1_float_int64_static) | 148 | mark_indirect_load_static_shape(indirect_load_rank3_axis1_float_int64_static) |
| 151 | -target_compile_definitions(indirect_load_rank3_axis1_float_int64_static_codegen_v2 PRIVATE | 149 | +mark_indirect_load_codegen_and_e2e(indirect_load_rank3_axis1_float_int64_static |
| 152 | - IL_DATA_FLOAT IL_INDEX_INT64 IL_OUTPUT_POST_TYPE=2 IL_RANDOM_INPUT_INDEX IL_DIRECT_INDEX) | ||
| 153 | -target_compile_definitions(indirect_load_rank3_axis1_float_int64_static_e2e_v2 PRIVATE | ||
| 154 | IL_DATA_FLOAT IL_INDEX_INT64 IL_OUTPUT_POST_TYPE=2 IL_RANDOM_INPUT_INDEX IL_DIRECT_INDEX) | 150 | IL_DATA_FLOAT IL_INDEX_INT64 IL_OUTPUT_POST_TYPE=2 IL_RANDOM_INPUT_INDEX IL_DIRECT_INDEX) |
| 155 | 151 | ||
| 156 | add_indirect_load_e2e_case(indirect_load_rank3_axis1_pow2_simd 3 1 0 0 0 0 3 4 64 32 1 4 64 32 1) | 152 | add_indirect_load_e2e_case(indirect_load_rank3_axis1_pow2_simd 3 1 0 0 0 0 3 4 64 32 1 4 64 32 1) |
| 157 | mark_indirect_load_static_shape(indirect_load_rank3_axis1_pow2_simd) | 153 | mark_indirect_load_static_shape(indirect_load_rank3_axis1_pow2_simd) |
| 158 | -target_compile_definitions(indirect_load_rank3_axis1_pow2_simd_codegen_v2 PRIVATE | 154 | +mark_indirect_load_codegen_and_e2e(indirect_load_rank3_axis1_pow2_simd IL_RANDOM_INPUT_INDEX IL_DIRECT_INDEX) |
| 159 | - IL_RANDOM_INPUT_INDEX IL_DIRECT_INDEX IL_EXPECT_MICRO_SIMD) | 155 | +target_compile_definitions(indirect_load_rank3_axis1_pow2_simd_codegen_v2 PRIVATE IL_EXPECT_MICRO_SIMD) |
| 160 | -target_compile_definitions(indirect_load_rank3_axis1_pow2_simd_e2e_v2 PRIVATE | ||
| 161 | - IL_RANDOM_INPUT_INDEX IL_DIRECT_INDEX) | ||
| 162 | 156 | ||
| 163 | add_indirect_load_e2e_case(indirect_load_rank3_axis1_pow2_gather 3 1 0 0 1 0 3 4 64 32 1 4 64 32 1) | 157 | add_indirect_load_e2e_case(indirect_load_rank3_axis1_pow2_gather 3 1 0 0 1 0 3 4 64 32 1 4 64 32 1) |
| 164 | mark_indirect_load_static_shape(indirect_load_rank3_axis1_pow2_gather) | 158 | mark_indirect_load_static_shape(indirect_load_rank3_axis1_pow2_gather) |
| 165 | mark_indirect_load_implementation(indirect_load_rank3_axis1_pow2_gather 1) | 159 | mark_indirect_load_implementation(indirect_load_rank3_axis1_pow2_gather 1) |
| 166 | -target_compile_definitions(indirect_load_rank3_axis1_pow2_gather_codegen_v2 PRIVATE | 160 | +mark_indirect_load_codegen_and_e2e(indirect_load_rank3_axis1_pow2_gather IL_RANDOM_INPUT_INDEX IL_DIRECT_INDEX) |
| 167 | - IL_RANDOM_INPUT_INDEX IL_DIRECT_INDEX) | ||
| 168 | -target_compile_definitions(indirect_load_rank3_axis1_pow2_gather_e2e_v2 PRIVATE | ||
| 169 | - IL_RANDOM_INPUT_INDEX IL_DIRECT_INDEX) | ||
| 170 | 161 | ||
| 171 | add_indirect_load_e2e_case(indirect_load_rank3_axis1_pow2_simt 3 1 0 0 2 1 3 4 64 32 1 4 64 32 1) | 162 | add_indirect_load_e2e_case(indirect_load_rank3_axis1_pow2_simt 3 1 0 0 2 1 3 4 64 32 1 4 64 32 1) |
| 172 | mark_indirect_load_static_shape(indirect_load_rank3_axis1_pow2_simt) | 163 | mark_indirect_load_static_shape(indirect_load_rank3_axis1_pow2_simt) |
| 173 | mark_indirect_load_simt_selected(indirect_load_rank3_axis1_pow2_simt) | 164 | mark_indirect_load_simt_selected(indirect_load_rank3_axis1_pow2_simt) |
| 174 | mark_indirect_load_simt_codegen(indirect_load_rank3_axis1_pow2_simt StaticPowerOfTwo 32) | 165 | mark_indirect_load_simt_codegen(indirect_load_rank3_axis1_pow2_simt StaticPowerOfTwo 32) |
| 175 | -target_compile_definitions(indirect_load_rank3_axis1_pow2_simt_codegen_v2 PRIVATE | 166 | +mark_indirect_load_codegen_and_e2e(indirect_load_rank3_axis1_pow2_simt IL_DATA_FLOAT IL_RANDOM_INPUT_INDEX IL_DIRECT_INDEX) |
| 176 | - IL_DATA_FLOAT IL_RANDOM_INPUT_INDEX IL_DIRECT_INDEX) | ||
| 177 | -target_compile_definitions(indirect_load_rank3_axis1_pow2_simt_e2e_v2 PRIVATE | ||
| 178 | - IL_DATA_FLOAT IL_RANDOM_INPUT_INDEX IL_DIRECT_INDEX) | ||
| 179 | 167 | ||
| 180 | add_indirect_load_e2e_case(indirect_load_rank3_axis1_float_int32_gather 3 1 0 0 1 0 7 16 32 10 1 16 32 10 1) | 168 | add_indirect_load_e2e_case(indirect_load_rank3_axis1_float_int32_gather 3 1 0 0 1 0 7 16 32 10 1 16 32 10 1) |
| 181 | mark_indirect_load_static_shape(indirect_load_rank3_axis1_float_int32_gather) | 169 | mark_indirect_load_static_shape(indirect_load_rank3_axis1_float_int32_gather) |
| 182 | mark_indirect_load_implementation(indirect_load_rank3_axis1_float_int32_gather 1) | 170 | mark_indirect_load_implementation(indirect_load_rank3_axis1_float_int32_gather 1) |
| 183 | -target_compile_definitions(indirect_load_rank3_axis1_float_int32_gather_codegen_v2 PRIVATE | 171 | +mark_indirect_load_codegen_and_e2e(indirect_load_rank3_axis1_float_int32_gather |
| 184 | IL_DATA_FLOAT IL_OUTPUT_POST_TYPE=2 IL_RANDOM_INPUT_INDEX IL_DIRECT_INDEX) | 172 | IL_DATA_FLOAT IL_OUTPUT_POST_TYPE=2 IL_RANDOM_INPUT_INDEX IL_DIRECT_INDEX) |
| 185 | -target_compile_definitions(indirect_load_rank3_axis1_float_int32_gather_e2e_v2 PRIVATE | ||
| 186 | - IL_DATA_FLOAT IL_OUTPUT_POST_TYPE=2 IL_RANDOM_INPUT_INDEX IL_DIRECT_INDEX) | ||
| 187 | - | ||
| 188 | -add_indirect_load_e2e_case(indirect_load_rank3_axis1_float_int64_gather_exact 3 1 0 0 1 0 7 16 32 10 1 16 32 10 1) | ||
| 189 | -mark_indirect_load_static_shape(indirect_load_rank3_axis1_float_int64_gather_exact) | ||
| 190 | -mark_indirect_load_implementation(indirect_load_rank3_axis1_float_int64_gather_exact 1) | ||
| 191 | -target_compile_definitions(indirect_load_rank3_axis1_float_int64_gather_exact_codegen_v2 PRIVATE | ||
| 192 | - IL_DATA_FLOAT IL_INDEX_INT64 IL_OUTPUT_POST_TYPE=2 IL_RANDOM_INPUT_INDEX IL_DIRECT_INDEX) | ||
| 193 | -target_compile_definitions(indirect_load_rank3_axis1_float_int64_gather_exact_e2e_v2 PRIVATE | ||
| 194 | - IL_DATA_FLOAT IL_INDEX_INT64 IL_OUTPUT_POST_TYPE=2 IL_RANDOM_INPUT_INDEX IL_DIRECT_INDEX) | ||
| 195 | 173 | ||
| 196 | add_indirect_load_e2e_case(indirect_load_rank3_axis1_float_int64_gather 3 1 0 0 1 0 7 16 31 10 1 16 31 10 1) | 174 | add_indirect_load_e2e_case(indirect_load_rank3_axis1_float_int64_gather 3 1 0 0 1 0 7 16 31 10 1 16 31 10 1) |
| 197 | mark_indirect_load_static_shape(indirect_load_rank3_axis1_float_int64_gather) | 175 | mark_indirect_load_static_shape(indirect_load_rank3_axis1_float_int64_gather) |
| 198 | mark_indirect_load_implementation(indirect_load_rank3_axis1_float_int64_gather 1) | 176 | mark_indirect_load_implementation(indirect_load_rank3_axis1_float_int64_gather 1) |
| 199 | -target_compile_definitions(indirect_load_rank3_axis1_float_int64_gather_codegen_v2 PRIVATE | 177 | +mark_indirect_load_codegen_and_e2e(indirect_load_rank3_axis1_float_int64_gather |
| 200 | - IL_DATA_FLOAT IL_INDEX_INT64 IL_OUTPUT_POST_TYPE=2 IL_RANDOM_INPUT_INDEX IL_DIRECT_INDEX) | ||
| 201 | -target_compile_definitions(indirect_load_rank3_axis1_float_int64_gather_e2e_v2 PRIVATE | ||
| 202 | IL_DATA_FLOAT IL_INDEX_INT64 IL_OUTPUT_POST_TYPE=2 IL_RANDOM_INPUT_INDEX IL_DIRECT_INDEX) | 178 | IL_DATA_FLOAT IL_INDEX_INT64 IL_OUTPUT_POST_TYPE=2 IL_RANDOM_INPUT_INDEX IL_DIRECT_INDEX) |
| 203 | 179 | ||
| 204 | add_indirect_load_e2e_case(indirect_load_rank4_axis1_float_int64_strided_post_simd | 180 | add_indirect_load_e2e_case(indirect_load_rank4_axis1_float_int64_strided_post_simd |
| 205 | 4 1 0 0 0 0 7 3 17 5 11 3 19 3 7) | 181 | 4 1 0 0 0 0 7 3 17 5 11 3 19 3 7) |
| 206 | mark_indirect_load_static_shape(indirect_load_rank4_axis1_float_int64_strided_post_simd) | 182 | mark_indirect_load_static_shape(indirect_load_rank4_axis1_float_int64_strided_post_simd) |
| 207 | -target_compile_definitions(indirect_load_rank4_axis1_float_int64_strided_post_simd_codegen_v2 PRIVATE | 183 | +mark_indirect_load_codegen_and_e2e(indirect_load_rank4_axis1_float_int64_strided_post_simd IL_DATA_FLOAT IL_INDEX_INT64) |
| 208 | - IL_DATA_FLOAT IL_INDEX_INT64) | ||
| 209 | -target_compile_definitions(indirect_load_rank4_axis1_float_int64_strided_post_simd_e2e_v2 PRIVATE | ||
| 210 | - IL_DATA_FLOAT IL_INDEX_INT64) | ||
| 211 | 184 | ||
| 212 | add_indirect_load_e2e_case(indirect_load_rank4_axis1_direct_index_abs_exp2_sum_simd 4 1 0 0 0 0 3 8 13 3 8 8 17 3 8) | 185 | add_indirect_load_e2e_case(indirect_load_rank4_axis1_direct_index_abs_exp2_sum_simd 4 1 0 0 0 0 3 8 13 3 8 8 17 3 8) |
| 213 | mark_indirect_load_static_shape(indirect_load_rank4_axis1_direct_index_abs_exp2_sum_simd) | 186 | mark_indirect_load_static_shape(indirect_load_rank4_axis1_direct_index_abs_exp2_sum_simd) |
| 214 | mark_indirect_load_output_post(indirect_load_rank4_axis1_direct_index_abs_exp2_sum_simd 8 1) | 187 | mark_indirect_load_output_post(indirect_load_rank4_axis1_direct_index_abs_exp2_sum_simd 8 1) |
| 215 | -target_compile_definitions(indirect_load_rank4_axis1_direct_index_abs_exp2_sum_simd_codegen_v2 PRIVATE | 188 | +mark_indirect_load_codegen_and_e2e(indirect_load_rank4_axis1_direct_index_abs_exp2_sum_simd |
| 216 | - IL_DATA_FLOAT IL_POST_REDUCE IL_POST_REDUCE_EXP2 IL_POST_REDUCE_ABS IL_EXPECT_SIMD_SELECTED | ||
| 217 | - IL_RANDOM_INPUT_INDEX IL_DIRECT_INDEX) | ||
| 218 | -target_compile_definitions(indirect_load_rank4_axis1_direct_index_abs_exp2_sum_simd_e2e_v2 PRIVATE | ||
| 219 | IL_DATA_FLOAT IL_POST_REDUCE IL_POST_REDUCE_EXP2 IL_POST_REDUCE_ABS IL_EXPECT_SIMD_SELECTED | 189 | IL_DATA_FLOAT IL_POST_REDUCE IL_POST_REDUCE_EXP2 IL_POST_REDUCE_ABS IL_EXPECT_SIMD_SELECTED |
| 220 | IL_RANDOM_INPUT_INDEX IL_DIRECT_INDEX) | 190 | IL_RANDOM_INPUT_INDEX IL_DIRECT_INDEX) |
| 221 | 191 | ||
| @@ -237,19 +207,16 @@ mark_indirect_load_simt_codegen(indirect_load_rank2_axis1_large_simt Recursive 6 | |||
| 237 | 207 | ||
| 238 | add_indirect_load_e2e_case(indirect_load_rank2_axis1_bf16_int64_simt 2 1 0 0 2 1 3 5 8 1 1 4 6 1 1) | 208 | add_indirect_load_e2e_case(indirect_load_rank2_axis1_bf16_int64_simt 2 1 0 0 2 1 3 5 8 1 1 4 6 1 1) |
| 239 | mark_indirect_load_output_post(indirect_load_rank2_axis1_bf16_int64_simt 2 1) | 209 | mark_indirect_load_output_post(indirect_load_rank2_axis1_bf16_int64_simt 2 1) |
| 240 | -target_compile_definitions(indirect_load_rank2_axis1_bf16_int64_simt_codegen_v2 PRIVATE IL_DATA_BF16 IL_INDEX_INT64) | 210 | +mark_indirect_load_codegen_and_e2e(indirect_load_rank2_axis1_bf16_int64_simt IL_DATA_BF16 IL_INDEX_INT64) |
| 241 | -target_compile_definitions(indirect_load_rank2_axis1_bf16_int64_simt_e2e_v2 PRIVATE IL_DATA_BF16 IL_INDEX_INT64) | ||
| 242 | 211 | ||
| 243 | add_indirect_load_e2e_case(indirect_load_rank2_axis1_uint32_int32_simd 2 1 0 1 0 0 3 5 8 1 1 4 6 1 1) | 212 | add_indirect_load_e2e_case(indirect_load_rank2_axis1_uint32_int32_simd 2 1 0 1 0 0 3 5 8 1 1 4 6 1 1) |
| 244 | mark_indirect_load_output_post(indirect_load_rank2_axis1_uint32_int32_simd 3 1) | 213 | mark_indirect_load_output_post(indirect_load_rank2_axis1_uint32_int32_simd 3 1) |
| 245 | -target_compile_definitions(indirect_load_rank2_axis1_uint32_int32_simd_codegen_v2 PRIVATE IL_DATA_UINT32) | 214 | +mark_indirect_load_codegen_and_e2e(indirect_load_rank2_axis1_uint32_int32_simd IL_DATA_UINT32) |
| 246 | -target_compile_definitions(indirect_load_rank2_axis1_uint32_int32_simd_e2e_v2 PRIVATE IL_DATA_UINT32) | ||
| 247 | 215 | ||
| 248 | add_indirect_load_e2e_case(indirect_load_rank4_axis1_add_sum_simt 4 1 0 0 0 1 3 8 13 3 8 8 12 3 8) | 216 | add_indirect_load_e2e_case(indirect_load_rank4_axis1_add_sum_simt 4 1 0 0 0 1 3 8 13 3 8 8 12 3 8) |
| 249 | mark_indirect_load_static_shape(indirect_load_rank4_axis1_add_sum_simt) | 217 | mark_indirect_load_static_shape(indirect_load_rank4_axis1_add_sum_simt) |
| 250 | mark_indirect_load_output_post(indirect_load_rank4_axis1_add_sum_simt 10 1) | 218 | mark_indirect_load_output_post(indirect_load_rank4_axis1_add_sum_simt 10 1) |
| 251 | -target_compile_definitions(indirect_load_rank4_axis1_add_sum_simt_codegen_v2 PRIVATE IL_DATA_FLOAT IL_POST_REDUCE IL_POST_REDUCE_ADD) | 219 | +mark_indirect_load_codegen_and_e2e(indirect_load_rank4_axis1_add_sum_simt IL_DATA_FLOAT IL_POST_REDUCE IL_POST_REDUCE_ADD) |
| 252 | -target_compile_definitions(indirect_load_rank4_axis1_add_sum_simt_e2e_v2 PRIVATE IL_DATA_FLOAT IL_POST_REDUCE IL_POST_REDUCE_ADD) | ||
| 253 | mark_indirect_load_simt_selected(indirect_load_rank4_axis1_add_sum_simt) | 220 | mark_indirect_load_simt_selected(indirect_load_rank4_axis1_add_sum_simt) |
| 254 | mark_indirect_load_simt_codegen(indirect_load_rank4_axis1_add_sum_simt StructuredMagic 32) | 221 | mark_indirect_load_simt_codegen(indirect_load_rank4_axis1_add_sum_simt StructuredMagic 32) |
| 255 | 222 | ||
| @@ -257,25 +224,20 @@ add_indirect_load_e2e_case(indirect_load_rank4_axis1_sum_axis2_with_a_fallback_s | |||
| 257 | mark_indirect_load_static_shape(indirect_load_rank4_axis1_sum_axis2_with_a_fallback_simt) | 224 | mark_indirect_load_static_shape(indirect_load_rank4_axis1_sum_axis2_with_a_fallback_simt) |
| 258 | mark_indirect_load_mixed_index_pre(indirect_load_rank4_axis1_sum_axis2_with_a_fallback_simt) | 225 | mark_indirect_load_mixed_index_pre(indirect_load_rank4_axis1_sum_axis2_with_a_fallback_simt) |
| 259 | mark_indirect_load_output_post(indirect_load_rank4_axis1_sum_axis2_with_a_fallback_simt 5 1) | 226 | mark_indirect_load_output_post(indirect_load_rank4_axis1_sum_axis2_with_a_fallback_simt 5 1) |
| 260 | -target_compile_definitions(indirect_load_rank4_axis1_sum_axis2_with_a_fallback_simt_codegen_v2 PRIVATE | 227 | +mark_indirect_load_codegen_and_e2e(indirect_load_rank4_axis1_sum_axis2_with_a_fallback_simt |
| 261 | - IL_DATA_FLOAT IL_POST_REDUCE IL_REDUCE_WITH_A IL_EXPECT_ONLY_SIMT) | ||
| 262 | -target_compile_definitions(indirect_load_rank4_axis1_sum_axis2_with_a_fallback_simt_e2e_v2 PRIVATE | ||
| 263 | IL_DATA_FLOAT IL_POST_REDUCE IL_REDUCE_WITH_A IL_EXPECT_ONLY_SIMT) | 228 | IL_DATA_FLOAT IL_POST_REDUCE IL_REDUCE_WITH_A IL_EXPECT_ONLY_SIMT) |
| 264 | 229 | ||
| 265 | add_indirect_load_e2e_case(indirect_load_rank4_axis1_sum_last_axis_simt 4 1 0 0 0 1 3 8 13 3 8 8 12 3 8) | 230 | add_indirect_load_e2e_case(indirect_load_rank4_axis1_sum_last_axis_simt 4 1 0 0 0 1 3 8 13 3 8 8 12 3 8) |
| 266 | mark_indirect_load_static_shape(indirect_load_rank4_axis1_sum_last_axis_simt) | 231 | mark_indirect_load_static_shape(indirect_load_rank4_axis1_sum_last_axis_simt) |
| 267 | mark_indirect_load_output_post(indirect_load_rank4_axis1_sum_last_axis_simt 6 1) | 232 | mark_indirect_load_output_post(indirect_load_rank4_axis1_sum_last_axis_simt 6 1) |
| 268 | -target_compile_definitions(indirect_load_rank4_axis1_sum_last_axis_simt_codegen_v2 PRIVATE IL_DATA_FLOAT IL_POST_REDUCE IL_REDUCE_LAST_AXIS) | 233 | +mark_indirect_load_codegen_and_e2e(indirect_load_rank4_axis1_sum_last_axis_simt IL_DATA_FLOAT IL_POST_REDUCE IL_REDUCE_LAST_AXIS) |
| 269 | -target_compile_definitions(indirect_load_rank4_axis1_sum_last_axis_simt_e2e_v2 PRIVATE IL_DATA_FLOAT IL_POST_REDUCE IL_REDUCE_LAST_AXIS) | ||
| 270 | mark_indirect_load_simt_selected(indirect_load_rank4_axis1_sum_last_axis_simt) | 234 | mark_indirect_load_simt_selected(indirect_load_rank4_axis1_sum_last_axis_simt) |
| 271 | 235 | ||
| 272 | add_indirect_load_e2e_case(indirect_load_rank4_axis1_full_prefix_bessel_k0_sum_simd 4 1 3 0 0 0 1 8 13 3 8 8 17 3 8) | 236 | add_indirect_load_e2e_case(indirect_load_rank4_axis1_full_prefix_bessel_k0_sum_simd 4 1 3 0 0 0 1 8 13 3 8 8 17 3 8) |
| 273 | mark_indirect_load_static_shape(indirect_load_rank4_axis1_full_prefix_bessel_k0_sum_simd) | 237 | mark_indirect_load_static_shape(indirect_load_rank4_axis1_full_prefix_bessel_k0_sum_simd) |
| 274 | mark_indirect_load_mixed_index_pre(indirect_load_rank4_axis1_full_prefix_bessel_k0_sum_simd) | 238 | mark_indirect_load_mixed_index_pre(indirect_load_rank4_axis1_full_prefix_bessel_k0_sum_simd) |
| 275 | mark_indirect_load_output_post(indirect_load_rank4_axis1_full_prefix_bessel_k0_sum_simd 11 0) | 239 | mark_indirect_load_output_post(indirect_load_rank4_axis1_full_prefix_bessel_k0_sum_simd 11 0) |
| 276 | -target_compile_definitions(indirect_load_rank4_axis1_full_prefix_bessel_k0_sum_simd_codegen_v2 PRIVATE | 240 | +mark_indirect_load_codegen_and_e2e(indirect_load_rank4_axis1_full_prefix_bessel_k0_sum_simd |
| 277 | - IL_DATA_FLOAT IL_POST_REDUCE IL_POST_REDUCE_SIMD) | ||
| 278 | -target_compile_definitions(indirect_load_rank4_axis1_full_prefix_bessel_k0_sum_simd_e2e_v2 PRIVATE | ||
| 279 | IL_DATA_FLOAT IL_POST_REDUCE IL_POST_REDUCE_SIMD) | 241 | IL_DATA_FLOAT IL_POST_REDUCE IL_POST_REDUCE_SIMD) |
| 280 | target_include_directories(indirect_load_rank4_axis1_full_prefix_bessel_k0_sum_simd_e2e_v2 PRIVATE | 242 | target_include_directories(indirect_load_rank4_axis1_full_prefix_bessel_k0_sum_simd_e2e_v2 PRIVATE |
| 281 | ${CODE_ROOT_DIR}/v35/ascendc/api_regbase) | 243 | ${CODE_ROOT_DIR}/v35/ascendc/api_regbase) |
| @@ -285,17 +247,254 @@ add_indirect_load_mixed_e2e_case(indirect_load_simt_elementwise_coverage 3 1 1 1 | |||
| 285 | mark_indirect_load_simt_elementwise_coverage(indirect_load_simt_elementwise_coverage) | 247 | mark_indirect_load_simt_elementwise_coverage(indirect_load_simt_elementwise_coverage) |
| 286 | mark_indirect_load_simt_codegen(indirect_load_simt_elementwise_coverage Recursive 64) | 248 | mark_indirect_load_simt_codegen(indirect_load_simt_elementwise_coverage Recursive 64) |
| 287 | 249 | ||
| 288 | -include(indirect_load_broadcast_cases.cmake) | 250 | +function(add_indirect_load_broadcast_test test_name template input_element_count index_element_count broadcast_axes_mask |
| 289 | -include(indirect_load_torch_gather_strided_cases.cmake) | 251 | + output_relu clear_broadcast_source_view) |
| 252 | + if(ARGC GREATER 7) | ||
| 253 | + set(input_broadcast ${ARGV7}) | ||
| 254 | + else() | ||
| 255 | + set(input_broadcast 1) | ||
| 256 | + endif() | ||
| 257 | + if(ARGC GREATER 8) | ||
| 258 | + set(index_broadcast ${ARGV8}) | ||
| 259 | + else() | ||
| 260 | + set(index_broadcast 1) | ||
| 261 | + endif() | ||
| 262 | + set(expect_simt 0) | ||
| 263 | + set(expect_sk 0) | ||
| 264 | + set(tiling_options) | ||
| 265 | + if(template STREQUAL "simt") | ||
| 266 | + set(expect_simt 1) | ||
| 267 | + set(tiling_options TILING_KEY 1) | ||
| 268 | + elseif(template STREQUAL "sk") | ||
| 269 | + set(expect_sk 1) | ||
| 270 | + set(tiling_options TILING_KEY 2) | ||
| 271 | + elseif(NOT template STREQUAL "simd") | ||
| 272 | + message(FATAL_ERROR "Unsupported IndirectLoad template: ${template}") | ||
| 273 | + endif() | ||
| 274 | + | ||
| 275 | + set(case_workdir ${CMAKE_CURRENT_BINARY_DIR}/${test_name}) | ||
| 276 | + file(MAKE_DIRECTORY ${case_workdir}) | ||
| 277 | + do_backend_e2e_st_test(${test_name} | ||
| 278 | + WORKDIR ${case_workdir} | ||
| 279 | + CODEGEN indirect_load_store_backend_generator.cpp | ||
| 280 | + ${tiling_options} | ||
| 281 | + KERNEL_SRC | ||
| 282 | + indirect_load_broadcast_test_kernel.cpp | ||
| 283 | + indirect_load_broadcast_test_tiling.cpp | ||
| 284 | + autofuse_tiling_data.h | ||
| 285 | + TEST_SRC test_e2e_indirect_load_store_kernel.cpp) | ||
| 286 | + set(case_definitions | ||
| 287 | + IL_CASE_BROADCAST | ||
| 288 | + IL_INPUT_BROADCAST=${input_broadcast} | ||
| 289 | + IL_INDEX_BROADCAST=${index_broadcast} | ||
| 290 | + IL_HAS_INPUT_ELEMENT=${input_element_count} | ||
| 291 | + IL_HAS_INDEX_ELEMENT=${index_element_count} | ||
| 292 | + IL_HAS_OUTPUT_RELU=${output_relu} | ||
| 293 | + IL_BROADCAST_AXES_MASK=${broadcast_axes_mask} | ||
| 294 | + IL_CLEAR_BROADCAST_SOURCE_VIEW=${clear_broadcast_source_view} | ||
| 295 | + IL_EXPECT_SIMT=${expect_simt} | ||
| 296 | + IL_EXPECT_SK=${expect_sk}) | ||
| 297 | + mark_indirect_load_codegen_and_e2e(${test_name} ${case_definitions}) | ||
| 298 | +endfunction() | ||
| 299 | + | ||
| 300 | +# Direct Broadcast covers IndirectLoad axis 2, its inner neighbor, and a degenerate source crossing the axis boundary. | ||
| 301 | +add_indirect_load_broadcast_test(indirect_load_broadcast_cross_boundary_simt_fallback_test simt 0 0 14 0 0) | ||
| 302 | +mark_indirect_load_codegen_and_e2e(indirect_load_broadcast_cross_boundary_simt_fallback_test | ||
| 303 | + IL_DEGENERATE_BROADCAST=1 IL_OUTPUT_S0=10 IL_OUTPUT_S1=10 IL_OUTPUT_S2=20 IL_OUTPUT_S3=20) | ||
| 304 | +add_indirect_load_broadcast_test(indirect_load_broadcast_axis_simd_test simd 1 1 4 0 0) | ||
| 305 | +add_indirect_load_broadcast_test(indirect_load_broadcast_inner_adjacent_simd_test simd 0 0 8 0 0) | ||
| 306 | +mark_indirect_load_codegen_and_e2e(indirect_load_broadcast_inner_adjacent_simd_test | ||
| 307 | + IL_INPUT_ABS_BEFORE_BROADCAST=1) | ||
| 308 | +add_indirect_load_broadcast_test(indirect_load_broadcast_continuous_simd_test simd 0 0 12 0 0) | ||
| 309 | +mark_indirect_load_codegen_and_e2e(indirect_load_broadcast_continuous_simd_test IL_CONTINUOUS_BROADCAST=1) | ||
| 310 | +add_indirect_load_broadcast_test(indirect_load_broadcast_continuous_index_simt_test simt 0 0 12 0 0 0 1) | ||
| 311 | +mark_indirect_load_codegen_and_e2e(indirect_load_broadcast_continuous_index_simt_test | ||
| 312 | + IL_CONTINUOUS_INDEX_BROADCAST=1) | ||
| 313 | +add_indirect_load_broadcast_test(indirect_load_broadcast_cross_boundary_simt_test simt 0 0 10 0 0) | ||
| 314 | +add_indirect_load_broadcast_test(indirect_load_broadcast_axis_simt_test simt 1 1 4 0 0) | ||
| 315 | +add_indirect_load_broadcast_test(indirect_load_broadcast_reduce_simt_fallback_test simt 0 0 2 0 0) | ||
| 316 | +mark_indirect_load_codegen_and_e2e(indirect_load_broadcast_reduce_simt_fallback_test IL_BROADCAST_POST_REDUCE=1) | ||
| 317 | + | ||
| 318 | +# A scalar Broadcast after an Element cannot use the physical-view inline path. SIMD keeps it as a regular op; | ||
| 319 | +# SIMT emits it inside the fused scalar body. | ||
| 320 | +add_indirect_load_broadcast_test(indirect_load_broadcast_retained_simd_test simd 0 0 10 0 0 0 0) | ||
| 321 | +mark_indirect_load_codegen_and_e2e(indirect_load_broadcast_retained_simd_test IL_RETAIN_BROADCAST=1) | ||
| 322 | +add_indirect_load_broadcast_test(indirect_load_broadcast_retained_simt_test simt 0 0 10 0 0 0 0) | ||
| 323 | +mark_indirect_load_codegen_and_e2e(indirect_load_broadcast_retained_simt_test IL_RETAIN_BROADCAST=1) | ||
| 324 | + | ||
| 325 | +# Keep one SK Broadcast regression outside the SIMD/SIMT matrix. | ||
| 326 | +add_indirect_load_broadcast_test(indirect_load_broadcast_elements_sk_test sk 2 2 3 0 0) | ||
| 327 | +add_indirect_load_broadcast_test(indirect_load_broadcast_index_physical_view_simt_test simt 2 2 3 0 0) | ||
| 328 | +mark_indirect_load_codegen_and_e2e(indirect_load_broadcast_index_physical_view_simt_test IL_AIC_REPRO=1) | ||
| 329 | + | ||
| 330 | +# Identity Broadcast keeps the index source axis/repeats unchanged while exercising the direct SIMD Broadcast path. | ||
| 331 | +add_indirect_load_broadcast_test(indirect_load_broadcast_identity_index_simd_test simd 0 0 0 0 0 0 1) | ||
| 332 | + | ||
| 333 | +# Regression: a unary elementwise op after an index Broadcast must retain the source physical view in SIMT. | ||
| 334 | +add_indirect_load_broadcast_test(indirect_load_broadcast_index_abs_simt_test simt 0 1 2 0 0 0 1) | ||
| 335 | +mark_indirect_load_codegen_and_e2e(indirect_load_broadcast_index_abs_simt_test IL_INDEX_ABS_DENSE_VIEW=1) | ||
| 336 | + | ||
| 337 | +# Regression: a three-input Where chain must remain in the SIMT index region before IndirectLoad. | ||
| 338 | +set(indirect_load_broadcast_index_where_simt_test_workdir | ||
| 339 | + ${CMAKE_CURRENT_BINARY_DIR}/indirect_load_broadcast_index_where_simt_test) | ||
| 340 | +file(MAKE_DIRECTORY ${indirect_load_broadcast_index_where_simt_test_workdir}) | ||
| 341 | +do_backend_e2e_st_test(indirect_load_broadcast_index_where_simt_test | ||
| 342 | + WORKDIR ${indirect_load_broadcast_index_where_simt_test_workdir} | ||
| 343 | + CODEGEN indirect_load_store_backend_generator.cpp | ||
| 344 | + TILING_KEY 1 | ||
| 345 | + KERNEL_SRC | ||
| 346 | + indirect_load_broadcast_where_test_kernel.cpp | ||
| 347 | + indirect_load_broadcast_where_test_tiling.cpp | ||
| 348 | + autofuse_tiling_data.h | ||
| 349 | + TEST_SRC test_e2e_indirect_load_store_kernel.cpp) | ||
| 350 | +mark_indirect_load_codegen_and_e2e(indirect_load_broadcast_index_where_simt_test IL_CASE_BROADCAST_WHERE) | ||
| 351 | + | ||
| 352 | +# Regression: SIMT post-reduce Output() evaluator uses full output_index instead of address.index_offset | ||
| 353 | +# to access index-tensor GM, causing OOB reads when output_index exceeds index tensor size. | ||
| 354 | +set(indirect_load_embedding_reduce_simt_test_workdir | ||
| 355 | + ${CMAKE_CURRENT_BINARY_DIR}/indirect_load_embedding_reduce_simt_test) | ||
| 356 | +file(MAKE_DIRECTORY ${indirect_load_embedding_reduce_simt_test_workdir}) | ||
| 357 | +do_backend_e2e_st_test(indirect_load_embedding_reduce_simt_test | ||
| 358 | + WORKDIR ${indirect_load_embedding_reduce_simt_test_workdir} | ||
| 359 | + CODEGEN indirect_load_store_backend_generator.cpp | ||
| 360 | + TILING_KEY 1 | ||
| 361 | + KERNEL_SRC | ||
| 362 | + indirect_load_embedding_reduce_simt_test_kernel.cpp | ||
| 363 | + indirect_load_embedding_reduce_simt_test_tiling.cpp | ||
| 364 | + autofuse_tiling_data.h | ||
| 365 | + TEST_SRC test_e2e_indirect_load_store_kernel.cpp) | ||
| 366 | +mark_indirect_load_codegen_and_e2e(indirect_load_embedding_reduce_simt_test IL_CASE_BROADCAST_WHERE IL_EMBEDDING_REDUCE) | ||
| 367 | + | ||
| 368 | +# Regression: a dual-input Add before IndirectLoad with a post-fused ReduceSum must keep a valid SIMT kernel. | ||
| 369 | +set(indirect_load_add_il_reduce_test_workdir | ||
| 370 | + ${CMAKE_CURRENT_BINARY_DIR}/indirect_load_add_il_reduce_test) | ||
| 371 | +file(MAKE_DIRECTORY ${indirect_load_add_il_reduce_test_workdir}) | ||
| 372 | +do_backend_e2e_st_test(indirect_load_add_il_reduce_test | ||
| 373 | + WORKDIR ${indirect_load_add_il_reduce_test_workdir} | ||
| 374 | + CODEGEN indirect_load_store_backend_generator.cpp | ||
| 375 | + TILING_KEY 1 | ||
| 376 | + KERNEL_SRC | ||
| 377 | + indirect_load_add_il_reduce_test_kernel.cpp | ||
| 378 | + indirect_load_add_il_reduce_test_tiling.cpp | ||
| 379 | + autofuse_tiling_data.h | ||
| 380 | + TEST_SRC test_e2e_indirect_load_store_kernel.cpp) | ||
| 381 | +mark_indirect_load_codegen_and_e2e(indirect_load_add_il_reduce_test IL_CASE_BROADCAST_WHERE) | ||
| 382 | +mark_indirect_load_codegen_and_e2e(indirect_load_add_il_reduce_test IL_ADD_IL_REDUCE) | ||
| 383 | + | ||
| 384 | +# Same-view tensor fan-in without Broadcast: the binary operation is coordinate-preserving. | ||
| 385 | +add_indirect_load_broadcast_test(indirect_load_index_binary_same_view_simd_test simd 0 1 0 0 0 0 0) | ||
| 386 | +mark_indirect_load_codegen_and_e2e(indirect_load_index_binary_same_view_simd_test | ||
| 387 | + IL_INDEX_BINARY_SAME_VIEW=1 IL_BINARY_ELEMENT_KIND=3) | ||
| 388 | + | ||
| 389 | +add_indirect_load_broadcast_test(indirect_load_complex_broadcast_simd_test simd 0 0 3 0 0) | ||
| 390 | +mark_indirect_load_codegen_and_e2e(indirect_load_complex_broadcast_simd_test | ||
| 391 | + IL_COMPLEX_BROADCAST=1 IL_BINARY_ELEMENT_KIND=3) | ||
| 392 | + | ||
| 393 | +add_indirect_load_broadcast_test(indirect_load_complex_broadcast_simt_test simt 0 0 3 0 0) | ||
| 394 | +mark_indirect_load_codegen_and_e2e(indirect_load_complex_broadcast_simt_test IL_COMPLEX_BROADCAST=1 IL_COMPLEX_SIMT=1) | ||
| 395 | + | ||
| 396 | +# Input Broadcast whose source is a multi-input binary element chain: the SIMD candidate keeps the direct Broadcast and | ||
| 397 | +# candidate consumes the computed source through its physical view (source multi-input scenario). | ||
| 398 | +# Note: SIMT rejects multi-input input sources by design (ValidateSimtTemplateRegion requires a Load | ||
| 399 | +# boundary), covered by UT SimdRegionMetadataAndSimtRejectsMultiInputRegion. | ||
| 400 | +add_indirect_load_broadcast_test(indirect_load_complex_input_broadcast_simd_test simd 0 0 12 0 0 1 0) | ||
| 401 | +mark_indirect_load_codegen_and_e2e(indirect_load_complex_input_broadcast_simd_test | ||
| 402 | + IL_COMPLEX_INPUT_BROADCAST=1 IL_BINARY_ELEMENT_KIND=3) | ||
| 403 | + | ||
| 404 | +function(add_indirect_load_stride_zero_test test_name template input_zero_stride_mask index_zero_stride_mask | ||
| 405 | + input_element_count index_element_count) | ||
| 406 | + set(expect_simt 0) | ||
| 407 | + set(expect_sk 0) | ||
| 408 | + set(tiling_options) | ||
| 409 | + if(template STREQUAL "simt") | ||
| 410 | + set(expect_simt 1) | ||
| 411 | + set(tiling_options TILING_KEY 1) | ||
| 412 | + elseif(template STREQUAL "sk") | ||
| 413 | + set(expect_sk 1) | ||
| 414 | + set(tiling_options TILING_KEY 2) | ||
| 415 | + elseif(NOT template STREQUAL "simd") | ||
| 416 | + message(FATAL_ERROR "Unsupported IndirectLoad template: ${template}") | ||
| 417 | + endif() | ||
| 418 | + | ||
| 419 | + set(case_workdir ${CMAKE_CURRENT_BINARY_DIR}/${test_name}) | ||
| 420 | + file(MAKE_DIRECTORY ${case_workdir}) | ||
| 421 | + do_backend_e2e_st_test(${test_name} | ||
| 422 | + WORKDIR ${case_workdir} | ||
| 423 | + CODEGEN indirect_load_store_backend_generator.cpp | ||
| 424 | + ${tiling_options} | ||
| 425 | + KERNEL_SRC | ||
| 426 | + indirect_load_stride_zero_test_kernel.cpp | ||
| 427 | + indirect_load_stride_zero_test_tiling.cpp | ||
| 428 | + autofuse_tiling_data.h | ||
| 429 | + TEST_SRC test_e2e_indirect_load_store_kernel.cpp) | ||
| 430 | + set(case_definitions | ||
| 431 | + IL_CASE_STRIDE_ZERO | ||
| 432 | + IL_INPUT_ZERO_STRIDE_MASK=${input_zero_stride_mask} | ||
| 433 | + IL_INDEX_ZERO_STRIDE_MASK=${index_zero_stride_mask} | ||
| 434 | + IL_HAS_INPUT_ELEMENT=${input_element_count} | ||
| 435 | + IL_HAS_INDEX_ELEMENT=${index_element_count} | ||
| 436 | + IL_EXPECT_SIMT=${expect_simt} | ||
| 437 | + IL_EXPECT_SK=${expect_sk}) | ||
| 438 | + mark_indirect_load_codegen_and_e2e(${test_name} ${case_definitions}) | ||
| 439 | +endfunction() | ||
| 440 | + | ||
| 441 | +# Mask bit d denotes that logical axis d has stride 0. The Element case covers both input and index zero-stride paths. | ||
| 442 | +# Direct zero-stride classification is covered by the layout and schedule UT. | ||
| 443 | +add_indirect_load_stride_zero_test(indirect_load_stride_zero_elements_simd_test simd 10 5 1 1) | ||
| 444 | +add_indirect_load_stride_zero_test(indirect_load_stride_zero_elements_simt_test simt 10 5 1 1) | ||
| 445 | +add_indirect_load_stride_zero_test(indirect_load_stride_zero_elements_sk_test sk 10 5 1 1) | ||
| 446 | + | ||
| 447 | +function(add_indirect_load_torch_gather_strided_test test_name tiling_key expect_simt expect_sk | ||
| 448 | + input_stride0 input_stride1 input_stride2 index_stride0 index_stride1 index_stride2) | ||
| 449 | + set(case_workdir ${CMAKE_CURRENT_BINARY_DIR}/${test_name}) | ||
| 450 | + file(MAKE_DIRECTORY ${case_workdir}) | ||
| 451 | + do_backend_e2e_st_test(${test_name} | ||
| 452 | + WORKDIR ${case_workdir} | ||
| 453 | + CODEGEN indirect_load_store_backend_generator.cpp | ||
| 454 | + TILING_KEY ${tiling_key} | ||
| 455 | + KERNEL_SRC | ||
| 456 | + indirect_load_torch_gather_strided_test_kernel.cpp | ||
| 457 | + indirect_load_torch_gather_strided_test_tiling.cpp | ||
| 458 | + autofuse_tiling_data.h | ||
| 459 | + TEST_SRC test_e2e_indirect_load_store_kernel.cpp) | ||
| 460 | + set(case_definitions | ||
| 461 | + IL_CASE_TORCH_STRIDED | ||
| 462 | + IL_EXPECT_TILING_KEY=${tiling_key} | ||
| 463 | + IL_EXPECT_SIMT=${expect_simt} | ||
| 464 | + IL_EXPECT_SK=${expect_sk} | ||
| 465 | + IL_INPUT_STRIDE0=${input_stride0} | ||
| 466 | + IL_INPUT_STRIDE1=${input_stride1} | ||
| 467 | + IL_INPUT_STRIDE2=${input_stride2} | ||
| 468 | + IL_INDEX_STRIDE0=${index_stride0} | ||
| 469 | + IL_INDEX_STRIDE1=${index_stride1} | ||
| 470 | + IL_INDEX_STRIDE2=${index_stride2}) | ||
| 471 | + mark_indirect_load_codegen_and_e2e(${test_name} ${case_definitions}) | ||
| 472 | +endfunction() | ||
| 473 | + | ||
| 474 | +# Cover non-contiguous layouts without a gap-kind/template Cartesian product: the inner-gap case also has an | ||
| 475 | +# outer gap, while the SIMT outer-gap case covers the pure outer-gap policy. Keep one SK runtime representative. | ||
| 476 | +add_indirect_load_torch_gather_strided_test(indirect_load_rank3_axis1_input_index_gap_simd_test | ||
| 477 | + 0 0 0 384 10 1 192 10 1) | ||
| 478 | +add_indirect_load_torch_gather_strided_test(indirect_load_rank3_axis1_input_index_gap_simt_test | ||
| 479 | + 0 1 0 384 10 1 192 10 1) | ||
| 480 | +add_indirect_load_torch_gather_strided_test(indirect_load_rank3_axis1_input_index_gap_sk_test | ||
| 481 | + 0 0 1 384 10 1 192 10 1) | ||
| 482 | +add_indirect_load_torch_gather_strided_test(indirect_load_rank3_axis1_input_index_outer_gap_simt_test | ||
| 483 | + 0 1 0 192 5 1 128 5 1) | ||
| 484 | +set(indirect_load_index_select_defs IL_INDEX_SELECT_CASE=1) | ||
| 485 | +mark_indirect_load_codegen_and_e2e(indirect_load_rank3_axis1_input_index_gap_simt_test | ||
| 486 | + ${indirect_load_index_select_defs}) | ||
| 487 | + | ||
| 290 | 488 | ||
| 291 | set(indirect_load_embedding_test_workdir ${CMAKE_CURRENT_BINARY_DIR}/indirect_load_embedding_test) | 489 | set(indirect_load_embedding_test_workdir ${CMAKE_CURRENT_BINARY_DIR}/indirect_load_embedding_test) |
| 292 | file(MAKE_DIRECTORY ${indirect_load_embedding_test_workdir}) | 490 | file(MAKE_DIRECTORY ${indirect_load_embedding_test_workdir}) |
| 293 | do_backend_e2e_st_test(indirect_load_embedding_test | 491 | do_backend_e2e_st_test(indirect_load_embedding_test |
| 294 | WORKDIR ${indirect_load_embedding_test_workdir} | 492 | WORKDIR ${indirect_load_embedding_test_workdir} |
| 295 | - CODEGEN indirect_load_embedding_backend_generator.cpp | 493 | + CODEGEN indirect_load_store_backend_generator.cpp |
| 296 | TILING_KEY 0 | 494 | TILING_KEY 0 |
| 297 | KERNEL_SRC | 495 | KERNEL_SRC |
| 298 | indirect_load_embedding_test_kernel.cpp | 496 | indirect_load_embedding_test_kernel.cpp |
| 299 | indirect_load_embedding_test_tiling.cpp | 497 | indirect_load_embedding_test_tiling.cpp |
| 300 | autofuse_tiling_data.h | 498 | autofuse_tiling_data.h |
| 301 | - TEST_SRC test_e2e_indirect_load_embedding_kernel.cpp) | 499 | + TEST_SRC test_e2e_indirect_load_store_kernel.cpp) |
| 500 | +mark_indirect_load_codegen_and_e2e(indirect_load_embedding_test IL_CASE_EMBEDDING) | ||
Dautofuse/tests/v35/st/backend_e2e_v2/indirect_load_store_test/indirect_load_backend_generator_common.h+0-270
| @@ -1,270 +0,0 @@ | |||
| 1 | -/** | ||
| 2 | - * Copyright (c) 2026 Huawei Technologies Co., Ltd. | ||
| 3 | - * This program is free software, you can redistribute it and/or modify it under the terms and conditions of | ||
| 4 | - * CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 5 | - * Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 6 | - * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 7 | - * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 8 | - * See LICENSE in the root of the software repository for the full text of the License. | ||
| 9 | - */ | ||
| 10 | - | ||
| 11 | - | ||
| 12 | - | ||
| 13 | - | ||
| 14 | - | ||
| 15 | - | ||
| 16 | - | ||
| 17 | - | ||
| 18 | - | ||
| 19 | - | ||
| 20 | - | ||
| 21 | - | ||
| 22 | - | ||
| 23 | - | ||
| 24 | - | ||
| 25 | - | ||
| 26 | - | ||
| 27 | - | ||
| 28 | - | ||
| 29 | - | ||
| 30 | - | ||
| 31 | - | ||
| 32 | - | ||
| 33 | - | ||
| 34 | - | ||
| 35 | - | ||
| 36 | - | ||
| 37 | - | ||
| 38 | -namespace indirect_load_test { | ||
| 39 | -constexpr char kTilingStub[] = R"( | ||
| 40 | - | ||
| 41 | - | ||
| 42 | -)"; | ||
| 43 | - | ||
| 44 | -template <typename Op> | ||
| 45 | -void SetView(Op &op, const std::vector<af::AxisId> &axes, const std::vector<af::Expression> &repeats, | ||
| 46 | - const std::vector<af::Expression> &strides, af::DataType dtype) { | ||
| 47 | - op.attr.sched.axis = axes; | ||
| 48 | - op.y.dtype = dtype; | ||
| 49 | - *op.y.axis = axes; | ||
| 50 | - *op.y.repeats = repeats; | ||
| 51 | - *op.y.strides = strides; | ||
| 52 | -} | ||
| 53 | - | ||
| 54 | -class BackendGraph { | ||
| 55 | - public: | ||
| 56 | - BackendGraph(const char *graph_name, const char *data_name, const char *index_name, af::DataType data_type) | ||
| 57 | - : fused_graph_(graph_name), data_name_(data_name), index_name_(index_name) { | ||
| 58 | - af::ascir_op::Data data(data_name, fused_graph_); | ||
| 59 | - data.ir_attr.SetIndex(0); | ||
| 60 | - af::ascir_op::Data index(index_name, fused_graph_); | ||
| 61 | - index.ir_attr.SetIndex(1); | ||
| 62 | - compute_graph_ = af::AscGraphUtils::GetComputeGraph(fused_graph_); | ||
| 63 | - if (compute_graph_ == nullptr) { | ||
| 64 | - return; | ||
| 65 | - } | ||
| 66 | - const auto data_desc = std::make_shared<af::GeTensorDesc>(); | ||
| 67 | - data_desc->SetDataType(data_type); | ||
| 68 | - const auto index_desc = std::make_shared<af::GeTensorDesc>(); | ||
| 69 | - index_desc->SetDataType(af::DT_INT64); | ||
| 70 | - const auto backend_desc = std::make_shared<af::OpDesc>("asc_backend", "AscBackend"); | ||
| 71 | - backend_desc->AddInputDesc(data_desc->Clone()); | ||
| 72 | - backend_desc->AddInputDesc(index_desc->Clone()); | ||
| 73 | - backend_desc->AddOutputDesc(data_desc->Clone()); | ||
| 74 | - backend_ = compute_graph_->AddNode(backend_desc); | ||
| 75 | - } | ||
| 76 | - | ||
| 77 | - [[nodiscard]] bool IsValid() const { | ||
| 78 | - return compute_graph_ != nullptr && backend_ != nullptr; | ||
| 79 | - } | ||
| 80 | - | ||
| 81 | - [[nodiscard]] af::ComputeGraphPtr Finalize(const std::shared_ptr<af::AscGraph> &sub_graph, | ||
| 82 | - const char *output_name) const { | ||
| 83 | - if (!IsValid()) { | ||
| 84 | - return nullptr; | ||
| 85 | - } | ||
| 86 | - const auto fuse_attrs = backend_->GetOpDesc()->GetOrCreateAttrsGroup<af::AutoFuseAttrs>(); | ||
| 87 | - if (fuse_attrs == nullptr) { | ||
| 88 | - return nullptr; | ||
| 89 | - } | ||
| 90 | - fuse_attrs->SetAscGraph(sub_graph); | ||
| 91 | - af::ascir_op::Output output(output_name); | ||
| 92 | - output.ir_attr.SetIndex(0); | ||
| 93 | - const auto output_node = compute_graph_->AddNode(af::OpDescUtils::GetOpDescFromOperator(output)); | ||
| 94 | - const auto data_node = fused_graph_.FindNode(data_name_.c_str()); | ||
| 95 | - const auto index_node = fused_graph_.FindNode(index_name_.c_str()); | ||
| 96 | - if (data_node == nullptr || index_node == nullptr || output_node == nullptr) { | ||
| 97 | - return nullptr; | ||
| 98 | - } | ||
| 99 | - const bool edges_added = | ||
| 100 | - af::GraphUtils::AddEdge(data_node->GetOutDataAnchor(0), backend_->GetInDataAnchor(0)) == ge::GRAPH_SUCCESS && | ||
| 101 | - af::GraphUtils::AddEdge(index_node->GetOutDataAnchor(0), backend_->GetInDataAnchor(1)) == ge::GRAPH_SUCCESS && | ||
| 102 | - af::GraphUtils::AddEdge(backend_->GetOutDataAnchor(0), output_node->GetInDataAnchor(0)) == ge::GRAPH_SUCCESS; | ||
| 103 | - return edges_added && compute_graph_->TopologicalSorting() == ge::GRAPH_SUCCESS ? compute_graph_ : nullptr; | ||
| 104 | - } | ||
| 105 | - | ||
| 106 | - private: | ||
| 107 | - af::AscGraph fused_graph_; | ||
| 108 | - std::string data_name_; | ||
| 109 | - std::string index_name_; | ||
| 110 | - af::ComputeGraphPtr compute_graph_; | ||
| 111 | - af::NodePtr backend_; | ||
| 112 | -}; | ||
| 113 | - | ||
| 114 | -inline bool ContainsTemplate(const ascir::ScheduledResult &candidate, ascir::TemplateId template_id) { | ||
| 115 | - for (const auto &group : candidate.schedule_groups) { | ||
| 116 | - for (const auto &graph : group.impl_graphs) { | ||
| 117 | - const auto indirect_load = ascgen_utils::indirect_load::FindIndirectLoadNode(graph); | ||
| 118 | - if (indirect_load != nullptr && ascir::GetTemplateIdOrDefault(*indirect_load) == template_id) { | ||
| 119 | - return true; | ||
| 120 | - } | ||
| 121 | - } | ||
| 122 | - } | ||
| 123 | - return false; | ||
| 124 | -} | ||
| 125 | - | ||
| 126 | -inline bool HasTemplate(const ascir::FusedScheduledResult &result, ascir::TemplateId template_id) { | ||
| 127 | - for (const auto &candidates : result.node_idx_to_scheduled_results) { | ||
| 128 | - for (const auto &candidate : candidates) { | ||
| 129 | - if (ContainsTemplate(candidate, template_id)) { | ||
| 130 | - return true; | ||
| 131 | - } | ||
| 132 | - } | ||
| 133 | - } | ||
| 134 | - return false; | ||
| 135 | -} | ||
| 136 | - | ||
| 137 | -inline void KeepOnlyTemplate(ascir::FusedScheduledResult &result, ascir::TemplateId template_id) { | ||
| 138 | - for (auto &candidates : result.node_idx_to_scheduled_results) { | ||
| 139 | - candidates.erase( | ||
| 140 | - std::remove_if(candidates.begin(), candidates.end(), | ||
| 141 | - [template_id](const auto &candidate) { return !ContainsTemplate(candidate, template_id); }), | ||
| 142 | - candidates.end()); | ||
| 143 | - } | ||
| 144 | -} | ||
| 145 | - | ||
| 146 | -inline ascir::TemplateId GetExpectedTemplate(bool expect_simt, bool expect_sk) { | ||
| 147 | - if (expect_sk) { | ||
| 148 | - return ascir::TemplateId::kIndirectLoadSK; | ||
| 149 | - } | ||
| 150 | - return expect_simt ? ascir::TemplateId::kIndirectLoadSimt : ascir::TemplateId::kIndirectLoadSimd; | ||
| 151 | -} | ||
| 152 | - | ||
| 153 | -inline const char *GetTemplateMarker(ascir::TemplateId template_id) { | ||
| 154 | - if (template_id == ascir::TemplateId::kIndirectLoadSK) { | ||
| 155 | - return "// IndirectLoad SK"; | ||
| 156 | - } | ||
| 157 | - return template_id == ascir::TemplateId::kIndirectLoadSimt ? "// IndirectLoad SIMT" : "// IndirectLoad SIMD"; | ||
| 158 | -} | ||
| 159 | - | ||
| 160 | -inline void BuildOutputPath(const std::shared_ptr<af::AscGraph> &graph, af::ascir_op::IndirectLoad &indirect_load, | ||
| 161 | - const std::vector<af::AxisId> &axes, const std::vector<af::Expression> &repeats, | ||
| 162 | - const std::vector<af::Expression> &strides, bool with_relu) { | ||
| 163 | - indirect_load.ir_attr.SetAxis(2); | ||
| 164 | - SetView(indirect_load, axes, repeats, strides, af::DT_FLOAT16); | ||
| 165 | - af::ascir_op::Store store("store"); | ||
| 166 | - graph->AddNode(store); | ||
| 167 | - if (with_relu) { | ||
| 168 | - af::ascir_op::Relu relu("output_relu"); | ||
| 169 | - graph->AddNode(relu); | ||
| 170 | - relu.x = indirect_load.y; | ||
| 171 | - SetView(relu, axes, repeats, strides, af::DT_FLOAT16); | ||
| 172 | - store.x = relu.y; | ||
| 173 | - } else { | ||
| 174 | - store.x = indirect_load.y; | ||
| 175 | - } | ||
| 176 | - SetView(store, axes, repeats, strides, af::DT_FLOAT16); | ||
| 177 | - af::ascir_op::Output output("y"); | ||
| 178 | - graph->AddNode(output); | ||
| 179 | - output.x = store.y; | ||
| 180 | - output.ir_attr.SetIndex(0); | ||
| 181 | - SetView(output, axes, repeats, strides, af::DT_FLOAT16); | ||
| 182 | -} | ||
| 183 | - | ||
| 184 | -template <typename GraphView, typename InputBuilder, typename IndexBuilder, typename OutputBuilder> | ||
| 185 | -std::shared_ptr<af::AscGraph> CreateSubGraph(GraphView view, InputBuilder build_input, IndexBuilder build_index, | ||
| 186 | - OutputBuilder build_output) { | ||
| 187 | - af::ascir_op::IndirectLoad indirect_load("indirect_load"); | ||
| 188 | - view.graph->AddNode(indirect_load); | ||
| 189 | - build_input(view, indirect_load); | ||
| 190 | - build_index(view, indirect_load); | ||
| 191 | - build_output(view, indirect_load); | ||
| 192 | - return view.graph; | ||
| 193 | -} | ||
| 194 | - | ||
| 195 | -inline bool SelectTemplate(const af::ComputeGraphPtr &graph, ascir::TemplateId expected_template, | ||
| 196 | - ascir::FusedScheduledResult &scheduled_result) { | ||
| 197 | - optimize::Optimizer optimizer(optimize::OptimizerOptions{.graph_type = optimize::GraphType::kFusedAscBackend}); | ||
| 198 | - if (optimizer.Optimize(graph, scheduled_result) != af::SUCCESS || !HasTemplate(scheduled_result, expected_template)) { | ||
| 199 | - return false; | ||
| 200 | - } | ||
| 201 | - KeepOnlyTemplate(scheduled_result, expected_template); | ||
| 202 | - return true; | ||
| 203 | -} | ||
| 204 | - | ||
| 205 | -inline void GenerateForTemplate(const af::ComputeGraphPtr &graph, const std::map<std::string, std::string> &shape_info, | ||
| 206 | - ascir::TemplateId expected_template, codegen::CodegenResult &result) { | ||
| 207 | - ascir::FusedScheduledResult scheduled_result; | ||
| 208 | - ASSERT_TRUE(SelectTemplate(graph, expected_template, scheduled_result)); | ||
| 209 | - codegen::Codegen codegen(codegen::CodegenOptions{}); | ||
| 210 | - ASSERT_EQ(codegen.Generate(shape_info, scheduled_result, result), af::SUCCESS); | ||
| 211 | -} | ||
| 212 | - | ||
| 213 | -inline bool WriteGeneratedFile(const std::string &path, const std::string &content) { | ||
| 214 | - std::fstream file(path, std::ios::out); | ||
| 215 | - if (!file.is_open()) { | ||
| 216 | - return false; | ||
| 217 | - } | ||
| 218 | - file << content; | ||
| 219 | - return file.good(); | ||
| 220 | -} | ||
| 221 | - | ||
| 222 | -inline void WriteGeneratedFiles(const codegen::CodegenResult &result) { | ||
| 223 | - const std::vector<std::string> parts = splitString(KERNEL_SRC_LIST, ':'); | ||
| 224 | - ASSERT_EQ(parts.size(), 3U); | ||
| 225 | - const std::array<std::string, 3> contents = {std::string(kTilingStub) + RemoveSubDirInclude(result.kernel), | ||
| 226 | - result.tiling, result.tiling_data}; | ||
| 227 | - for (size_t i = 0UL; i < parts.size(); ++i) { | ||
| 228 | - ASSERT_TRUE(WriteGeneratedFile(parts[i], contents[i])) << parts[i]; | ||
| 229 | - } | ||
| 230 | -} | ||
| 231 | - | ||
| 232 | -inline void SetUpBackendRuntime() { | ||
| 233 | - dlog_setlevel(ASCGEN_MODULE_NAME, DLOG_ERROR, 0); | ||
| 234 | - ge::PlatformContext::GetInstance().Reset(); | ||
| 235 | - ge::RuntimeStub::SetInstance(std::make_shared<af::RuntimeStubV2>()); | ||
| 236 | -} | ||
| 237 | - | ||
| 238 | -inline void TearDownBackendRuntime() { | ||
| 239 | - dlog_setlevel(ASCGEN_MODULE_NAME, DLOG_ERROR, 0); | ||
| 240 | - ge::RuntimeStub::Reset(); | ||
| 241 | -} | ||
| 242 | - | ||
| 243 | -class BackendE2e : public testing::Test { | ||
| 244 | - protected: | ||
| 245 | - void SetUp() override { | ||
| 246 | - SetUpBackendRuntime(); | ||
| 247 | - } | ||
| 248 | - | ||
| 249 | - void TearDown() override { | ||
| 250 | - TearDownBackendRuntime(); | ||
| 251 | - } | ||
| 252 | -}; | ||
| 253 | - | ||
| 254 | -class PrecisionBackendE2e : public BackendE2e { | ||
| 255 | - protected: | ||
| 256 | - void SetUp() override { | ||
| 257 | - setenv("AUTOFUSE_FLAGS", "--autofuse_enhance_precision_blacklist=all", 1); | ||
| 258 | - af::pre_process::PreProcessConfig::Instance().Reset(); | ||
| 259 | - BackendE2e::SetUp(); | ||
| 260 | - } | ||
| 261 | - | ||
| 262 | - void TearDown() override { | ||
| 263 | - unsetenv("AUTOFUSE_FLAGS"); | ||
| 264 | - af::pre_process::PreProcessConfig::Instance().Reset(); | ||
| 265 | - BackendE2e::TearDown(); | ||
| 266 | - } | ||
| 267 | -}; | ||
| 268 | -} // namespace indirect_load_test | ||
| 269 | - | ||
| 270 | - | ||
Dautofuse/tests/v35/st/backend_e2e_v2/indirect_load_store_test/indirect_load_broadcast_backend_generator.cpp+0-966
| @@ -1,966 +0,0 @@ | |||
| 1 | -/** | ||
| 2 | - * Copyright (c) 2026 Huawei Technologies Co., Ltd. | ||
| 3 | - * This program is free software, you can redistribute it and/or modify it under the terms and conditions of | ||
| 4 | - * CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 5 | - * Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 6 | - * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 7 | - * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 8 | - * See LICENSE in the root of the software repository for the full text of the License. | ||
| 9 | - */ | ||
| 10 | - | ||
| 11 | - | ||
| 12 | - | ||
| 13 | - | ||
| 14 | - | ||
| 15 | - | ||
| 16 | - | ||
| 17 | - | ||
| 18 | - | ||
| 19 | - | ||
| 20 | - | ||
| 21 | - | ||
| 22 | - | ||
| 23 | - | ||
| 24 | - | ||
| 25 | - | ||
| 26 | - | ||
| 27 | - | ||
| 28 | - | ||
| 29 | - | ||
| 30 | - | ||
| 31 | - | ||
| 32 | - | ||
| 33 | - | ||
| 34 | - | ||
| 35 | - | ||
| 36 | - | ||
| 37 | - | ||
| 38 | - | ||
| 39 | - | ||
| 40 | - | ||
| 41 | - | ||
| 42 | - | ||
| 43 | - | ||
| 44 | - | ||
| 45 | - | ||
| 46 | - | ||
| 47 | - | ||
| 48 | - | ||
| 49 | - | ||
| 50 | - | ||
| 51 | - | ||
| 52 | - | ||
| 53 | - | ||
| 54 | - | ||
| 55 | - | ||
| 56 | - | ||
| 57 | - | ||
| 58 | - | ||
| 59 | - | ||
| 60 | - | ||
| 61 | - | ||
| 62 | - | ||
| 63 | - | ||
| 64 | - | ||
| 65 | - | ||
| 66 | - | ||
| 67 | - | ||
| 68 | - | ||
| 69 | - | ||
| 70 | - | ||
| 71 | - | ||
| 72 | - | ||
| 73 | - | ||
| 74 | - | ||
| 75 | - | ||
| 76 | - | ||
| 77 | -namespace { | ||
| 78 | -constexpr int64_t kOutputS0 = IL_OUTPUT_S0; | ||
| 79 | -constexpr int64_t kOutputS1 = IL_OUTPUT_S1; | ||
| 80 | -constexpr int64_t kOutputS2 = IL_OUTPUT_S2; | ||
| 81 | -constexpr int64_t kOutputS3 = IL_OUTPUT_S3; | ||
| 82 | -constexpr std::array<int64_t, 4> kOutputShape = {kOutputS0, kOutputS1, kOutputS2, kOutputS3}; | ||
| 83 | -constexpr bool kComplexBroadcast = IL_COMPLEX_BROADCAST; | ||
| 84 | -constexpr bool kComplexSimt = IL_COMPLEX_SIMT; | ||
| 85 | -constexpr bool kComplexInputBroadcast = IL_COMPLEX_INPUT_BROADCAST; | ||
| 86 | -constexpr bool kComplexIndexBroadcast = IL_COMPLEX_INDEX_BROADCAST; | ||
| 87 | -constexpr bool kIndexBinarySameView = IL_INDEX_BINARY_SAME_VIEW; | ||
| 88 | -constexpr bool kIndexAbsDenseView = IL_INDEX_ABS_DENSE_VIEW; | ||
| 89 | -constexpr int32_t kBinaryElementKind = IL_BINARY_ELEMENT_KIND; | ||
| 90 | -constexpr bool kRetainBroadcast = IL_RETAIN_BROADCAST; | ||
| 91 | -constexpr bool kDegenerateBroadcast = IL_DEGENERATE_BROADCAST; | ||
| 92 | -constexpr bool kContinuousBroadcast = IL_CONTINUOUS_BROADCAST; | ||
| 93 | -constexpr bool kContinuousIndexBroadcast = IL_CONTINUOUS_INDEX_BROADCAST; | ||
| 94 | -constexpr int32_t kInputElementCount = IL_HAS_INPUT_ELEMENT; | ||
| 95 | -constexpr int32_t kIndexElementCount = IL_HAS_INDEX_ELEMENT; | ||
| 96 | -constexpr bool kHasOutputRelu = IL_HAS_OUTPUT_RELU; | ||
| 97 | -constexpr bool kInputBroadcast = IL_INPUT_BROADCAST && !kComplexBroadcast && !kComplexInputBroadcast; | ||
| 98 | -constexpr bool kIndexBroadcast = IL_INDEX_BROADCAST; | ||
| 99 | -constexpr uint32_t kBroadcastAxesMask = IL_BROADCAST_AXES_MASK; | ||
| 100 | -constexpr bool kClearBroadcastSourceView = IL_CLEAR_BROADCAST_SOURCE_VIEW; | ||
| 101 | -constexpr bool kExpectSimt = IL_EXPECT_SIMT; | ||
| 102 | -constexpr bool kExpectSk = IL_EXPECT_SK; | ||
| 103 | -constexpr bool kAicRepro = IL_AIC_REPRO; | ||
| 104 | - | ||
| 105 | -using indirect_load_test::SetView; | ||
| 106 | - | ||
| 107 | -struct TensorView { | ||
| 108 | - std::vector<af::AxisId> axes; | ||
| 109 | - std::vector<af::Expression> repeats; | ||
| 110 | - std::vector<af::Expression> strides; | ||
| 111 | - af::DataType dtype; | ||
| 112 | -}; | ||
| 113 | - | ||
| 114 | -struct BroadcastGraphView { | ||
| 115 | - std::shared_ptr<af::AscGraph> graph; | ||
| 116 | - TensorView input; | ||
| 117 | - TensorView output; | ||
| 118 | - TensorView input_source; | ||
| 119 | - TensorView index_source; | ||
| 120 | - TensorView input_broadcast; | ||
| 121 | - TensorView input_intermediate_broadcast; | ||
| 122 | - TensorView index_broadcast; | ||
| 123 | - TensorView index_intermediate_broadcast; | ||
| 124 | -}; | ||
| 125 | - | ||
| 126 | -template <typename Op> | ||
| 127 | -void SetView(Op &op, const TensorView &view) { | ||
| 128 | - SetView(op, view.axes, view.repeats, view.strides, view.dtype); | ||
| 129 | -} | ||
| 130 | - | ||
| 131 | -template <typename Op> | ||
| 132 | -void ClearView(Op &op) { | ||
| 133 | - op.y.axis->clear(); | ||
| 134 | - op.y.repeats->clear(); | ||
| 135 | - op.y.strides->clear(); | ||
| 136 | -} | ||
| 137 | - | ||
| 138 | -std::vector<af::Expression> MakeDenseStrides(const std::vector<af::Expression> &repeats) { | ||
| 139 | - std::vector<af::Expression> strides(repeats.size(), af::ops::One); | ||
| 140 | - af::Expression stride = af::ops::One; | ||
| 141 | - for (size_t index = repeats.size(); index > 0UL; --index) { | ||
| 142 | - const size_t dim = index - 1UL; | ||
| 143 | - strides[dim] = stride; | ||
| 144 | - stride = stride * repeats[dim]; | ||
| 145 | - } | ||
| 146 | - return strides; | ||
| 147 | -} | ||
| 148 | - | ||
| 149 | -TensorView MakeIntermediateBroadcastView(const TensorView &broadcast_view, const TensorView &logical_view) { | ||
| 150 | - constexpr size_t kFirstBroadcastAxis = 3UL; | ||
| 151 | - TensorView view = broadcast_view; | ||
| 152 | - view.repeats[kFirstBroadcastAxis] = logical_view.repeats[kFirstBroadcastAxis]; | ||
| 153 | - view.strides = MakeDenseStrides(view.repeats); | ||
| 154 | - for (size_t dim = 0UL; dim < view.strides.size(); ++dim) { | ||
| 155 | - if ((kBroadcastAxesMask & (1U << dim)) != 0U && | ||
| 156 | - af::SymbolicUtils::StaticCheckEq(view.repeats[dim], af::ops::One) == af::TriBool::kTrue) { | ||
| 157 | - view.strides[dim] = af::ops::Zero; | ||
| 158 | - } | ||
| 159 | - } | ||
| 160 | - return view; | ||
| 161 | -} | ||
| 162 | - | ||
| 163 | -std::array<int64_t, 4> MakeLogicalStrides(bool broadcast) { | ||
| 164 | - std::array<int64_t, 4> strides{}; | ||
| 165 | - int64_t stride = 1; | ||
| 166 | - for (size_t index = kOutputShape.size(); index > 0UL; --index) { | ||
| 167 | - const size_t dim = index - 1UL; | ||
| 168 | - const bool is_broadcast_axis = broadcast && (kBroadcastAxesMask & (1U << dim)) != 0U; | ||
| 169 | - strides[dim] = is_broadcast_axis ? 0 : stride; | ||
| 170 | - if (!is_broadcast_axis) { | ||
| 171 | - stride *= kOutputShape[dim]; | ||
| 172 | - } | ||
| 173 | - } | ||
| 174 | - return strides; | ||
| 175 | -} | ||
| 176 | - | ||
| 177 | -std::string MakeShapeArgs(bool is_sk) { | ||
| 178 | - const auto input_strides = MakeLogicalStrides(kInputBroadcast && !kDegenerateBroadcast); | ||
| 179 | - const auto index_strides = MakeLogicalStrides(kIndexBroadcast && !kDegenerateBroadcast); | ||
| 180 | - std::ostringstream stream; | ||
| 181 | - if (is_sk) { | ||
| 182 | - stream << ", 4"; | ||
| 183 | - } | ||
| 184 | - for (const int64_t size : kOutputShape) { | ||
| 185 | - stream << ", " << size; | ||
| 186 | - } | ||
| 187 | - for (const int64_t stride : input_strides) { | ||
| 188 | - stream << ", " << stride; | ||
| 189 | - } | ||
| 190 | - for (const int64_t stride : index_strides) { | ||
| 191 | - stream << ", " << stride; | ||
| 192 | - } | ||
| 193 | - stream << ");"; | ||
| 194 | - return stream.str(); | ||
| 195 | -} | ||
| 196 | - | ||
| 197 | -BroadcastGraphView CreateGraphView() { | ||
| 198 | - BroadcastGraphView view; | ||
| 199 | - view.graph = std::make_shared<af::AscGraph>("indirect_load_broadcast_test"); | ||
| 200 | - std::vector<af::AxisId> input_source_axis_candidates; | ||
| 201 | - std::vector<af::AxisId> index_source_axis_candidates; | ||
| 202 | - for (size_t dim = 0UL; dim < kOutputShape.size(); ++dim) { | ||
| 203 | - const auto input_size = view.graph->CreateSizeVar(kOutputShape[dim]); | ||
| 204 | - const auto output_size = view.graph->CreateSizeVar(kOutputShape[dim]); | ||
| 205 | - view.input.repeats.emplace_back(input_size); | ||
| 206 | - view.output.repeats.emplace_back(output_size); | ||
| 207 | - view.input.axes.emplace_back(view.graph->CreateAxis(("z" + std::to_string(dim)).c_str(), input_size).id); | ||
| 208 | - view.output.axes.emplace_back(view.graph->CreateAxis(("z" + std::to_string(dim + 4UL)).c_str(), output_size).id); | ||
| 209 | - input_source_axis_candidates.emplace_back( | ||
| 210 | - view.graph->CreateAxis(("z" + std::to_string(dim) + "_input").c_str(), af::ops::One).id); | ||
| 211 | - index_source_axis_candidates.emplace_back( | ||
| 212 | - view.graph->CreateAxis(("z" + std::to_string(dim + 4UL) + "_index").c_str(), af::ops::One).id); | ||
| 213 | - } | ||
| 214 | - view.input.dtype = af::DT_FLOAT16; | ||
| 215 | - view.output.dtype = af::DT_INT64; | ||
| 216 | - view.input_source = view.input; | ||
| 217 | - view.index_source = view.output; | ||
| 218 | - for (size_t dim = 0UL; dim < kOutputShape.size(); ++dim) { | ||
| 219 | - if ((kBroadcastAxesMask & (1U << dim)) != 0U) { | ||
| 220 | - view.input_source.axes[dim] = input_source_axis_candidates[dim]; | ||
| 221 | - view.index_source.axes[dim] = index_source_axis_candidates[dim]; | ||
| 222 | - view.input_source.repeats[dim] = af::ops::One; | ||
| 223 | - view.index_source.repeats[dim] = af::ops::One; | ||
| 224 | - } | ||
| 225 | - } | ||
| 226 | - if constexpr (kIndexAbsDenseView) { | ||
| 227 | - view.index_source.axes = view.output.axes; | ||
| 228 | - } | ||
| 229 | - view.input.strides = MakeDenseStrides(view.input.repeats); | ||
| 230 | - view.output.strides = MakeDenseStrides(view.output.repeats); | ||
| 231 | - view.input_source.strides = MakeDenseStrides(view.input_source.repeats); | ||
| 232 | - view.index_source.strides = MakeDenseStrides(view.index_source.repeats); | ||
| 233 | - view.input_broadcast = view.input; | ||
| 234 | - view.index_broadcast = view.output; | ||
| 235 | - for (size_t dim = 0UL; dim < kOutputShape.size(); ++dim) { | ||
| 236 | - view.input_broadcast.strides[dim] = | ||
| 237 | - (kBroadcastAxesMask & (1U << dim)) == 0U ? view.input_source.strides[dim] : af::ops::Zero; | ||
| 238 | - view.index_broadcast.strides[dim] = | ||
| 239 | - (kBroadcastAxesMask & (1U << dim)) == 0U ? view.index_source.strides[dim] : af::ops::Zero; | ||
| 240 | - } | ||
| 241 | - view.input_intermediate_broadcast = view.input_broadcast; | ||
| 242 | - if constexpr (kContinuousBroadcast) { | ||
| 243 | - view.input_intermediate_broadcast = MakeIntermediateBroadcastView(view.input_broadcast, view.input); | ||
| 244 | - } | ||
| 245 | - view.index_intermediate_broadcast = view.index_broadcast; | ||
| 246 | - if constexpr (kContinuousIndexBroadcast) { | ||
| 247 | - view.index_intermediate_broadcast = MakeIntermediateBroadcastView(view.index_broadcast, view.output); | ||
| 248 | - } | ||
| 249 | - return view; | ||
| 250 | -} | ||
| 251 | - | ||
| 252 | -template <typename Destination> | ||
| 253 | -void ConnectAbsChain(const std::shared_ptr<af::AscGraph> &graph, const char *prefix, int32_t count, | ||
| 254 | - const af::AscOpOutput &source, const TensorView &view, Destination &destination) { | ||
| 255 | - std::vector<std::unique_ptr<af::ascir_op::Abs>> elements; | ||
| 256 | - for (int32_t i = 0; i < count; ++i) { | ||
| 257 | - const auto name = std::string(prefix) + std::to_string(i); | ||
| 258 | - auto element = std::make_unique<af::ascir_op::Abs>(name.c_str()); | ||
| 259 | - graph->AddNode(*element); | ||
| 260 | - element->attr.api.compute_type = af::ComputeType::kComputeElewise; | ||
| 261 | - element->x = elements.empty() ? source : elements.back()->y; | ||
| 262 | - SetView(*element, view); | ||
| 263 | - elements.emplace_back(std::move(element)); | ||
| 264 | - } | ||
| 265 | - destination = elements.empty() ? source : elements.back()->y; | ||
| 266 | -} | ||
| 267 | - | ||
| 268 | -af::AscOpOutput ConnectBinaryElement(const std::shared_ptr<af::AscGraph> &graph, const char *name, | ||
| 269 | - const af::AscOpOutput &lhs, const af::AscOpOutput &rhs, const TensorView &view) { | ||
| 270 | - if constexpr (kBinaryElementKind == 1) { | ||
| 271 | - af::ascir_op::Mul element(name); | ||
| 272 | - graph->AddNode(element); | ||
| 273 | - element.x1 = lhs; | ||
| 274 | - element.x2 = rhs; | ||
| 275 | - element.attr.api.compute_type = af::ComputeType::kComputeElewise; | ||
| 276 | - SetView(element, view); | ||
| 277 | - return element.y; | ||
| 278 | - } else if constexpr (kBinaryElementKind == 2) { | ||
| 279 | - af::ascir_op::Sub element(name); | ||
| 280 | - graph->AddNode(element); | ||
| 281 | - element.x1 = lhs; | ||
| 282 | - element.x2 = rhs; | ||
| 283 | - element.attr.api.compute_type = af::ComputeType::kComputeElewise; | ||
| 284 | - SetView(element, view); | ||
| 285 | - return element.y; | ||
| 286 | - } else if constexpr (kBinaryElementKind == 3) { | ||
| 287 | - af::ascir_op::Maximum element(name); | ||
| 288 | - graph->AddNode(element); | ||
| 289 | - element.x1 = lhs; | ||
| 290 | - element.x2 = rhs; | ||
| 291 | - element.attr.api.compute_type = af::ComputeType::kComputeElewise; | ||
| 292 | - SetView(element, view); | ||
| 293 | - return element.y; | ||
| 294 | - } else { | ||
| 295 | - af::ascir_op::Add element(name); | ||
| 296 | - graph->AddNode(element); | ||
| 297 | - element.x1 = lhs; | ||
| 298 | - element.x2 = rhs; | ||
| 299 | - element.attr.api.compute_type = af::ComputeType::kComputeElewise; | ||
| 300 | - SetView(element, view); | ||
| 301 | - return element.y; | ||
| 302 | - } | ||
| 303 | -} | ||
| 304 | - | ||
| 305 | -void BuildInputPath(const BroadcastGraphView &view, af::ascir_op::IndirectLoad &indirect_load) { | ||
| 306 | - if constexpr (kComplexSimt) { | ||
| 307 | - af::ascir_op::Data x("x"); | ||
| 308 | - af::ascir_op::Load input_load("input_load"); | ||
| 309 | - view.graph->AddNode(x); | ||
| 310 | - view.graph->AddNode(input_load); | ||
| 311 | - x.ir_attr.SetIndex(0); | ||
| 312 | - input_load.x = x.y; | ||
| 313 | - SetView(x, view.input); | ||
| 314 | - SetView(input_load, view.input); | ||
| 315 | - indirect_load.x1 = input_load.y; | ||
| 316 | - return; | ||
| 317 | - } | ||
| 318 | - constexpr bool use_broadcast = kInputBroadcast || kComplexBroadcast; | ||
| 319 | - af::ascir_op::Data x("x"); | ||
| 320 | - view.graph->AddNode(x); | ||
| 321 | - x.ir_attr.SetIndex(0); | ||
| 322 | - SetView(x, use_broadcast ? view.input_source : view.input); | ||
| 323 | - if (kClearBroadcastSourceView && kInputBroadcast) { | ||
| 324 | - ClearView(x); | ||
| 325 | - } | ||
| 326 | - af::ascir_op::Load input_load("input_load"); | ||
| 327 | - view.graph->AddNode(input_load); | ||
| 328 | - input_load.x = x.y; | ||
| 329 | - SetView(input_load, use_broadcast ? view.input_source : view.input); | ||
| 330 | - if (use_broadcast) { | ||
| 331 | - if constexpr (kContinuousBroadcast) { | ||
| 332 | - af::ascir_op::Broadcast first_broadcast("input_first_broadcast"); | ||
| 333 | - af::ascir_op::Broadcast second_broadcast("input_second_broadcast"); | ||
| 334 | - view.graph->AddNode(first_broadcast); | ||
| 335 | - view.graph->AddNode(second_broadcast); | ||
| 336 | - first_broadcast.attr.api.compute_type = af::ComputeType::kComputeBroadcast; | ||
| 337 | - second_broadcast.attr.api.compute_type = af::ComputeType::kComputeBroadcast; | ||
| 338 | - first_broadcast.x = input_load.y; | ||
| 339 | - second_broadcast.x = first_broadcast.y; | ||
| 340 | - SetView(first_broadcast, view.input_intermediate_broadcast); | ||
| 341 | - SetView(second_broadcast, view.input_broadcast); | ||
| 342 | - ConnectAbsChain(view.graph, "input_abs_", kInputElementCount, second_broadcast.y, view.input_broadcast, | ||
| 343 | - indirect_load.x1); | ||
| 344 | - return; | ||
| 345 | - } | ||
| 346 | - af::ascir_op::Broadcast broadcast("input_broadcast"); | ||
| 347 | - view.graph->AddNode(broadcast); | ||
| 348 | - broadcast.attr.api.compute_type = af::ComputeType::kComputeBroadcast; | ||
| 349 | - broadcast.x = input_load.y; | ||
| 350 | - SetView(broadcast, view.input_broadcast); | ||
| 351 | - if constexpr (kComplexBroadcast) { | ||
| 352 | - af::ascir_op::Add input_add("input_add"); | ||
| 353 | - view.graph->AddNode(input_add); | ||
| 354 | - input_add.x1 = broadcast.y; | ||
| 355 | - input_add.x2 = input_load.y; | ||
| 356 | - SetView(input_add, view.input_broadcast); | ||
| 357 | - indirect_load.x1 = input_add.y; | ||
| 358 | - } else { | ||
| 359 | - ConnectAbsChain(view.graph, "input_abs_", kInputElementCount, broadcast.y, view.input_broadcast, | ||
| 360 | - indirect_load.x1); | ||
| 361 | - } | ||
| 362 | - } else { | ||
| 363 | - indirect_load.x1 = input_load.y; | ||
| 364 | - } | ||
| 365 | -} | ||
| 366 | - | ||
| 367 | -void BuildIndexPath(const BroadcastGraphView &view, af::ascir_op::IndirectLoad &indirect_load) { | ||
| 368 | - af::ascir_op::Data index("index"); | ||
| 369 | - view.graph->AddNode(index); | ||
| 370 | - index.ir_attr.SetIndex(1); | ||
| 371 | - SetView(index, kIndexBroadcast ? view.index_source : view.output); | ||
| 372 | - if (kClearBroadcastSourceView && kIndexBroadcast) { | ||
| 373 | - ClearView(index); | ||
| 374 | - } | ||
| 375 | - af::ascir_op::Load index_load("index_load"); | ||
| 376 | - view.graph->AddNode(index_load); | ||
| 377 | - index_load.x = index.y; | ||
| 378 | - SetView(index_load, kIndexBroadcast ? view.index_source : view.output); | ||
| 379 | - if (kIndexBroadcast) { | ||
| 380 | - if constexpr (kContinuousIndexBroadcast) { | ||
| 381 | - af::ascir_op::Broadcast first_broadcast("index_first_broadcast"); | ||
| 382 | - af::ascir_op::Broadcast second_broadcast("index_second_broadcast"); | ||
| 383 | - view.graph->AddNode(first_broadcast); | ||
| 384 | - view.graph->AddNode(second_broadcast); | ||
| 385 | - first_broadcast.attr.api.compute_type = af::ComputeType::kComputeBroadcast; | ||
| 386 | - second_broadcast.attr.api.compute_type = af::ComputeType::kComputeBroadcast; | ||
| 387 | - first_broadcast.x = index_load.y; | ||
| 388 | - second_broadcast.x = first_broadcast.y; | ||
| 389 | - SetView(first_broadcast, view.index_intermediate_broadcast); | ||
| 390 | - SetView(second_broadcast, view.index_broadcast); | ||
| 391 | - ConnectAbsChain(view.graph, "index_abs_", kIndexElementCount, second_broadcast.y, view.index_broadcast, | ||
| 392 | - indirect_load.x2); | ||
| 393 | - return; | ||
| 394 | - } | ||
| 395 | - af::ascir_op::Broadcast broadcast("index_broadcast"); | ||
| 396 | - view.graph->AddNode(broadcast); | ||
| 397 | - broadcast.attr.api.compute_type = af::ComputeType::kComputeBroadcast; | ||
| 398 | - broadcast.x = index_load.y; | ||
| 399 | - SetView(broadcast, kIndexAbsDenseView ? view.output : view.index_broadcast); | ||
| 400 | - const TensorView &index_element_view = kIndexAbsDenseView ? view.output : view.index_broadcast; | ||
| 401 | - ConnectAbsChain(view.graph, "index_abs_", kIndexElementCount, broadcast.y, index_element_view, indirect_load.x2); | ||
| 402 | - } else { | ||
| 403 | - ConnectAbsChain(view.graph, "index_abs_", kIndexElementCount, index_load.y, view.output, indirect_load.x2); | ||
| 404 | - } | ||
| 405 | -} | ||
| 406 | - | ||
| 407 | -void BuildOutputPath(const BroadcastGraphView &view, af::ascir_op::IndirectLoad &indirect_load) { | ||
| 408 | - | ||
| 409 | - af::ascir_op::Sum sum("output_sum"); | ||
| 410 | - af::ascir_op::Store store("store"); | ||
| 411 | - af::ascir_op::Output output("y"); | ||
| 412 | - view.graph->AddNode(sum); | ||
| 413 | - view.graph->AddNode(store); | ||
| 414 | - view.graph->AddNode(output); | ||
| 415 | - indirect_load.ir_attr.SetAxis(2); | ||
| 416 | - SetView(indirect_load, view.output.axes, view.output.repeats, view.output.strides, af::DT_FLOAT16); | ||
| 417 | - sum.attr.api.compute_type = af::ComputeType::kComputeReduce; | ||
| 418 | - sum.attr.sched.axis = view.output.axes; | ||
| 419 | - sum.x = indirect_load.y; | ||
| 420 | - auto reduce_repeats = view.output.repeats; | ||
| 421 | - auto reduce_strides = view.output.strides; | ||
| 422 | - reduce_repeats[2] = af::ops::One; | ||
| 423 | - reduce_repeats[3] = af::ops::One; | ||
| 424 | - reduce_strides[0] = view.output.repeats[1]; | ||
| 425 | - reduce_strides[1] = af::ops::One; | ||
| 426 | - reduce_strides[2] = af::ops::Zero; | ||
| 427 | - reduce_strides[3] = af::ops::Zero; | ||
| 428 | - SetView(sum, view.output.axes, reduce_repeats, reduce_strides, af::DT_FLOAT16); | ||
| 429 | - store.x = sum.y; | ||
| 430 | - SetView(store, view.output.axes, reduce_repeats, reduce_strides, af::DT_FLOAT16); | ||
| 431 | - output.x = store.y; | ||
| 432 | - output.ir_attr.SetIndex(0); | ||
| 433 | - SetView(output, view.output.axes, reduce_repeats, reduce_strides, af::DT_FLOAT16); | ||
| 434 | - return; | ||
| 435 | - | ||
| 436 | - if constexpr (kRetainBroadcast) { | ||
| 437 | - TensorView source_view = view.output; | ||
| 438 | - source_view.dtype = af::DT_FLOAT16; | ||
| 439 | - for (size_t dim = 0UL; dim < source_view.repeats.size(); ++dim) { | ||
| 440 | - if ((kBroadcastAxesMask & (1U << dim)) != 0U) { | ||
| 441 | - source_view.repeats[dim] = af::ops::One; | ||
| 442 | - } | ||
| 443 | - } | ||
| 444 | - source_view.strides = MakeDenseStrides(source_view.repeats); | ||
| 445 | - af::ascir_op::Scalar source("output_source", *view.graph); | ||
| 446 | - source.ir_attr.SetValue("1.5"); | ||
| 447 | - source.y.dtype = af::DT_FLOAT16; | ||
| 448 | - af::ascir_op::Abs source_abs("output_source_abs"); | ||
| 449 | - af::ascir_op::Broadcast broadcast("output_retained_broadcast"); | ||
| 450 | - af::ascir_op::Add output_add("output_add"); | ||
| 451 | - af::ascir_op::Store store("store"); | ||
| 452 | - af::ascir_op::Output output("y"); | ||
| 453 | - view.graph->AddNode(source_abs); | ||
| 454 | - view.graph->AddNode(broadcast); | ||
| 455 | - view.graph->AddNode(output_add); | ||
| 456 | - view.graph->AddNode(store); | ||
| 457 | - view.graph->AddNode(output); | ||
| 458 | - source_abs.x = source.y; | ||
| 459 | - broadcast.x = source_abs.y; | ||
| 460 | - broadcast.attr.api.compute_type = af::ComputeType::kComputeBroadcast; | ||
| 461 | - output_add.x1 = indirect_load.y; | ||
| 462 | - output_add.x2 = broadcast.y; | ||
| 463 | - indirect_load.ir_attr.SetAxis(2); | ||
| 464 | - SetView(indirect_load, view.output.axes, view.output.repeats, view.output.strides, af::DT_FLOAT16); | ||
| 465 | - SetView(source_abs, source_view); | ||
| 466 | - SetView(broadcast, view.output.axes, view.output.repeats, view.output.strides, af::DT_FLOAT16); | ||
| 467 | - SetView(output_add, view.output.axes, view.output.repeats, view.output.strides, af::DT_FLOAT16); | ||
| 468 | - store.x = output_add.y; | ||
| 469 | - SetView(store, view.output.axes, view.output.repeats, view.output.strides, af::DT_FLOAT16); | ||
| 470 | - output.x = store.y; | ||
| 471 | - output.ir_attr.SetIndex(0); | ||
| 472 | - SetView(output, view.output.axes, view.output.repeats, view.output.strides, af::DT_FLOAT16); | ||
| 473 | - return; | ||
| 474 | - } | ||
| 475 | - if constexpr (kComplexBroadcast) { | ||
| 476 | - af::ascir_op::Scalar scalar0("output_scalar0", *view.graph); | ||
| 477 | - af::ascir_op::Scalar scalar1("output_scalar1", *view.graph); | ||
| 478 | - scalar0.ir_attr.SetValue("0.5"); | ||
| 479 | - scalar1.ir_attr.SetValue("1.0"); | ||
| 480 | - scalar0.y.dtype = af::DT_FLOAT16; | ||
| 481 | - scalar1.y.dtype = af::DT_FLOAT16; | ||
| 482 | - af::ascir_op::Broadcast broadcast0("output_broadcast0"); | ||
| 483 | - af::ascir_op::Broadcast broadcast1("output_broadcast1"); | ||
| 484 | - af::ascir_op::Add scalar_add("output_scalar_add"); | ||
| 485 | - af::ascir_op::Add output_add("output_add"); | ||
| 486 | - af::ascir_op::Store store("store"); | ||
| 487 | - af::ascir_op::Output output("y"); | ||
| 488 | - for (af::ascir_op::Broadcast *broadcast : {&broadcast0, &broadcast1}) { | ||
| 489 | - view.graph->AddNode(*broadcast); | ||
| 490 | - broadcast->attr.api.compute_type = af::ComputeType::kComputeBroadcast; | ||
| 491 | - SetView(*broadcast, view.output.axes, view.output.repeats, view.output.strides, af::DT_FLOAT16); | ||
| 492 | - } | ||
| 493 | - view.graph->AddNode(scalar_add); | ||
| 494 | - view.graph->AddNode(output_add); | ||
| 495 | - view.graph->AddNode(store); | ||
| 496 | - view.graph->AddNode(output); | ||
| 497 | - broadcast0.x = scalar0.y; | ||
| 498 | - broadcast1.x = scalar1.y; | ||
| 499 | - scalar_add.x1 = broadcast0.y; | ||
| 500 | - scalar_add.x2 = broadcast1.y; | ||
| 501 | - output_add.x1 = indirect_load.y; | ||
| 502 | - output_add.x2 = scalar_add.y; | ||
| 503 | - indirect_load.ir_attr.SetAxis(2); | ||
| 504 | - SetView(indirect_load, view.output.axes, view.output.repeats, view.output.strides, af::DT_FLOAT16); | ||
| 505 | - SetView(scalar_add, view.output.axes, view.output.repeats, view.output.strides, af::DT_FLOAT16); | ||
| 506 | - SetView(output_add, view.output.axes, view.output.repeats, view.output.strides, af::DT_FLOAT16); | ||
| 507 | - store.x = output_add.y; | ||
| 508 | - SetView(store, view.output.axes, view.output.repeats, view.output.strides, af::DT_FLOAT16); | ||
| 509 | - output.x = store.y; | ||
| 510 | - output.ir_attr.SetIndex(0); | ||
| 511 | - SetView(output, view.output.axes, view.output.repeats, view.output.strides, af::DT_FLOAT16); | ||
| 512 | - return; | ||
| 513 | - } | ||
| 514 | - indirect_load_test::BuildOutputPath(view.graph, indirect_load, view.output.axes, view.output.repeats, | ||
| 515 | - view.output.strides, kHasOutputRelu); | ||
| 516 | -} | ||
| 517 | - | ||
| 518 | -void BuildComplexIndexPath(const BroadcastGraphView &view, af::ascir_op::IndirectLoad &indirect_load) { | ||
| 519 | - af::ascir_op::Data index("index"); | ||
| 520 | - af::ascir_op::Load index_load("index_load"); | ||
| 521 | - view.graph->AddNode(index); | ||
| 522 | - view.graph->AddNode(index_load); | ||
| 523 | - index.ir_attr.SetIndex(1); | ||
| 524 | - index_load.x = index.y; | ||
| 525 | - SetView(index, view.index_source); | ||
| 526 | - SetView(index_load, view.index_source); | ||
| 527 | - | ||
| 528 | - af::ascir_op::Scalar scalar0("index_scalar0", *view.graph); | ||
| 529 | - af::ascir_op::Scalar scalar1("index_scalar1", *view.graph); | ||
| 530 | - scalar0.ir_attr.SetValue("0"); | ||
| 531 | - scalar1.ir_attr.SetValue("0"); | ||
| 532 | - scalar0.y.dtype = af::DT_INT64; | ||
| 533 | - scalar1.y.dtype = af::DT_INT64; | ||
| 534 | - af::ascir_op::Broadcast broadcast0("index_broadcast0"); | ||
| 535 | - af::ascir_op::Broadcast broadcast1("index_broadcast1"); | ||
| 536 | - af::ascir_op::Add scalar_add("index_scalar_add"); | ||
| 537 | - af::ascir_op::Broadcast final_broadcast("index_final_broadcast"); | ||
| 538 | - for (af::ascir_op::Broadcast *broadcast : {&broadcast0, &broadcast1}) { | ||
| 539 | - view.graph->AddNode(*broadcast); | ||
| 540 | - broadcast->attr.api.compute_type = af::ComputeType::kComputeBroadcast; | ||
| 541 | - SetView(*broadcast, view.index_source); | ||
| 542 | - } | ||
| 543 | - view.graph->AddNode(scalar_add); | ||
| 544 | - view.graph->AddNode(final_broadcast); | ||
| 545 | - final_broadcast.attr.api.compute_type = af::ComputeType::kComputeBroadcast; | ||
| 546 | - broadcast0.x = scalar0.y; | ||
| 547 | - broadcast1.x = scalar1.y; | ||
| 548 | - scalar_add.x1 = broadcast0.y; | ||
| 549 | - scalar_add.x2 = broadcast1.y; | ||
| 550 | - const auto index_add = ConnectBinaryElement(view.graph, "index_add", index_load.y, scalar_add.y, view.index_source); | ||
| 551 | - final_broadcast.x = index_add; | ||
| 552 | - SetView(scalar_add, view.index_source); | ||
| 553 | - SetView(final_broadcast, view.index_broadcast); | ||
| 554 | - indirect_load.x2 = final_broadcast.y; | ||
| 555 | -} | ||
| 556 | - | ||
| 557 | -void BuildSameViewIndexBinaryPath(const BroadcastGraphView &view, af::ascir_op::IndirectLoad &indirect_load) { | ||
| 558 | - af::ascir_op::Data index("same_view_index"); | ||
| 559 | - af::ascir_op::Load index_load("same_view_index_load"); | ||
| 560 | - af::ascir_op::Abs index_abs("same_view_index_abs"); | ||
| 561 | - af::ascir_op::Maximum index_maximum("same_view_index_maximum"); | ||
| 562 | - view.graph->AddNode(index); | ||
| 563 | - view.graph->AddNode(index_load); | ||
| 564 | - view.graph->AddNode(index_abs); | ||
| 565 | - view.graph->AddNode(index_maximum); | ||
| 566 | - index.ir_attr.SetIndex(1); | ||
| 567 | - index_load.x = index.y; | ||
| 568 | - index_abs.x = index_load.y; | ||
| 569 | - index_maximum.x1 = index_load.y; | ||
| 570 | - index_maximum.x2 = index_abs.y; | ||
| 571 | - index_abs.attr.api.compute_type = af::ComputeType::kComputeElewise; | ||
| 572 | - index_maximum.attr.api.compute_type = af::ComputeType::kComputeElewise; | ||
| 573 | - SetView(index, view.output); | ||
| 574 | - SetView(index_load, view.output); | ||
| 575 | - SetView(index_abs, view.output); | ||
| 576 | - SetView(index_maximum, view.output); | ||
| 577 | - indirect_load.x2 = index_maximum.y; | ||
| 578 | -} | ||
| 579 | - | ||
| 580 | -void BuildComplexInputPath(const BroadcastGraphView &view, af::ascir_op::IndirectLoad &indirect_load) { | ||
| 581 | - af::ascir_op::Data x("x"); | ||
| 582 | - af::ascir_op::Load input_load("input_load"); | ||
| 583 | - view.graph->AddNode(x); | ||
| 584 | - view.graph->AddNode(input_load); | ||
| 585 | - x.ir_attr.SetIndex(0); | ||
| 586 | - input_load.x = x.y; | ||
| 587 | - SetView(x, view.input_source); | ||
| 588 | - SetView(input_load, view.input_source); | ||
| 589 | - | ||
| 590 | - af::ascir_op::Scalar scalar0("input_scalar0", *view.graph); | ||
| 591 | - af::ascir_op::Scalar scalar1("input_scalar1", *view.graph); | ||
| 592 | - scalar0.ir_attr.SetValue("0.0"); | ||
| 593 | - scalar1.ir_attr.SetValue("0.0"); | ||
| 594 | - scalar0.y.dtype = af::DT_FLOAT16; | ||
| 595 | - scalar1.y.dtype = af::DT_FLOAT16; | ||
| 596 | - af::ascir_op::Broadcast broadcast0("input_scalar_broadcast0"); | ||
| 597 | - af::ascir_op::Broadcast broadcast1("input_scalar_broadcast1"); | ||
| 598 | - af::ascir_op::Add scalar_add("input_scalar_add"); | ||
| 599 | - af::ascir_op::Broadcast final_broadcast("input_final_broadcast"); | ||
| 600 | - for (af::ascir_op::Broadcast *broadcast : {&broadcast0, &broadcast1}) { | ||
| 601 | - view.graph->AddNode(*broadcast); | ||
| 602 | - broadcast->attr.api.compute_type = af::ComputeType::kComputeBroadcast; | ||
| 603 | - SetView(*broadcast, view.input_source); | ||
| 604 | - } | ||
| 605 | - view.graph->AddNode(scalar_add); | ||
| 606 | - view.graph->AddNode(final_broadcast); | ||
| 607 | - final_broadcast.attr.api.compute_type = af::ComputeType::kComputeBroadcast; | ||
| 608 | - broadcast0.x = scalar0.y; | ||
| 609 | - broadcast1.x = scalar1.y; | ||
| 610 | - scalar_add.x1 = broadcast0.y; | ||
| 611 | - scalar_add.x2 = broadcast1.y; | ||
| 612 | - const auto input_add = | ||
| 613 | - ConnectBinaryElement(view.graph, "input_source_add", input_load.y, scalar_add.y, view.input_source); | ||
| 614 | - final_broadcast.x = input_add; | ||
| 615 | - SetView(scalar_add, view.input_source); | ||
| 616 | - SetView(final_broadcast, view.input_broadcast); | ||
| 617 | - indirect_load.x1 = final_broadcast.y; | ||
| 618 | -} | ||
| 619 | - | ||
| 620 | -af::ComputeGraphPtr CreateGraph() { | ||
| 621 | - indirect_load_test::BackendGraph backend("indirect_load_broadcast_test", "data0", "data1", af::DT_FLOAT16); | ||
| 622 | - const auto build_index = kComplexBroadcast || kComplexIndexBroadcast ? BuildComplexIndexPath | ||
| 623 | - : kIndexBinarySameView ? BuildSameViewIndexBinaryPath | ||
| 624 | - : BuildIndexPath; | ||
| 625 | - const auto build_input = kComplexInputBroadcast ? BuildComplexInputPath : BuildInputPath; | ||
| 626 | - return backend.Finalize( | ||
| 627 | - indirect_load_test::CreateSubGraph(CreateGraphView(), build_input, build_index, BuildOutputPath), "output"); | ||
| 628 | -} | ||
| 629 | - | ||
| 630 | -af::ComputeGraphPtr CreateAicReproGraph() { | ||
| 631 | - auto graph = std::make_shared<af::AscGraph>("indirect_load_aic_repro"); | ||
| 632 | - const af::Expression input_s0 = graph->CreateSizeVar(100000); | ||
| 633 | - const af::Expression output_s0 = graph->CreateSizeVar(1024); | ||
| 634 | - const af::Expression s1 = graph->CreateSizeVar(1024); | ||
| 635 | - const af::Expression one = af::ops::One; | ||
| 636 | - const af::AxisId input_axis0 = graph->CreateAxis("x0", input_s0).id; | ||
| 637 | - const af::AxisId input_axis1 = graph->CreateAxis("x1", s1).id; | ||
| 638 | - const af::AxisId output_axis0 = graph->CreateAxis("y0", output_s0).id; | ||
| 639 | - const af::AxisId output_axis1 = graph->CreateAxis("y1", s1).id; | ||
| 640 | - | ||
| 641 | - af::ascir_op::Data input("input"); | ||
| 642 | - af::ascir_op::Load input_load("input_load"); | ||
| 643 | - graph->AddNode(input); | ||
| 644 | - graph->AddNode(input_load); | ||
| 645 | - input.ir_attr.SetIndex(0); | ||
| 646 | - input_load.x = input.y; | ||
| 647 | - SetView(input, {input_axis0, input_axis1}, {input_s0, s1}, {s1, one}, af::DT_FLOAT); | ||
| 648 | - SetView(input_load, {input_axis0, input_axis1}, {input_s0, s1}, {s1, one}, af::DT_FLOAT); | ||
| 649 | - | ||
| 650 | - af::ascir_op::Data index("index"); | ||
| 651 | - af::ascir_op::Load index_load("index_load"); | ||
| 652 | - af::ascir_op::Broadcast index_broadcast("index_broadcast"); | ||
| 653 | - graph->AddNode(index); | ||
| 654 | - graph->AddNode(index_load); | ||
| 655 | - graph->AddNode(index_broadcast); | ||
| 656 | - index.ir_attr.SetIndex(1); | ||
| 657 | - index_load.x = index.y; | ||
| 658 | - index_broadcast.x = index_load.y; | ||
| 659 | - index_broadcast.attr.api.compute_type = af::ComputeType::kComputeBroadcast; | ||
| 660 | - SetView(index, {output_axis0, output_axis1}, {output_s0, one}, {one, af::ops::Zero}, af::DT_INT64); | ||
| 661 | - SetView(index_load, {output_axis0, output_axis1}, {output_s0, one}, {one, af::ops::Zero}, af::DT_INT64); | ||
| 662 | - SetView(index_broadcast, {output_axis0, output_axis1}, {output_s0, s1}, {s1, one}, af::DT_INT64); | ||
| 663 | - | ||
| 664 | - af::ascir_op::IndirectLoad indirect_load("indirect_load"); | ||
| 665 | - af::ascir_op::Store store("store"); | ||
| 666 | - af::ascir_op::Output output("output"); | ||
| 667 | - graph->AddNode(indirect_load); | ||
| 668 | - graph->AddNode(store); | ||
| 669 | - graph->AddNode(output); | ||
| 670 | - indirect_load.x1 = input_load.y; | ||
| 671 | - indirect_load.x2 = index_broadcast.y; | ||
| 672 | - indirect_load.ir_attr.SetAxis(0); | ||
| 673 | - SetView(indirect_load, {output_axis0, output_axis1}, {output_s0, s1}, {s1, one}, af::DT_FLOAT); | ||
| 674 | - store.x = indirect_load.y; | ||
| 675 | - SetView(store, {output_axis0, output_axis1}, {output_s0, s1}, {s1, one}, af::DT_FLOAT); | ||
| 676 | - output.x = store.y; | ||
| 677 | - output.ir_attr.SetIndex(0); | ||
| 678 | - SetView(output, {output_axis0, output_axis1}, {output_s0, s1}, {s1, one}, af::DT_FLOAT); | ||
| 679 | - | ||
| 680 | - indirect_load_test::BackendGraph backend("indirect_load_aic_repro", "data0", "data1", af::DT_FLOAT); | ||
| 681 | - return backend.Finalize(graph, "output"); | ||
| 682 | -} | ||
| 683 | - | ||
| 684 | -void CheckSkKernel(const std::string &kernel) { | ||
| 685 | - EXPECT_NE(kernel.find("// IndirectLoad SK"), std::string::npos); | ||
| 686 | - EXPECT_EQ(kernel.find("// IndirectLoad SIMD"), std::string::npos); | ||
| 687 | - EXPECT_EQ(kernel.find("// IndirectLoad SIMT"), std::string::npos); | ||
| 688 | - EXPECT_EQ(kernel.find("BroadcastExtend<"), std::string::npos); | ||
| 689 | - EXPECT_NE(kernel.find(MakeShapeArgs(true)), std::string::npos); | ||
| 690 | -} | ||
| 691 | - | ||
| 692 | -void CheckSimtKernel(const std::string &kernel) { | ||
| 693 | - EXPECT_NE(kernel.find("// IndirectLoad SIMT"), std::string::npos); | ||
| 694 | - EXPECT_NE(kernel.find("IndirectLoadSimt<"), std::string::npos); | ||
| 695 | - EXPECT_EQ(kernel.find("// IndirectLoad SIMD"), std::string::npos); | ||
| 696 | - if constexpr (kOutputS0 == 4 && kOutputS1 == 5 && kOutputS2 == 4 && kOutputS3 == 16) { | ||
| 697 | - EXPECT_NE(kernel.find(MakeShapeArgs(false)), std::string::npos); | ||
| 698 | - } | ||
| 699 | - EXPECT_EQ(kernel.find("AscendC::BroadcastExtend<"), std::string::npos); | ||
| 700 | -} | ||
| 701 | - | ||
| 702 | -void CheckSimdElements(const std::string &kernel) { | ||
| 703 | - const auto input_abs_pos = kernel.find("Abs("); | ||
| 704 | - const auto indirect_load_pos = kernel.find("// IndirectLoad SIMD"); | ||
| 705 | - ASSERT_NE(indirect_load_pos, std::string::npos); | ||
| 706 | - if (kInputElementCount > 0 || kIndexElementCount > 0) { | ||
| 707 | - ASSERT_NE(input_abs_pos, std::string::npos); | ||
| 708 | - EXPECT_LT(input_abs_pos, indirect_load_pos); | ||
| 709 | - } else { | ||
| 710 | - EXPECT_EQ(input_abs_pos, std::string::npos); | ||
| 711 | - } | ||
| 712 | - const auto output_relu_pos = kernel.find("Relu("); | ||
| 713 | - EXPECT_EQ(output_relu_pos == std::string::npos, !kHasOutputRelu); | ||
| 714 | -} | ||
| 715 | - | ||
| 716 | -void CheckSimdKernel(const std::string &kernel) { | ||
| 717 | - EXPECT_NE(kernel.find("// IndirectLoad SIMD"), std::string::npos); | ||
| 718 | - EXPECT_NE(kernel.find("IndirectLoadSimd<"), std::string::npos); | ||
| 719 | - if constexpr (kDegenerateBroadcast) { | ||
| 720 | - EXPECT_NE(kernel.find(", 20, 10, 10, 20, 20, 4000, 400, 20, 1);"), std::string::npos); | ||
| 721 | - EXPECT_NE(kernel.find("Duplicate(local_7[0], local_6.GetValue(0), local_7_actual_size);"), std::string::npos); | ||
| 722 | - EXPECT_NE(kernel.find("Duplicate(local_9[0], local_8.GetValue(0), local_9_actual_size);"), std::string::npos); | ||
| 723 | - EXPECT_NE(kernel.find("const uint32_t local_7_actual_size = (400 - 1) + 1;"), std::string::npos); | ||
| 724 | - EXPECT_NE(kernel.find("const uint32_t local_9_actual_size = (400 - 1) + 1;"), std::string::npos); | ||
| 725 | - } else { | ||
| 726 | - | ||
| 727 | - EXPECT_NE(kernel.find("ReduceSum"), std::string::npos); | ||
| 728 | - | ||
| 729 | - if constexpr (!(kIndexBroadcast && kBroadcastAxesMask == 0U) && !kIndexBinarySameView) { | ||
| 730 | - EXPECT_NE(kernel.find(MakeShapeArgs(false)), std::string::npos); | ||
| 731 | - } | ||
| 732 | - | ||
| 733 | - } | ||
| 734 | - if constexpr (!kDegenerateBroadcast && !IL_BROADCAST_POST_REDUCE) { | ||
| 735 | - EXPECT_NE(kernel.find("const int64_t indirect_load_outert_axis_size = 1;"), std::string::npos); | ||
| 736 | - EXPECT_NE(kernel.find("block_dim_offset = indirect_load_outerTB * t->indirect_load_outerTb_size"), | ||
| 737 | - std::string::npos); | ||
| 738 | - } | ||
| 739 | - if (!kIndexBroadcast && kBroadcastAxesMask == 2U) { | ||
| 740 | - EXPECT_NE(kernel.find("global_0[(int64_t)z4 * (int64_t)64 + 0 + 0]"), std::string::npos); | ||
| 741 | - EXPECT_EQ(kernel.find("global_0[(int64_t)z4 * (int64_t)64 + (int64_t)z5 * (int64_t)64"), std::string::npos); | ||
| 742 | - } | ||
| 743 | - EXPECT_EQ(kernel.find("AscendC::BroadcastExtend<"), std::string::npos); | ||
| 744 | - CheckSimdElements(kernel); | ||
| 745 | -} | ||
| 746 | - | ||
| 747 | -void CheckGeneratedKernel(const std::string &kernel, ascir::TemplateId template_id) { | ||
| 748 | - if (template_id == ascir::TemplateId::kIndirectLoadSK) { | ||
| 749 | - CheckSkKernel(kernel); | ||
| 750 | - } else if (template_id == ascir::TemplateId::kIndirectLoadSimt) { | ||
| 751 | - CheckSimtKernel(kernel); | ||
| 752 | - } else { | ||
| 753 | - CheckSimdKernel(kernel); | ||
| 754 | - } | ||
| 755 | -} | ||
| 756 | - | ||
| 757 | -const af::Axis *FindDerivedAxis(const af::AscGraph &graph, af::Axis::Type type, af::AxisId from) { | ||
| 758 | - for (const auto &axis : graph.GetAllAxis()) { | ||
| 759 | - if (axis->type == type && axis->from == std::vector<af::AxisId>{from}) { | ||
| 760 | - return axis.get(); | ||
| 761 | - } | ||
| 762 | - } | ||
| 763 | - return nullptr; | ||
| 764 | -} | ||
| 765 | - | ||
| 766 | -void ExpectComplexNodeSchedule(const af::AscNodePtr &node, const std::vector<af::AxisId> &axes, | ||
| 767 | - af::AxisId vectorized_axis) { | ||
| 768 | - ASSERT_NE(node, nullptr); | ||
| 769 | - EXPECT_EQ(node->attr.sched.axis, axes) << node->GetName(); | ||
| 770 | - ASSERT_GE(axes.size(), 2UL); | ||
| 771 | - EXPECT_EQ(node->attr.sched.loop_axis, axes[axes.size() - 2UL]) << node->GetName(); | ||
| 772 | - ASSERT_FALSE(node->outputs().empty()); | ||
| 773 | - EXPECT_EQ(node->outputs()[0]->attr.vectorized_axis, std::vector<af::AxisId>{vectorized_axis}) << node->GetName(); | ||
| 774 | -} | ||
| 775 | - | ||
| 776 | -void ExpectComplexOuterAxes(af::AscGraph &graph, const ascgen_utils::indirect_load::TemplateAxes &axes, | ||
| 777 | - const ascgen_utils::indirect_load::TemplateLogicalView &view, | ||
| 778 | - std::vector<af::AxisId> &outer_loops) { | ||
| 779 | - const af::Axis *outer = graph.FindAxis(axes.outer_axis); | ||
| 780 | - const af::Axis *inner = graph.FindAxis(axes.inner_axis); | ||
| 781 | - const af::Axis *input_inner = graph.FindAxis(axes.input_inner_axis); | ||
| 782 | - const af::Axis *index_inner = graph.FindAxis(axes.index_inner_axis); | ||
| 783 | - ASSERT_NE(outer, nullptr); | ||
| 784 | - ASSERT_NE(inner, nullptr); | ||
| 785 | - ASSERT_NE(input_inner, nullptr); | ||
| 786 | - ASSERT_NE(index_inner, nullptr); | ||
| 787 | - EXPECT_EQ(outer->from, std::vector<af::AxisId>({view.output.axis_ids[0], view.output.axis_ids[1]})); | ||
| 788 | - EXPECT_EQ(inner->from, std::vector<af::AxisId>({view.output.axis_ids[2], view.output.axis_ids[3]})); | ||
| 789 | - EXPECT_EQ(input_inner->from, std::vector<af::AxisId>({view.input.axis_ids[2], view.input.axis_ids[3]})); | ||
| 790 | - EXPECT_EQ(index_inner->from, std::vector<af::AxisId>({view.output.axis_ids[2], view.output.axis_ids[3]})); | ||
| 791 | - const af::Axis *tile_outer = FindDerivedAxis(graph, af::Axis::Type::kAxisTypeTileOuter, outer->id); | ||
| 792 | - const af::Axis *tile_inner = FindDerivedAxis(graph, af::Axis::Type::kAxisTypeTileInner, outer->id); | ||
| 793 | - ASSERT_NE(tile_outer, nullptr); | ||
| 794 | - ASSERT_NE(tile_inner, nullptr); | ||
| 795 | - const af::Axis *block_outer = FindDerivedAxis(graph, af::Axis::Type::kAxisTypeBlockOuter, tile_outer->id); | ||
| 796 | - const af::Axis *block_inner = FindDerivedAxis(graph, af::Axis::Type::kAxisTypeBlockInner, tile_outer->id); | ||
| 797 | - ASSERT_NE(block_outer, nullptr); | ||
| 798 | - ASSERT_NE(block_inner, nullptr); | ||
| 799 | - outer_loops = {block_outer->id, block_inner->id, tile_inner->id}; | ||
| 800 | -} | ||
| 801 | - | ||
| 802 | -void ExpectComplexLogicalView(const ascgen_utils::indirect_load::TemplateLogicalView &view) { | ||
| 803 | - ASSERT_EQ(view.output.axis_ids.size(), 4UL); | ||
| 804 | - ASSERT_EQ(view.input.axis_ids.size(), 4UL); | ||
| 805 | - EXPECT_EQ(view.input.kind, ascgen_utils::indirect_load::IndirectLoadLayoutKind::kZeroStrideCompact); | ||
| 806 | - EXPECT_EQ(view.index.kind, ascgen_utils::indirect_load::IndirectLoadLayoutKind::kZeroStrideCompact); | ||
| 807 | - ASSERT_EQ(view.input.physical_repeats.size(), 4UL); | ||
| 808 | - ASSERT_EQ(view.index.physical_repeats.size(), 4UL); | ||
| 809 | - EXPECT_TRUE(ascgen_utils::ExpressEq(view.input.physical_repeats[0], af::ops::One)); | ||
| 810 | - EXPECT_TRUE(ascgen_utils::ExpressEq(view.input.physical_repeats[1], af::ops::One)); | ||
| 811 | - EXPECT_TRUE(ascgen_utils::ExpressEq(view.index.physical_repeats[0], af::ops::One)); | ||
| 812 | - EXPECT_TRUE(ascgen_utils::ExpressEq(view.index.physical_repeats[1], af::ops::One)); | ||
| 813 | -} | ||
| 814 | - | ||
| 815 | -void ExpectComplexBrcRewrite(af::AscGraph &graph, const af::AscNodePtr &input_add, const af::AscNodePtr &index_vf, | ||
| 816 | - const af::AscNodePtr &post_vf) { | ||
| 817 | - const bool is_expected_binary = kBinaryElementKind == 1 ? af::ops::IsOps<af::ascir_op::Mul>(input_add) | ||
| 818 | - : kBinaryElementKind == 2 ? af::ops::IsOps<af::ascir_op::Sub>(input_add) | ||
| 819 | - : kBinaryElementKind == 3 ? af::ops::IsOps<af::ascir_op::Maximum>(input_add) | ||
| 820 | - : af::ops::IsOps<af::ascir_op::Add>(input_add); | ||
| 821 | - ASSERT_TRUE(is_expected_binary); | ||
| 822 | - ASSERT_TRUE(af::ops::IsOps<af::ascir_op::VectorFunc>(index_vf)); | ||
| 823 | - ASSERT_TRUE(af::ops::IsOps<af::ascir_op::VectorFunc>(post_vf)); | ||
| 824 | - EXPECT_EQ(index_vf->inputs.Size(), 3UL); | ||
| 825 | - EXPECT_EQ(post_vf->inputs.Size(), 3UL); | ||
| 826 | - EXPECT_EQ(graph.FindNode("input_broadcast"), nullptr); | ||
| 827 | - ASSERT_EQ(input_add->inputs.Size(), 2UL); | ||
| 828 | - for (size_t i = 0UL; i < input_add->inputs.Size(); ++i) { | ||
| 829 | - const af::AscNodePtr producer = ascgen_utils::indirect_load::GetInputProducer(input_add, i); | ||
| 830 | - ASSERT_NE(producer, nullptr); | ||
| 831 | - EXPECT_EQ(producer->GetName(), "input_load") << "input index=" << i; | ||
| 832 | - } | ||
| 833 | - for (const char *name : {"index_final_broadcast", "index_broadcast0", "index_broadcast1", "index_scalar_add", | ||
| 834 | - "output_broadcast0", "output_broadcast1", "output_scalar_add"}) { | ||
| 835 | - EXPECT_EQ(graph.FindNode(name), nullptr) << name; | ||
| 836 | - } | ||
| 837 | -} | ||
| 838 | - | ||
| 839 | -void ExpectComplexSimdSchedule(af::AscGraph &graph, const af::AscNodePtr &indirect_load) { | ||
| 840 | - ascgen_utils::indirect_load::TemplateAxes axes; | ||
| 841 | - ascgen_utils::indirect_load::TemplateLogicalView view; | ||
| 842 | - ASSERT_EQ(ascgen_utils::indirect_load::GetTemplateAxes(indirect_load, axes), af::SUCCESS); | ||
| 843 | - ASSERT_EQ(ascgen_utils::indirect_load::GetTemplateLogicalView(indirect_load, view), af::SUCCESS); | ||
| 844 | - ExpectComplexLogicalView(view); | ||
| 845 | - std::vector<af::AxisId> outer_loops; | ||
| 846 | - ExpectComplexOuterAxes(graph, axes, view, outer_loops); | ||
| 847 | - ASSERT_EQ(outer_loops.size(), 3UL); | ||
| 848 | - const std::vector<af::AxisId> index_axes = {outer_loops[0], outer_loops[1], outer_loops[2], axes.index_inner_axis}; | ||
| 849 | - const std::vector<af::AxisId> input_axes = {outer_loops[0], outer_loops[1], outer_loops[2], axes.input_inner_axis}; | ||
| 850 | - const std::vector<af::AxisId> output_axes = {outer_loops[0], outer_loops[1], outer_loops[2], axes.inner_axis}; | ||
| 851 | - const af::AscNodePtr input_add = ascgen_utils::indirect_load::GetInputProducer(indirect_load, 0UL); | ||
| 852 | - const af::AscNodePtr index_vf = ascgen_utils::indirect_load::GetInputProducer(indirect_load, 1UL); | ||
| 853 | - const af::AscNodePtr post_vf = ascgen_utils::indirect_load::GetOnlyOutputConsumer(indirect_load); | ||
| 854 | - ExpectComplexBrcRewrite(graph, input_add, index_vf, post_vf); | ||
| 855 | - ExpectComplexNodeSchedule(graph.FindNode("index_load"), index_axes, axes.index_inner_axis); | ||
| 856 | - ExpectComplexNodeSchedule(index_vf, index_axes, axes.index_inner_axis); | ||
| 857 | - ExpectComplexNodeSchedule(graph.FindNode("input_load"), input_axes, axes.input_inner_axis); | ||
| 858 | - ExpectComplexNodeSchedule(input_add, input_axes, axes.input_inner_axis); | ||
| 859 | - for (const af::AscNodePtr &node : {indirect_load, post_vf, graph.FindNode("store")}) { | ||
| 860 | - ASSERT_NE(node, nullptr); | ||
| 861 | - EXPECT_EQ(node->attr.sched.axis, output_axes) << node->GetName(); | ||
| 862 | - EXPECT_EQ(node->attr.sched.loop_axis, outer_loops.back()) << node->GetName(); | ||
| 863 | - ASSERT_FALSE(node->outputs().empty()); | ||
| 864 | - EXPECT_EQ(node->outputs()[0]->attr.vectorized_axis, axes.vectorized_axes) << node->GetName(); | ||
| 865 | - } | ||
| 866 | - std::vector<af::AscNodePtr> nodes; | ||
| 867 | - for (const af::AscNodePtr &node : graph.GetAllNodes()) { | ||
| 868 | - nodes.emplace_back(node); | ||
| 869 | - } | ||
| 870 | - const auto pos = [&](const af::AscNodePtr &node) { return std::find(nodes.begin(), nodes.end(), node); }; | ||
| 871 | - EXPECT_LT(pos(graph.FindNode("index_load")), pos(index_vf)); | ||
| 872 | - EXPECT_LT(pos(graph.FindNode("input_load")), pos(input_add)); | ||
| 873 | - EXPECT_LT(pos(input_add), pos(indirect_load)); | ||
| 874 | - EXPECT_LT(pos(indirect_load), pos(post_vf)); | ||
| 875 | - EXPECT_LT(pos(post_vf), pos(graph.FindNode("store"))); | ||
| 876 | -} | ||
| 877 | - | ||
| 878 | -} // namespace | ||
| 879 | - | ||
| 880 | -using TestBackendIndirectLoadBroadcastE2e = indirect_load_test::PrecisionBackendE2e; | ||
| 881 | - | ||
| 882 | -TEST_F(TestBackendIndirectLoadBroadcastE2e, IndirectLoadBroadcastCodegen) { | ||
| 883 | - const auto graph = kAicRepro ? CreateAicReproGraph() : CreateGraph(); | ||
| 884 | - ASSERT_NE(graph, nullptr); | ||
| 885 | - if (kAicRepro) { | ||
| 886 | - ascir::FusedScheduledResult scheduled_result; | ||
| 887 | - optimize::Optimizer optimizer(optimize::OptimizerOptions{.graph_type = optimize::GraphType::kFusedAscBackend}); | ||
| 888 | - ASSERT_EQ(optimizer.Optimize(graph, scheduled_result), af::SUCCESS); | ||
| 889 | - ASSERT_TRUE(indirect_load_test::HasTemplate(scheduled_result, ascir::TemplateId::kIndirectLoadSimt)); | ||
| 890 | - codegen::Codegen codegen(codegen::CodegenOptions{}); | ||
| 891 | - codegen::CodegenResult result; | ||
| 892 | - ASSERT_EQ(codegen.Generate({}, scheduled_result, result), af::SUCCESS); | ||
| 893 | - EXPECT_NE(result.kernel.find("// IndirectLoad SIMT"), std::string::npos); | ||
| 894 | - EXPECT_NE(result.kernel.find("IndirectLoadSimtStridedPolicy<uint32_t, 2, 0, 3ULL, 1ULL>"), std::string::npos); | ||
| 895 | - EXPECT_NE(result.kernel.find(", 1024, 1024, 1024, 1, 1, 0);"), std::string::npos); | ||
| 896 | - indirect_load_test::WriteGeneratedFiles(result); | ||
| 897 | - return; | ||
| 898 | - } | ||
| 899 | - const std::map<std::string, std::string> shape_info = {{"s0", "stub_s0"}, {"s1", "stub_s1"}, {"s2", "stub_s2"}, | ||
| 900 | - {"s3", "stub_s3"}, {"s4", "stub_s4"}, {"s5", "stub_s5"}, | ||
| 901 | - {"s6", "stub_s6"}, {"s7", "stub_s7"}}; | ||
| 902 | - const auto expected_template = indirect_load_test::GetExpectedTemplate(kExpectSimt, kExpectSk); | ||
| 903 | - if constexpr (kRetainBroadcast) { | ||
| 904 | - ascir::FusedScheduledResult scheduled_result; | ||
| 905 | - ASSERT_TRUE(indirect_load_test::SelectTemplate(graph, expected_template, scheduled_result)); | ||
| 906 | - codegen::Codegen codegen(codegen::CodegenOptions{}); | ||
| 907 | - codegen::CodegenResult result; | ||
| 908 | - ASSERT_EQ(codegen.Generate(shape_info, scheduled_result, result), af::SUCCESS); | ||
| 909 | - EXPECT_NE(result.kernel.find(kExpectSimt ? "IndirectLoadSimt<" : "IndirectLoadSimd<"), std::string::npos); | ||
| 910 | - if constexpr (kExpectSimt) { | ||
| 911 | - EXPECT_EQ(result.kernel.find("BroadcastExtend<"), std::string::npos); | ||
| 912 | - } else { | ||
| 913 | - EXPECT_NE(result.kernel.find("BroadcastExtend<"), std::string::npos); | ||
| 914 | - } | ||
| 915 | - indirect_load_test::WriteGeneratedFiles(result); | ||
| 916 | - return; | ||
| 917 | - } | ||
| 918 | - if constexpr (kComplexBroadcast) { | ||
| 919 | - ascir::FusedScheduledResult scheduled_result; | ||
| 920 | - ASSERT_TRUE(indirect_load_test::SelectTemplate(graph, expected_template, scheduled_result)); | ||
| 921 | - size_t template_graph_count = 0UL; | ||
| 922 | - for (auto &candidates : scheduled_result.node_idx_to_scheduled_results) { | ||
| 923 | - for (auto &candidate : candidates) { | ||
| 924 | - for (auto &group : candidate.schedule_groups) { | ||
| 925 | - for (auto &impl_graph : group.impl_graphs) { | ||
| 926 | - const af::AscNodePtr indirect_load = ascgen_utils::indirect_load::FindIndirectLoadNode(impl_graph); | ||
| 927 | - if (indirect_load != nullptr && ascir::GetTemplateIdOrDefault(*indirect_load) == expected_template) { | ||
| 928 | - ++template_graph_count; | ||
| 929 | - if (expected_template == ascir::TemplateId::kIndirectLoadSimd) { | ||
| 930 | - ExpectComplexSimdSchedule(impl_graph, indirect_load); | ||
| 931 | - } | ||
| 932 | - } | ||
| 933 | - } | ||
| 934 | - } | ||
| 935 | - } | ||
| 936 | - } | ||
| 937 | - EXPECT_GT(template_graph_count, 0UL); | ||
| 938 | - codegen::Codegen codegen(codegen::CodegenOptions{}); | ||
| 939 | - codegen::CodegenResult result; | ||
| 940 | - ASSERT_EQ(codegen.Generate(shape_info, scheduled_result, result), af::SUCCESS); | ||
| 941 | - EXPECT_NE(result.kernel.find(expected_template == ascir::TemplateId::kIndirectLoadSimd ? "IndirectLoadSimd<" | ||
| 942 | - : "IndirectLoadSimt<"), | ||
| 943 | - std::string::npos); | ||
| 944 | - const bool has_binary_element = | ||
| 945 | - result.kernel.find("Add(") != std::string::npos || result.kernel.find("Mul(") != std::string::npos || | ||
| 946 | - result.kernel.find("Sub(") != std::string::npos || result.kernel.find("Maximum(") != std::string::npos; | ||
| 947 | - EXPECT_TRUE(has_binary_element); | ||
| 948 | - EXPECT_EQ(result.kernel.find("BroadcastExtend<"), std::string::npos); | ||
| 949 | - indirect_load_test::WriteGeneratedFiles(result); | ||
| 950 | - return; | ||
| 951 | - } | ||
| 952 | - if constexpr (kComplexInputBroadcast) { | ||
| 953 | - ascir::FusedScheduledResult scheduled_result; | ||
| 954 | - ASSERT_TRUE(indirect_load_test::SelectTemplate(graph, expected_template, scheduled_result)); | ||
| 955 | - codegen::Codegen codegen(codegen::CodegenOptions{}); | ||
| 956 | - codegen::CodegenResult result; | ||
| 957 | - ASSERT_EQ(codegen.Generate(shape_info, scheduled_result, result), af::SUCCESS); | ||
| 958 | - EXPECT_NE(result.kernel.find(indirect_load_test::GetTemplateMarker(expected_template)), std::string::npos); | ||
| 959 | - indirect_load_test::WriteGeneratedFiles(result); | ||
| 960 | - return; | ||
| 961 | - } | ||
| 962 | - codegen::CodegenResult result; | ||
| 963 | - indirect_load_test::GenerateForTemplate(graph, shape_info, expected_template, result); | ||
| 964 | - CheckGeneratedKernel(result.kernel, expected_template); | ||
| 965 | - indirect_load_test::WriteGeneratedFiles(result); | ||
| 966 | -} | ||
Dautofuse/tests/v35/st/backend_e2e_v2/indirect_load_store_test/indirect_load_broadcast_cases.cmake+0-178
| @@ -1,178 +0,0 @@ | |||
| 1 | -function(add_indirect_load_broadcast_test test_name template input_element_count index_element_count broadcast_axes_mask | ||
| 2 | - output_relu clear_broadcast_source_view) | ||
| 3 | - if(ARGC GREATER 7) | ||
| 4 | - set(input_broadcast ${ARGV7}) | ||
| 5 | - else() | ||
| 6 | - set(input_broadcast 1) | ||
| 7 | - endif() | ||
| 8 | - if(ARGC GREATER 8) | ||
| 9 | - set(index_broadcast ${ARGV8}) | ||
| 10 | - else() | ||
| 11 | - set(index_broadcast 1) | ||
| 12 | - endif() | ||
| 13 | - set(expect_simt 0) | ||
| 14 | - set(expect_sk 0) | ||
| 15 | - set(tiling_options) | ||
| 16 | - if(template STREQUAL "simt") | ||
| 17 | - set(expect_simt 1) | ||
| 18 | - set(tiling_options TILING_KEY 1) | ||
| 19 | - elseif(template STREQUAL "sk") | ||
| 20 | - set(expect_sk 1) | ||
| 21 | - set(tiling_options TILING_KEY 2) | ||
| 22 | - elseif(NOT template STREQUAL "simd") | ||
| 23 | - message(FATAL_ERROR "Unsupported IndirectLoad template: ${template}") | ||
| 24 | - endif() | ||
| 25 | - | ||
| 26 | - set(case_workdir ${CMAKE_CURRENT_BINARY_DIR}/${test_name}) | ||
| 27 | - file(MAKE_DIRECTORY ${case_workdir}) | ||
| 28 | - do_backend_e2e_st_test(${test_name} | ||
| 29 | - WORKDIR ${case_workdir} | ||
| 30 | - CODEGEN indirect_load_broadcast_backend_generator.cpp | ||
| 31 | - ${tiling_options} | ||
| 32 | - KERNEL_SRC | ||
| 33 | - indirect_load_broadcast_test_kernel.cpp | ||
| 34 | - indirect_load_broadcast_test_tiling.cpp | ||
| 35 | - autofuse_tiling_data.h | ||
| 36 | - TEST_SRC test_e2e_indirect_load_broadcast_kernel.cpp) | ||
| 37 | - set(case_definitions | ||
| 38 | - IL_INPUT_BROADCAST=${input_broadcast} | ||
| 39 | - IL_INDEX_BROADCAST=${index_broadcast} | ||
| 40 | - IL_HAS_INPUT_ELEMENT=${input_element_count} | ||
| 41 | - IL_HAS_INDEX_ELEMENT=${index_element_count} | ||
| 42 | - IL_HAS_OUTPUT_RELU=${output_relu} | ||
| 43 | - IL_BROADCAST_AXES_MASK=${broadcast_axes_mask} | ||
| 44 | - IL_CLEAR_BROADCAST_SOURCE_VIEW=${clear_broadcast_source_view} | ||
| 45 | - IL_EXPECT_SIMT=${expect_simt} | ||
| 46 | - IL_EXPECT_SK=${expect_sk}) | ||
| 47 | - target_compile_definitions(${test_name}_codegen_v2 PRIVATE ${case_definitions}) | ||
| 48 | - target_compile_definitions(${test_name}_e2e_v2 PRIVATE ${case_definitions}) | ||
| 49 | -endfunction() | ||
| 50 | - | ||
| 51 | -# Direct Broadcast covers IndirectLoad axis 2, its inner neighbor, and a degenerate source crossing the axis boundary. | ||
| 52 | -add_indirect_load_broadcast_test(indirect_load_broadcast_cross_boundary_simt_fallback_test simt 0 0 14 0 0) | ||
| 53 | -target_compile_definitions(indirect_load_broadcast_cross_boundary_simt_fallback_test_codegen_v2 PRIVATE | ||
| 54 | - IL_DEGENERATE_BROADCAST=1 IL_OUTPUT_S0=10 IL_OUTPUT_S1=10 IL_OUTPUT_S2=20 IL_OUTPUT_S3=20) | ||
| 55 | -target_compile_definitions(indirect_load_broadcast_cross_boundary_simt_fallback_test_e2e_v2 PRIVATE | ||
| 56 | - IL_DEGENERATE_BROADCAST=1 IL_OUTPUT_S0=10 IL_OUTPUT_S1=10 IL_OUTPUT_S2=20 IL_OUTPUT_S3=20) | ||
| 57 | -add_indirect_load_broadcast_test(indirect_load_broadcast_axis_simd_test simd 0 0 4 0 0) | ||
| 58 | -add_indirect_load_broadcast_test(indirect_load_broadcast_inner_adjacent_simd_test simd 0 0 8 0 0) | ||
| 59 | -add_indirect_load_broadcast_test(indirect_load_broadcast_continuous_simd_test simd 0 0 12 0 0) | ||
| 60 | -target_compile_definitions(indirect_load_broadcast_continuous_simd_test_codegen_v2 PRIVATE IL_CONTINUOUS_BROADCAST=1) | ||
| 61 | -target_compile_definitions(indirect_load_broadcast_continuous_simd_test_e2e_v2 PRIVATE IL_CONTINUOUS_BROADCAST=1) | ||
| 62 | -add_indirect_load_broadcast_test(indirect_load_broadcast_continuous_index_simt_test simt 0 0 12 0 0 0 1) | ||
| 63 | -target_compile_definitions(indirect_load_broadcast_continuous_index_simt_test_codegen_v2 PRIVATE | ||
| 64 | - IL_CONTINUOUS_INDEX_BROADCAST=1) | ||
| 65 | -target_compile_definitions(indirect_load_broadcast_continuous_index_simt_test_e2e_v2 PRIVATE | ||
| 66 | - IL_CONTINUOUS_INDEX_BROADCAST=1) | ||
| 67 | -add_indirect_load_broadcast_test(indirect_load_broadcast_cross_boundary_simt_test simt 0 0 10 0 0) | ||
| 68 | -add_indirect_load_broadcast_test(indirect_load_broadcast_axis_simt_test simt 0 0 4 0 0) | ||
| 69 | -add_indirect_load_broadcast_test(indirect_load_broadcast_reduce_simt_fallback_test simt 0 0 2 0 0) | ||
| 70 | -target_compile_definitions(indirect_load_broadcast_reduce_simt_fallback_test_codegen_v2 PRIVATE IL_BROADCAST_POST_REDUCE=1) | ||
| 71 | -target_compile_definitions(indirect_load_broadcast_reduce_simt_fallback_test_e2e_v2 PRIVATE IL_BROADCAST_POST_REDUCE=1) | ||
| 72 | - | ||
| 73 | -# A scalar Broadcast after an Element cannot use the physical-view inline path. SIMD keeps it as a regular op; | ||
| 74 | -# SIMT emits it inside the fused scalar body. | ||
| 75 | -add_indirect_load_broadcast_test(indirect_load_broadcast_retained_simd_test simd 0 0 10 0 0 0 0) | ||
| 76 | -target_compile_definitions(indirect_load_broadcast_retained_simd_test_codegen_v2 PRIVATE IL_RETAIN_BROADCAST=1) | ||
| 77 | -target_compile_definitions(indirect_load_broadcast_retained_simd_test_e2e_v2 PRIVATE IL_RETAIN_BROADCAST=1) | ||
| 78 | -add_indirect_load_broadcast_test(indirect_load_broadcast_retained_simt_test simt 0 0 10 0 0 0 0) | ||
| 79 | -target_compile_definitions(indirect_load_broadcast_retained_simt_test_codegen_v2 PRIVATE IL_RETAIN_BROADCAST=1) | ||
| 80 | -target_compile_definitions(indirect_load_broadcast_retained_simt_test_e2e_v2 PRIVATE IL_RETAIN_BROADCAST=1) | ||
| 81 | - | ||
| 82 | -# Keep one SK Broadcast regression outside the SIMD/SIMT matrix. | ||
| 83 | -add_indirect_load_broadcast_test(indirect_load_broadcast_elements_sk_test sk 2 2 3 0 0) | ||
| 84 | -add_indirect_load_broadcast_test(indirect_load_broadcast_index_physical_view_simt_test simt 2 2 3 0 0) | ||
| 85 | -target_compile_definitions(indirect_load_broadcast_index_physical_view_simt_test_codegen_v2 PRIVATE IL_AIC_REPRO=1) | ||
| 86 | -target_compile_definitions(indirect_load_broadcast_index_physical_view_simt_test_e2e_v2 PRIVATE IL_AIC_REPRO=1) | ||
| 87 | - | ||
| 88 | -# Identity Broadcast keeps the index source axis/repeats unchanged but still exercises the direct-Broadcast fold. | ||
| 89 | -add_indirect_load_broadcast_test(indirect_load_broadcast_identity_index_simd_test simd 0 0 0 0 0 0 1) | ||
| 90 | - | ||
| 91 | -# Regression: a unary elementwise op after an index Broadcast must retain the source physical view in SIMT. | ||
| 92 | -add_indirect_load_broadcast_test(indirect_load_broadcast_index_abs_simt_test simt 0 1 2 0 0 0 1) | ||
| 93 | -target_compile_definitions(indirect_load_broadcast_index_abs_simt_test_codegen_v2 PRIVATE IL_INDEX_ABS_DENSE_VIEW=1) | ||
| 94 | -target_compile_definitions(indirect_load_broadcast_index_abs_simt_test_e2e_v2 PRIVATE IL_INDEX_ABS_DENSE_VIEW=1) | ||
| 95 | - | ||
| 96 | -# Regression: a three-input Where chain must remain in the SIMT index region before IndirectLoad. | ||
| 97 | -set(indirect_load_broadcast_index_where_simt_test_workdir | ||
| 98 | - ${CMAKE_CURRENT_BINARY_DIR}/indirect_load_broadcast_index_where_simt_test) | ||
| 99 | -file(MAKE_DIRECTORY ${indirect_load_broadcast_index_where_simt_test_workdir}) | ||
| 100 | -do_backend_e2e_st_test(indirect_load_broadcast_index_where_simt_test | ||
| 101 | - WORKDIR ${indirect_load_broadcast_index_where_simt_test_workdir} | ||
| 102 | - CODEGEN indirect_load_broadcast_where_backend_generator.cpp | ||
| 103 | - TILING_KEY 1 | ||
| 104 | - KERNEL_SRC | ||
| 105 | - indirect_load_broadcast_where_test_kernel.cpp | ||
| 106 | - indirect_load_broadcast_where_test_tiling.cpp | ||
| 107 | - autofuse_tiling_data.h | ||
| 108 | - TEST_SRC test_e2e_indirect_load_broadcast_where_kernel.cpp) | ||
| 109 | - | ||
| 110 | -# Same-view tensor fan-in without Broadcast: the binary operation is coordinate-preserving. | ||
| 111 | -add_indirect_load_broadcast_test(indirect_load_index_binary_same_view_simd_test simd 0 1 0 0 0 0 0) | ||
| 112 | -target_compile_definitions(indirect_load_index_binary_same_view_simd_test_codegen_v2 PRIVATE | ||
| 113 | - IL_INDEX_BINARY_SAME_VIEW=1 IL_BINARY_ELEMENT_KIND=3) | ||
| 114 | -target_compile_definitions(indirect_load_index_binary_same_view_simd_test_e2e_v2 PRIVATE | ||
| 115 | - IL_INDEX_BINARY_SAME_VIEW=1 IL_BINARY_ELEMENT_KIND=3) | ||
| 116 | - | ||
| 117 | -add_indirect_load_broadcast_test(indirect_load_complex_broadcast_simd_test simd 0 0 3 0 0) | ||
| 118 | -target_compile_definitions(indirect_load_complex_broadcast_simd_test_codegen_v2 PRIVATE IL_COMPLEX_BROADCAST=1) | ||
| 119 | -target_compile_definitions(indirect_load_complex_broadcast_simd_test_e2e_v2 PRIVATE IL_COMPLEX_BROADCAST=1) | ||
| 120 | - | ||
| 121 | -add_indirect_load_broadcast_test(indirect_load_complex_broadcast_simt_test simt 0 0 3 0 0) | ||
| 122 | -target_compile_definitions(indirect_load_complex_broadcast_simt_test_codegen_v2 PRIVATE IL_COMPLEX_BROADCAST=1 | ||
| 123 | - IL_COMPLEX_SIMT=1) | ||
| 124 | -target_compile_definitions(indirect_load_complex_broadcast_simt_test_e2e_v2 PRIVATE IL_COMPLEX_BROADCAST=1 | ||
| 125 | - IL_COMPLEX_SIMT=1) | ||
| 126 | - | ||
| 127 | -# Input Broadcast whose source is a multi-input Add: the direct Broadcast is inlined and the SIMD | ||
| 128 | -# candidate consumes the computed source through its physical view (source multi-input scenario). | ||
| 129 | -# Note: SIMT rejects multi-input input sources by design (ValidateSimtTemplateRegion requires a Load | ||
| 130 | -# boundary), covered by UT SimdRegionMetadataAndSimtRejectsMultiInputRegion. | ||
| 131 | -add_indirect_load_broadcast_test(indirect_load_complex_input_broadcast_simd_test simd 0 0 12 0 0 1 0) | ||
| 132 | -target_compile_definitions(indirect_load_complex_input_broadcast_simd_test_codegen_v2 PRIVATE | ||
| 133 | - IL_COMPLEX_INPUT_BROADCAST=1) | ||
| 134 | -target_compile_definitions(indirect_load_complex_input_broadcast_simd_test_e2e_v2 PRIVATE | ||
| 135 | - IL_COMPLEX_INPUT_BROADCAST=1) | ||
| 136 | - | ||
| 137 | -function(add_indirect_load_stride_zero_test test_name template input_zero_stride_mask index_zero_stride_mask | ||
| 138 | - input_element_count index_element_count) | ||
| 139 | - set(expect_simt 0) | ||
| 140 | - set(expect_sk 0) | ||
| 141 | - set(tiling_options) | ||
| 142 | - if(template STREQUAL "simt") | ||
| 143 | - set(expect_simt 1) | ||
| 144 | - set(tiling_options TILING_KEY 1) | ||
| 145 | - elseif(template STREQUAL "sk") | ||
| 146 | - set(expect_sk 1) | ||
| 147 | - set(tiling_options TILING_KEY 2) | ||
| 148 | - elseif(NOT template STREQUAL "simd") | ||
| 149 | - message(FATAL_ERROR "Unsupported IndirectLoad template: ${template}") | ||
| 150 | - endif() | ||
| 151 | - | ||
| 152 | - set(case_workdir ${CMAKE_CURRENT_BINARY_DIR}/${test_name}) | ||
| 153 | - file(MAKE_DIRECTORY ${case_workdir}) | ||
| 154 | - do_backend_e2e_st_test(${test_name} | ||
| 155 | - WORKDIR ${case_workdir} | ||
| 156 | - CODEGEN indirect_load_stride_zero_backend_generator.cpp | ||
| 157 | - ${tiling_options} | ||
| 158 | - KERNEL_SRC | ||
| 159 | - indirect_load_stride_zero_test_kernel.cpp | ||
| 160 | - indirect_load_stride_zero_test_tiling.cpp | ||
| 161 | - autofuse_tiling_data.h | ||
| 162 | - TEST_SRC test_e2e_indirect_load_stride_zero_kernel.cpp) | ||
| 163 | - set(case_definitions | ||
| 164 | - IL_INPUT_ZERO_STRIDE_MASK=${input_zero_stride_mask} | ||
| 165 | - IL_INDEX_ZERO_STRIDE_MASK=${index_zero_stride_mask} | ||
| 166 | - IL_HAS_INPUT_ELEMENT=${input_element_count} | ||
| 167 | - IL_HAS_INDEX_ELEMENT=${index_element_count} | ||
| 168 | - IL_EXPECT_SIMT=${expect_simt} | ||
| 169 | - IL_EXPECT_SK=${expect_sk}) | ||
| 170 | - target_compile_definitions(${test_name}_codegen_v2 PRIVATE ${case_definitions}) | ||
| 171 | - target_compile_definitions(${test_name}_e2e_v2 PRIVATE ${case_definitions}) | ||
| 172 | -endfunction() | ||
| 173 | - | ||
| 174 | -# Mask bit d denotes that logical axis d has stride 0. The Element case covers both input and index zero-stride paths. | ||
| 175 | -# Direct zero-stride classification is covered by the layout and schedule UT. | ||
| 176 | -add_indirect_load_stride_zero_test(indirect_load_stride_zero_elements_simd_test simd 10 5 1 1) | ||
| 177 | -add_indirect_load_stride_zero_test(indirect_load_stride_zero_elements_simt_test simt 10 5 1 1) | ||
| 178 | -add_indirect_load_stride_zero_test(indirect_load_stride_zero_elements_sk_test sk 10 5 1 1) | ||
Dautofuse/tests/v35/st/backend_e2e_v2/indirect_load_store_test/indirect_load_broadcast_where_backend_generator.cpp+0-212
| @@ -1,212 +0,0 @@ | |||
| 1 | -/** | ||
| 2 | - * Copyright (c) 2026 Huawei Technologies Co., Ltd. | ||
| 3 | - * This program is free software, you can redistribute it and/or modify it under the terms and conditions of | ||
| 4 | - * CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 5 | - * Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 6 | - * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 7 | - * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 8 | - * See LICENSE in the root of the software repository for the full text of the License. | ||
| 9 | - */ | ||
| 10 | - | ||
| 11 | - | ||
| 12 | - | ||
| 13 | -namespace { | ||
| 14 | -constexpr int64_t kRows = 6400; | ||
| 15 | -constexpr int64_t kColumns = 32; | ||
| 16 | -constexpr int64_t kTableRows = 315511; | ||
| 17 | -constexpr char kGraphName[] = "indirect_load_broadcast_index_where_simt_test"; | ||
| 18 | - | ||
| 19 | -using indirect_load_test::SetView; | ||
| 20 | - | ||
| 21 | -struct WhereGraphView { | ||
| 22 | - std::shared_ptr<af::AscGraph> graph; | ||
| 23 | - af::AxisId rows_axis; | ||
| 24 | - af::AxisId columns_axis; | ||
| 25 | - af::Expression rows; | ||
| 26 | - af::Expression columns; | ||
| 27 | - af::Expression table_rows; | ||
| 28 | -}; | ||
| 29 | - | ||
| 30 | -WhereGraphView CreateGraphView() { | ||
| 31 | - WhereGraphView view; | ||
| 32 | - view.graph = std::make_shared<af::AscGraph>(kGraphName); | ||
| 33 | - view.rows = view.graph->CreateSizeVar(kRows); | ||
| 34 | - view.columns = view.graph->CreateSizeVar(kColumns); | ||
| 35 | - view.table_rows = view.graph->CreateSizeVar(kTableRows); | ||
| 36 | - view.rows_axis = view.graph->CreateAxis("a0", view.rows).id; | ||
| 37 | - view.columns_axis = view.graph->CreateAxis("a1", view.columns).id; | ||
| 38 | - return view; | ||
| 39 | -} | ||
| 40 | - | ||
| 41 | -std::shared_ptr<af::AscGraph> CreateSubGraph() { | ||
| 42 | - const WhereGraphView view = CreateGraphView(); | ||
| 43 | - const auto axes = std::vector<af::AxisId>{view.rows_axis, view.columns_axis}; | ||
| 44 | - const auto output_repeats = std::vector<af::Expression>{view.rows, view.columns}; | ||
| 45 | - const auto output_strides = std::vector<af::Expression>{view.columns, af::ops::One}; | ||
| 46 | - const auto index_repeats = std::vector<af::Expression>{view.rows, af::ops::One}; | ||
| 47 | - const auto index_strides = std::vector<af::Expression>{af::ops::One, af::ops::Zero}; | ||
| 48 | - const auto table_repeats = std::vector<af::Expression>{view.table_rows, view.columns}; | ||
| 49 | - const auto table_strides = std::vector<af::Expression>{view.columns, af::ops::One}; | ||
| 50 | - | ||
| 51 | - af::ascir_op::Data index0("index0", *view.graph); | ||
| 52 | - index0.ir_attr.SetIndex(0); | ||
| 53 | - SetView(index0, axes, index_repeats, index_strides, af::DT_INT64); | ||
| 54 | - af::ascir_op::Load index0_load("index0_load"); | ||
| 55 | - view.graph->AddNode(index0_load); | ||
| 56 | - index0_load.x = index0.y; | ||
| 57 | - SetView(index0_load, axes, index_repeats, index_strides, af::DT_INT64); | ||
| 58 | - | ||
| 59 | - af::ascir_op::Broadcast index0_broadcast("index0_broadcast"); | ||
| 60 | - view.graph->AddNode(index0_broadcast); | ||
| 61 | - index0_broadcast.attr.api.compute_type = af::ComputeType::kComputeBroadcast; | ||
| 62 | - index0_broadcast.x = index0_load.y; | ||
| 63 | - SetView(index0_broadcast, axes, output_repeats, output_strides, af::DT_INT64); | ||
| 64 | - | ||
| 65 | - af::ascir_op::Cast index0_cast("index0_cast"); | ||
| 66 | - view.graph->AddNode(index0_cast); | ||
| 67 | - index0_cast.x = index0_broadcast.y; | ||
| 68 | - SetView(index0_cast, axes, output_repeats, output_strides, af::DT_FLOAT); | ||
| 69 | - | ||
| 70 | - af::ascir_op::Scalar minus_one("minus_one", *view.graph); | ||
| 71 | - minus_one.ir_attr.SetValue("-1"); | ||
| 72 | - minus_one.y.dtype = af::DT_INT64; | ||
| 73 | - af::ascir_op::Broadcast minus_one_row("minus_one_row"); | ||
| 74 | - view.graph->AddNode(minus_one_row); | ||
| 75 | - minus_one_row.attr.api.compute_type = af::ComputeType::kComputeBroadcast; | ||
| 76 | - minus_one_row.x = minus_one.y; | ||
| 77 | - SetView(minus_one_row, axes, index_repeats, index_strides, af::DT_INT64); | ||
| 78 | - af::ascir_op::Broadcast minus_one_full("minus_one_full"); | ||
| 79 | - view.graph->AddNode(minus_one_full); | ||
| 80 | - minus_one_full.attr.api.compute_type = af::ComputeType::kComputeBroadcast; | ||
| 81 | - minus_one_full.x = minus_one_row.y; | ||
| 82 | - SetView(minus_one_full, axes, output_repeats, output_strides, af::DT_INT64); | ||
| 83 | - af::ascir_op::Cast minus_one_cast("minus_one_cast"); | ||
| 84 | - view.graph->AddNode(minus_one_cast); | ||
| 85 | - minus_one_cast.x = minus_one_full.y; | ||
| 86 | - SetView(minus_one_cast, axes, output_repeats, output_strides, af::DT_FLOAT); | ||
| 87 | - | ||
| 88 | - af::ascir_op::Eq equal("equal"); | ||
| 89 | - view.graph->AddNode(equal); | ||
| 90 | - equal.x1 = index0_cast.y; | ||
| 91 | - equal.x2 = minus_one_cast.y; | ||
| 92 | - SetView(equal, axes, output_repeats, output_strides, af::DT_BOOL); | ||
| 93 | - | ||
| 94 | - af::ascir_op::Data index2("index2", *view.graph); | ||
| 95 | - index2.ir_attr.SetIndex(2); | ||
| 96 | - SetView(index2, axes, index_repeats, index_strides, af::DT_INT64); | ||
| 97 | - af::ascir_op::Load index2_load("index2_load"); | ||
| 98 | - view.graph->AddNode(index2_load); | ||
| 99 | - index2_load.x = index2.y; | ||
| 100 | - SetView(index2_load, axes, index_repeats, index_strides, af::DT_INT64); | ||
| 101 | - af::ascir_op::Broadcast index2_broadcast("index2_broadcast"); | ||
| 102 | - view.graph->AddNode(index2_broadcast); | ||
| 103 | - index2_broadcast.attr.api.compute_type = af::ComputeType::kComputeBroadcast; | ||
| 104 | - index2_broadcast.x = index2_load.y; | ||
| 105 | - SetView(index2_broadcast, axes, output_repeats, output_strides, af::DT_INT64); | ||
| 106 | - | ||
| 107 | - af::ascir_op::Where where("where"); | ||
| 108 | - view.graph->AddNode(where); | ||
| 109 | - where.x1 = equal.y; | ||
| 110 | - where.x2 = index2_broadcast.y; | ||
| 111 | - where.x3 = index0_broadcast.y; | ||
| 112 | - SetView(where, axes, output_repeats, output_strides, af::DT_INT64); | ||
| 113 | - | ||
| 114 | - af::ascir_op::Data table("table", *view.graph); | ||
| 115 | - table.ir_attr.SetIndex(1); | ||
| 116 | - SetView(table, axes, table_repeats, table_strides, af::DT_FLOAT); | ||
| 117 | - af::ascir_op::Load table_load("table_load"); | ||
| 118 | - view.graph->AddNode(table_load); | ||
| 119 | - table_load.x = table.y; | ||
| 120 | - SetView(table_load, axes, table_repeats, table_strides, af::DT_FLOAT); | ||
| 121 | - | ||
| 122 | - af::ascir_op::IndirectLoad indirect_load("indirect_load"); | ||
| 123 | - view.graph->AddNode(indirect_load); | ||
| 124 | - indirect_load.x1 = table_load.y; | ||
| 125 | - indirect_load.x2 = where.y; | ||
| 126 | - indirect_load.ir_attr.SetAxis(0); | ||
| 127 | - indirect_load.ir_attr.SetNegative_index_support(true); | ||
| 128 | - indirect_load.ir_attr.SetNeed_check_bound(true); | ||
| 129 | - SetView(indirect_load, axes, output_repeats, output_strides, af::DT_FLOAT); | ||
| 130 | - | ||
| 131 | - af::ascir_op::Store store("store"); | ||
| 132 | - view.graph->AddNode(store); | ||
| 133 | - store.x = indirect_load.y; | ||
| 134 | - SetView(store, axes, output_repeats, output_strides, af::DT_FLOAT); | ||
| 135 | - af::ascir_op::Output output("output"); | ||
| 136 | - view.graph->AddNode(output); | ||
| 137 | - output.ir_attr.SetIndex(0); | ||
| 138 | - output.x = store.y; | ||
| 139 | - SetView(output, axes, output_repeats, output_strides, af::DT_FLOAT); | ||
| 140 | - return view.graph; | ||
| 141 | -} | ||
| 142 | - | ||
| 143 | -class ThreeInputBackendGraph { | ||
| 144 | - public: | ||
| 145 | - explicit ThreeInputBackendGraph(const char *graph_name) : fused_graph_(graph_name) { | ||
| 146 | - af::ascir_op::Data index0("input0", fused_graph_); | ||
| 147 | - af::ascir_op::Data table("input1", fused_graph_); | ||
| 148 | - af::ascir_op::Data index2("input2", fused_graph_); | ||
| 149 | - index0.ir_attr.SetIndex(0); | ||
| 150 | - table.ir_attr.SetIndex(1); | ||
| 151 | - index2.ir_attr.SetIndex(2); | ||
| 152 | - compute_graph_ = af::AscGraphUtils::GetComputeGraph(fused_graph_); | ||
| 153 | - if (compute_graph_ == nullptr) { | ||
| 154 | - return; | ||
| 155 | - } | ||
| 156 | - const auto index_desc = std::make_shared<af::GeTensorDesc>(); | ||
| 157 | - index_desc->SetDataType(af::DT_INT64); | ||
| 158 | - const auto table_desc = std::make_shared<af::GeTensorDesc>(); | ||
| 159 | - table_desc->SetDataType(af::DT_FLOAT); | ||
| 160 | - const auto backend_desc = std::make_shared<af::OpDesc>("asc_backend", "AscBackend"); | ||
| 161 | - backend_desc->AddInputDesc(index_desc->Clone()); | ||
| 162 | - backend_desc->AddInputDesc(table_desc->Clone()); | ||
| 163 | - backend_desc->AddInputDesc(index_desc->Clone()); | ||
| 164 | - backend_desc->AddOutputDesc(table_desc->Clone()); | ||
| 165 | - backend_ = compute_graph_->AddNode(backend_desc); | ||
| 166 | - } | ||
| 167 | - | ||
| 168 | - af::ComputeGraphPtr Finalize(const std::shared_ptr<af::AscGraph> &sub_graph) { | ||
| 169 | - if (compute_graph_ == nullptr || backend_ == nullptr) { | ||
| 170 | - return nullptr; | ||
| 171 | - } | ||
| 172 | - const auto attrs = backend_->GetOpDesc()->GetOrCreateAttrsGroup<af::AutoFuseAttrs>(); | ||
| 173 | - if (attrs == nullptr) { | ||
| 174 | - return nullptr; | ||
| 175 | - } | ||
| 176 | - attrs->SetAscGraph(sub_graph); | ||
| 177 | - af::ascir_op::Output output("output"); | ||
| 178 | - output.ir_attr.SetIndex(0); | ||
| 179 | - const auto output_node = compute_graph_->AddNode(af::OpDescUtils::GetOpDescFromOperator(output)); | ||
| 180 | - const auto input0 = fused_graph_.FindNode("input0"); | ||
| 181 | - const auto input1 = fused_graph_.FindNode("input1"); | ||
| 182 | - const auto input2 = fused_graph_.FindNode("input2"); | ||
| 183 | - if (output_node == nullptr || input0 == nullptr || input1 == nullptr || input2 == nullptr) { | ||
| 184 | - return nullptr; | ||
| 185 | - } | ||
| 186 | - const bool edges_added = | ||
| 187 | - af::GraphUtils::AddEdge(input0->GetOutDataAnchor(0), backend_->GetInDataAnchor(0)) == ge::GRAPH_SUCCESS && | ||
| 188 | - af::GraphUtils::AddEdge(input1->GetOutDataAnchor(0), backend_->GetInDataAnchor(1)) == ge::GRAPH_SUCCESS && | ||
| 189 | - af::GraphUtils::AddEdge(input2->GetOutDataAnchor(0), backend_->GetInDataAnchor(2)) == ge::GRAPH_SUCCESS && | ||
| 190 | - af::GraphUtils::AddEdge(backend_->GetOutDataAnchor(0), output_node->GetInDataAnchor(0)) == ge::GRAPH_SUCCESS; | ||
| 191 | - return edges_added && compute_graph_->TopologicalSorting() == ge::GRAPH_SUCCESS ? compute_graph_ : nullptr; | ||
| 192 | - } | ||
| 193 | - | ||
| 194 | - private: | ||
| 195 | - af::AscGraph fused_graph_; | ||
| 196 | - af::ComputeGraphPtr compute_graph_; | ||
| 197 | - af::NodePtr backend_; | ||
| 198 | -}; | ||
| 199 | -} // namespace | ||
| 200 | - | ||
| 201 | -using TestBackendIndirectLoadBroadcastWhereE2e = indirect_load_test::PrecisionBackendE2e; | ||
| 202 | - | ||
| 203 | -TEST_F(TestBackendIndirectLoadBroadcastWhereE2e, GeneratesWhereIndirectLoadSimtKernel) { | ||
| 204 | - ThreeInputBackendGraph backend(kGraphName); | ||
| 205 | - const auto graph = backend.Finalize(CreateSubGraph()); | ||
| 206 | - ASSERT_NE(graph, nullptr); | ||
| 207 | - codegen::CodegenResult result; | ||
| 208 | - indirect_load_test::GenerateForTemplate(graph, {}, ascir::TemplateId::kIndirectLoadSimt, result); | ||
| 209 | - EXPECT_NE(result.kernel.find("// IndirectLoad SIMT"), std::string::npos); | ||
| 210 | - EXPECT_NE(result.kernel.find("IndirectLoadSimt"), std::string::npos); | ||
| 211 | - indirect_load_test::WriteGeneratedFiles(result); | ||
| 212 | -} | ||
Dautofuse/tests/v35/st/backend_e2e_v2/indirect_load_store_test/indirect_load_embedding_backend_generator.cpp+0-112
| @@ -1,112 +0,0 @@ | |||
| 1 | -/** | ||
| 2 | - * Copyright (c) 2026 Huawei Technologies Co., Ltd. | ||
| 3 | - * This program is free software, you can redistribute it and/or modify it under the terms and conditions of | ||
| 4 | - * CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 5 | - * Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 6 | - * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 7 | - * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 8 | - * See LICENSE in the root of the software repository for the full text of the License. | ||
| 9 | - */ | ||
| 10 | - | ||
| 11 | - | ||
| 12 | - | ||
| 13 | -namespace { | ||
| 14 | -constexpr char kGraphName[] = "indirect_load_embedding_test"; | ||
| 15 | -constexpr int64_t kInputRows = 100; | ||
| 16 | -constexpr int64_t kEmbeddingSize = 16; | ||
| 17 | -constexpr int64_t kIndexRows = 92; | ||
| 18 | - | ||
| 19 | -using indirect_load_test::SetView; | ||
| 20 | - | ||
| 21 | -struct EmbeddingGraphView { | ||
| 22 | - std::shared_ptr<af::AscGraph> graph; | ||
| 23 | - af::AxisId input_row_axis; | ||
| 24 | - af::AxisId input_inner_axis; | ||
| 25 | - af::AxisId output_row_axis; | ||
| 26 | - af::AxisId output_inner_axis; | ||
| 27 | - af::Expression input_rows; | ||
| 28 | - af::Expression embedding_size; | ||
| 29 | - af::Expression index_rows; | ||
| 30 | -}; | ||
| 31 | - | ||
| 32 | -EmbeddingGraphView CreateGraphView() { | ||
| 33 | - EmbeddingGraphView view; | ||
| 34 | - view.graph = std::make_shared<af::AscGraph>(kGraphName); | ||
| 35 | - view.input_rows = view.graph->CreateSizeVar(kInputRows); | ||
| 36 | - view.embedding_size = view.graph->CreateSizeVar(kEmbeddingSize); | ||
| 37 | - view.index_rows = view.graph->CreateSizeVar(kIndexRows); | ||
| 38 | - view.input_row_axis = view.graph->CreateAxis("input_row", view.input_rows).id; | ||
| 39 | - view.input_inner_axis = view.graph->CreateAxis("embedding_inner", view.embedding_size).id; | ||
| 40 | - view.output_row_axis = view.graph->CreateAxis("output_row", view.index_rows).id; | ||
| 41 | - view.output_inner_axis = view.graph->CreateAxis("output_inner", view.embedding_size).id; | ||
| 42 | - return view; | ||
| 43 | -} | ||
| 44 | - | ||
| 45 | -std::shared_ptr<af::AscGraph> CreateSubGraph() { | ||
| 46 | - const EmbeddingGraphView view = CreateGraphView(); | ||
| 47 | - const std::vector<af::AxisId> input_axes = {view.input_row_axis, view.input_inner_axis}; | ||
| 48 | - const std::vector<af::Expression> input_repeats = {view.input_rows, view.embedding_size}; | ||
| 49 | - const std::vector<af::Expression> input_strides = {view.embedding_size, af::ops::One}; | ||
| 50 | - const std::vector<af::AxisId> output_axes = {view.output_row_axis, view.output_inner_axis}; | ||
| 51 | - const std::vector<af::Expression> output_repeats = {view.index_rows, view.embedding_size}; | ||
| 52 | - const std::vector<af::Expression> output_strides = {view.embedding_size, af::ops::One}; | ||
| 53 | - const std::vector<af::Expression> index_repeats = {view.index_rows, af::ops::One}; | ||
| 54 | - const std::vector<af::Expression> index_strides = {af::ops::One, af::ops::Zero}; | ||
| 55 | - | ||
| 56 | - af::ascir_op::Data input("input", *view.graph); | ||
| 57 | - input.ir_attr.SetIndex(0); | ||
| 58 | - af::ascir_op::Load input_load("input_load"); | ||
| 59 | - view.graph->AddNode(input_load); | ||
| 60 | - input_load.x = input.y; | ||
| 61 | - SetView(input_load, input_axes, input_repeats, input_strides, af::DT_FLOAT); | ||
| 62 | - | ||
| 63 | - af::ascir_op::Data index("index", *view.graph); | ||
| 64 | - index.ir_attr.SetIndex(1); | ||
| 65 | - af::ascir_op::Load index_load("index_load"); | ||
| 66 | - view.graph->AddNode(index_load); | ||
| 67 | - index_load.x = index.y; | ||
| 68 | - SetView(index_load, output_axes, index_repeats, index_strides, af::DT_INT64); | ||
| 69 | - | ||
| 70 | - af::ascir_op::Broadcast index_broadcast("index_broadcast"); | ||
| 71 | - view.graph->AddNode(index_broadcast); | ||
| 72 | - index_broadcast.attr.api.compute_type = af::ComputeType::kComputeBroadcast; | ||
| 73 | - index_broadcast.x = index_load.y; | ||
| 74 | - SetView(index_broadcast, output_axes, output_repeats, output_strides, af::DT_INT64); | ||
| 75 | - | ||
| 76 | - af::ascir_op::IndirectLoad indirect_load("indirect_load"); | ||
| 77 | - view.graph->AddNode(indirect_load); | ||
| 78 | - indirect_load.x1 = input_load.y; | ||
| 79 | - indirect_load.x2 = index_broadcast.y; | ||
| 80 | - indirect_load.ir_attr.SetAxis(0); | ||
| 81 | - SetView(indirect_load, output_axes, output_repeats, output_strides, af::DT_FLOAT); | ||
| 82 | - | ||
| 83 | - af::ascir_op::Store store("store"); | ||
| 84 | - view.graph->AddNode(store); | ||
| 85 | - store.x = indirect_load.y; | ||
| 86 | - SetView(store, output_axes, output_repeats, output_strides, af::DT_FLOAT); | ||
| 87 | - | ||
| 88 | - af::ascir_op::Output output("output"); | ||
| 89 | - view.graph->AddNode(output); | ||
| 90 | - output.ir_attr.SetIndex(0); | ||
| 91 | - output.x = store.y; | ||
| 92 | - SetView(output, output_axes, output_repeats, output_strides, af::DT_FLOAT); | ||
| 93 | - return view.graph; | ||
| 94 | -} | ||
| 95 | - | ||
| 96 | -af::ComputeGraphPtr CreateGraph() { | ||
| 97 | - indirect_load_test::BackendGraph backend(kGraphName, "input0", "input1", af::DT_FLOAT); | ||
| 98 | - return backend.Finalize(CreateSubGraph(), "output0"); | ||
| 99 | -} | ||
| 100 | -} // namespace | ||
| 101 | - | ||
| 102 | -using TestBackendIndirectLoadEmbeddingE2e = indirect_load_test::BackendE2e; | ||
| 103 | - | ||
| 104 | -TEST_F(TestBackendIndirectLoadEmbeddingE2e, GeneratesEmbeddingIndirectLoadKernel) { | ||
| 105 | - const auto graph = CreateGraph(); | ||
| 106 | - ASSERT_NE(graph, nullptr); | ||
| 107 | - codegen::CodegenResult result; | ||
| 108 | - indirect_load_test::GenerateForTemplate(graph, {}, ascir::TemplateId::kIndirectLoadSimd, result); | ||
| 109 | - EXPECT_NE(result.kernel.find("// IndirectLoad SIMD"), std::string::npos); | ||
| 110 | - EXPECT_NE(result.kernel.find("IndirectLoadSimd<"), std::string::npos); | ||
| 111 | - indirect_load_test::WriteGeneratedFiles(result); | ||
| 112 | -} | ||
Dautofuse/tests/v35/st/backend_e2e_v2/indirect_load_store_test/indirect_load_kernel_test_common.h+0-68
| @@ -1,68 +0,0 @@ | |||
| 1 | -/** | ||
| 2 | - * Copyright (c) 2026 Huawei Technologies Co., Ltd. | ||
| 3 | - * This program is free software, you can redistribute it and/or modify it under the terms and conditions of | ||
| 4 | - * CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 5 | - * Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 6 | - * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 7 | - * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 8 | - * See LICENSE in the root of the software repository for the full text of the License. | ||
| 9 | - */ | ||
| 10 | - | ||
| 11 | - | ||
| 12 | - | ||
| 13 | - | ||
| 14 | - | ||
| 15 | - | ||
| 16 | - | ||
| 17 | - | ||
| 18 | - | ||
| 19 | - | ||
| 20 | - | ||
| 21 | - | ||
| 22 | - | ||
| 23 | -extern "C" int64_t AutofuseTiling(AutofuseTilingData *, uint32_t *, uint32_t *, uint32_t, uint32_t); | ||
| 24 | - | ||
| 25 | -namespace indirect_load_test { | ||
| 26 | -inline void GmFree(void *ptr) { | ||
| 27 | - AscendC::GmFree(ptr); | ||
| 28 | -} | ||
| 29 | - | ||
| 30 | -template <typename DataType, typename IndexType> | ||
| 31 | -struct KernelData { | ||
| 32 | - KernelData(int64_t input_count, int64_t index_count, int64_t output_count) | ||
| 33 | - : input(reinterpret_cast<DataType *>(AscendC::GmAlloc(input_count * sizeof(DataType))), GmFree), | ||
| 34 | - index(reinterpret_cast<IndexType *>(AscendC::GmAlloc(index_count * sizeof(IndexType))), GmFree), | ||
| 35 | - output(reinterpret_cast<DataType *>(AscendC::GmAlloc(output_count * sizeof(DataType))), GmFree), | ||
| 36 | - expected(static_cast<size_t>(output_count)) {} | ||
| 37 | - | ||
| 38 | - [[nodiscard]] bool IsValid() const { | ||
| 39 | - return input != nullptr && index != nullptr && output != nullptr; | ||
| 40 | - } | ||
| 41 | - | ||
| 42 | - std::unique_ptr<DataType, decltype(&GmFree)> input; | ||
| 43 | - std::unique_ptr<IndexType, decltype(&GmFree)> index; | ||
| 44 | - std::unique_ptr<DataType, decltype(&GmFree)> output; | ||
| 45 | - std::vector<DataType> expected; | ||
| 46 | -}; | ||
| 47 | - | ||
| 48 | -struct KernelTiling { | ||
| 49 | - KernelTiling() : workspace(nullptr, GmFree) { | ||
| 50 | - EXPECT_EQ(AutofuseTiling(&data, &workspace_size, &block_dim, 48U, 192U * 1024U), 0); | ||
| 51 | - EXPECT_GT(data.block_dim, 0U); | ||
| 52 | - if (workspace_size != 0U) { | ||
| 53 | - workspace.reset(reinterpret_cast<uint8_t *>(AscendC::GmAlloc(workspace_size))); | ||
| 54 | - } | ||
| 55 | - } | ||
| 56 | - | ||
| 57 | - [[nodiscard]] bool IsValid() const { | ||
| 58 | - return workspace_size == 0U || workspace != nullptr; | ||
| 59 | - } | ||
| 60 | - | ||
| 61 | - AutofuseTilingData data{}; | ||
| 62 | - uint32_t workspace_size = 0U; | ||
| 63 | - uint32_t block_dim = 48U; | ||
| 64 | - std::unique_ptr<uint8_t, decltype(&GmFree)> workspace; | ||
| 65 | -}; | ||
| 66 | -} // namespace indirect_load_test | ||
| 67 | - | ||
| 68 | - | ||
Mautofuse/tests/v35/st/backend_e2e_v2/indirect_load_store_test/indirect_load_store_backend_generator.cpp+2219-4
Dautofuse/tests/v35/st/backend_e2e_v2/indirect_load_store_test/indirect_load_stride_zero_backend_generator.cpp+0-128
| @@ -1,128 +0,0 @@ | |||
| 1 | -/** | ||
| 2 | - * Copyright (c) 2026 Huawei Technologies Co., Ltd. | ||
| 3 | - * This program is free software, you can redistribute it and/or modify it under the terms and conditions of | ||
| 4 | - * CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 5 | - * Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 6 | - * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 7 | - * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 8 | - * See LICENSE in the root of the software repository for the full text of the License. | ||
| 9 | - */ | ||
| 10 | - | ||
| 11 | - | ||
| 12 | - | ||
| 13 | -namespace { | ||
| 14 | -constexpr uint32_t kInputZeroStrideMask = IL_INPUT_ZERO_STRIDE_MASK; | ||
| 15 | -constexpr uint32_t kIndexZeroStrideMask = IL_INDEX_ZERO_STRIDE_MASK; | ||
| 16 | -constexpr int32_t kInputElementCount = IL_HAS_INPUT_ELEMENT; | ||
| 17 | -constexpr int32_t kIndexElementCount = IL_HAS_INDEX_ELEMENT; | ||
| 18 | -constexpr bool kExpectSimt = IL_EXPECT_SIMT; | ||
| 19 | -constexpr bool kExpectSk = IL_EXPECT_SK; | ||
| 20 | - | ||
| 21 | -using indirect_load_test::SetView; | ||
| 22 | - | ||
| 23 | -struct StrideZeroGraphView { | ||
| 24 | - std::shared_ptr<af::AscGraph> graph; | ||
| 25 | - std::vector<af::AxisId> axes; | ||
| 26 | - std::vector<af::Expression> repeats; | ||
| 27 | - std::vector<af::Expression> dense_strides; | ||
| 28 | - std::vector<af::Expression> input_strides; | ||
| 29 | - std::vector<af::Expression> index_strides; | ||
| 30 | -}; | ||
| 31 | - | ||
| 32 | -std::vector<af::Expression> MakeStrides(const std::vector<af::Expression> &repeats, uint32_t zero_stride_mask) { | ||
| 33 | - std::vector<af::Expression> strides(repeats.size(), af::ops::Zero); | ||
| 34 | - af::Expression stride = af::ops::One; | ||
| 35 | - for (size_t index = repeats.size(); index > 0UL; --index) { | ||
| 36 | - const size_t dim = index - 1UL; | ||
| 37 | - if ((zero_stride_mask & (1U << dim)) == 0U) { | ||
| 38 | - strides[dim] = stride; | ||
| 39 | - stride = stride * repeats[dim]; | ||
| 40 | - } | ||
| 41 | - } | ||
| 42 | - return strides; | ||
| 43 | -} | ||
| 44 | - | ||
| 45 | -StrideZeroGraphView CreateGraphView() { | ||
| 46 | - StrideZeroGraphView view; | ||
| 47 | - view.graph = std::make_shared<af::AscGraph>("indirect_load_stride_zero_test"); | ||
| 48 | - const std::vector<int64_t> shape = {4, 5, 4, 16}; | ||
| 49 | - for (size_t dim = 0; dim < shape.size(); ++dim) { | ||
| 50 | - const auto size = view.graph->CreateSizeVar(shape[dim]); | ||
| 51 | - view.repeats.emplace_back(size); | ||
| 52 | - view.axes.emplace_back(view.graph->CreateAxis(("z" + std::to_string(dim)).c_str(), size).id); | ||
| 53 | - } | ||
| 54 | - view.dense_strides = MakeStrides(view.repeats, 0U); | ||
| 55 | - view.input_strides = MakeStrides(view.repeats, kInputZeroStrideMask); | ||
| 56 | - view.index_strides = MakeStrides(view.repeats, kIndexZeroStrideMask); | ||
| 57 | - return view; | ||
| 58 | -} | ||
| 59 | - | ||
| 60 | -void BuildInputPath(const StrideZeroGraphView &view, af::ascir_op::IndirectLoad &indirect_load) { | ||
| 61 | - af::ascir_op::Data x("x"); | ||
| 62 | - view.graph->AddNode(x); | ||
| 63 | - x.ir_attr.SetIndex(0); | ||
| 64 | - SetView(x, view.axes, view.repeats, view.input_strides, af::DT_FLOAT16); | ||
| 65 | - af::ascir_op::Load input_load("input_load"); | ||
| 66 | - view.graph->AddNode(input_load); | ||
| 67 | - input_load.x = x.y; | ||
| 68 | - SetView(input_load, view.axes, view.repeats, view.input_strides, af::DT_FLOAT16); | ||
| 69 | - | ||
| 70 | - std::vector<std::unique_ptr<af::ascir_op::Abs>> input_elements; | ||
| 71 | - for (int32_t i = 0; i < kInputElementCount; ++i) { | ||
| 72 | - const auto name = "input_abs_" + std::to_string(i); | ||
| 73 | - auto input_abs = std::make_unique<af::ascir_op::Abs>(name.c_str()); | ||
| 74 | - view.graph->AddNode(*input_abs); | ||
| 75 | - input_abs->attr.api.compute_type = af::ComputeType::kComputeElewise; | ||
| 76 | - input_abs->x = i == 0 ? input_load.y : input_elements.back()->y; | ||
| 77 | - SetView(*input_abs, view.axes, view.repeats, view.input_strides, af::DT_FLOAT16); | ||
| 78 | - input_elements.emplace_back(std::move(input_abs)); | ||
| 79 | - } | ||
| 80 | - indirect_load.x1 = input_elements.empty() ? input_load.y : input_elements.back()->y; | ||
| 81 | -} | ||
| 82 | - | ||
| 83 | -void BuildIndexPath(const StrideZeroGraphView &view, af::ascir_op::IndirectLoad &indirect_load) { | ||
| 84 | - af::ascir_op::Data index("index"); | ||
| 85 | - view.graph->AddNode(index); | ||
| 86 | - index.ir_attr.SetIndex(1); | ||
| 87 | - SetView(index, view.axes, view.repeats, view.index_strides, af::DT_INT64); | ||
| 88 | - af::ascir_op::Load index_load("index_load"); | ||
| 89 | - view.graph->AddNode(index_load); | ||
| 90 | - index_load.x = index.y; | ||
| 91 | - SetView(index_load, view.axes, view.repeats, view.index_strides, af::DT_INT64); | ||
| 92 | - std::vector<std::unique_ptr<af::ascir_op::Abs>> index_elements; | ||
| 93 | - for (int32_t i = 0; i < kIndexElementCount; ++i) { | ||
| 94 | - const auto name = "index_abs_" + std::to_string(i); | ||
| 95 | - auto index_abs = std::make_unique<af::ascir_op::Abs>(name.c_str()); | ||
| 96 | - view.graph->AddNode(*index_abs); | ||
| 97 | - index_abs->attr.api.compute_type = af::ComputeType::kComputeElewise; | ||
| 98 | - index_abs->x = i == 0 ? index_load.y : index_elements.back()->y; | ||
| 99 | - SetView(*index_abs, view.axes, view.repeats, view.index_strides, af::DT_INT64); | ||
| 100 | - index_elements.emplace_back(std::move(index_abs)); | ||
| 101 | - } | ||
| 102 | - indirect_load.x2 = index_elements.empty() ? index_load.y : index_elements.back()->y; | ||
| 103 | -} | ||
| 104 | - | ||
| 105 | -void BuildOutputPath(const StrideZeroGraphView &view, af::ascir_op::IndirectLoad &indirect_load) { | ||
| 106 | - indirect_load_test::BuildOutputPath(view.graph, indirect_load, view.axes, view.repeats, view.dense_strides, true); | ||
| 107 | -} | ||
| 108 | - | ||
| 109 | -af::ComputeGraphPtr CreateGraph() { | ||
| 110 | - indirect_load_test::BackendGraph backend("indirect_load_stride_zero_test", "data0", "data1", af::DT_FLOAT16); | ||
| 111 | - return backend.Finalize( | ||
| 112 | - indirect_load_test::CreateSubGraph(CreateGraphView(), BuildInputPath, BuildIndexPath, BuildOutputPath), "output"); | ||
| 113 | -} | ||
| 114 | -} // namespace | ||
| 115 | - | ||
| 116 | -using TestBackendIndirectLoadStrideZeroE2e = indirect_load_test::PrecisionBackendE2e; | ||
| 117 | - | ||
| 118 | -TEST_F(TestBackendIndirectLoadStrideZeroE2e, GeneratesSelectedTemplateWithoutBroadcastMaterialization) { | ||
| 119 | - const auto graph = CreateGraph(); | ||
| 120 | - ASSERT_NE(graph, nullptr); | ||
| 121 | - const std::map<std::string, std::string> shape_info; | ||
| 122 | - const auto expected_template = indirect_load_test::GetExpectedTemplate(kExpectSimt, kExpectSk); | ||
| 123 | - codegen::CodegenResult result; | ||
| 124 | - indirect_load_test::GenerateForTemplate(graph, shape_info, expected_template, result); | ||
| 125 | - EXPECT_NE(result.kernel.find(indirect_load_test::GetTemplateMarker(expected_template)), std::string::npos); | ||
| 126 | - EXPECT_EQ(result.kernel.find("BroadcastExtend<"), std::string::npos); | ||
| 127 | - indirect_load_test::WriteGeneratedFiles(result); | ||
| 128 | -} | ||
Dautofuse/tests/v35/st/backend_e2e_v2/indirect_load_store_test/indirect_load_torch_gather_strided_backend_generator.cpp+0-171
| @@ -1,171 +0,0 @@ | |||
| 1 | -/** | ||
| 2 | - * Copyright (c) 2026 Huawei Technologies Co., Ltd. | ||
| 3 | - * This program is free software, you can redistribute it and/or modify it under the terms and conditions of | ||
| 4 | - * CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 5 | - * Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 6 | - * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 7 | - * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 8 | - * See LICENSE in the root of the software repository for the full text of the License. | ||
| 9 | - */ | ||
| 10 | - | ||
| 11 | - | ||
| 12 | - | ||
| 13 | -namespace { | ||
| 14 | -constexpr char kGraphName[] = "indirect_load_torch_gather_strided_test"; | ||
| 15 | -constexpr bool kExpectSimt = IL_EXPECT_SIMT; | ||
| 16 | -constexpr bool kExpectSk = IL_EXPECT_SK; | ||
| 17 | - | ||
| 18 | -constexpr int64_t kDim0 = 30; | ||
| 19 | -constexpr int64_t kInputDim1 = 6; | ||
| 20 | -constexpr int64_t kOutputDim1 = 3; | ||
| 21 | -constexpr int64_t kDim2 = 23; | ||
| 22 | -constexpr int64_t kEffectiveInputStride0 = 138; | ||
| 23 | -constexpr int64_t kEffectiveInputStride1 = 23; | ||
| 24 | -constexpr int64_t kEffectiveInputStride2 = 1; | ||
| 25 | - | ||
| 26 | -constexpr int64_t kDim0 = 8; | ||
| 27 | -constexpr int64_t kInputDim1 = 32; | ||
| 28 | -constexpr int64_t kOutputDim1 = 16; | ||
| 29 | -constexpr int64_t kDim2 = 5; | ||
| 30 | - | ||
| 31 | -constexpr int64_t kInputStride0 = IL_INPUT_STRIDE0; | ||
| 32 | -constexpr int64_t kInputStride1 = IL_INPUT_STRIDE1; | ||
| 33 | -constexpr int64_t kInputStride2 = IL_INPUT_STRIDE2; | ||
| 34 | -constexpr int64_t kIndexStride0 = IL_INDEX_STRIDE0; | ||
| 35 | -constexpr int64_t kIndexStride1 = IL_INDEX_STRIDE1; | ||
| 36 | -constexpr int64_t kIndexStride2 = IL_INDEX_STRIDE2; | ||
| 37 | - | ||
| 38 | -constexpr int64_t kEffectiveInputStride0 = kInputStride0; | ||
| 39 | -constexpr int64_t kEffectiveInputStride1 = kInputStride1; | ||
| 40 | -constexpr int64_t kEffectiveInputStride2 = kInputStride2; | ||
| 41 | - | ||
| 42 | - | ||
| 43 | -using indirect_load_test::SetView; | ||
| 44 | - | ||
| 45 | -struct TorchGatherGraphView { | ||
| 46 | - std::shared_ptr<af::AscGraph> graph; | ||
| 47 | - std::vector<af::AxisId> axes; | ||
| 48 | - std::vector<af::Expression> output_sizes; | ||
| 49 | - std::vector<af::Expression> output_strides; | ||
| 50 | - std::vector<af::Expression> index_strides; | ||
| 51 | - std::vector<af::Expression> input_sizes; | ||
| 52 | - std::vector<af::Expression> input_strides; | ||
| 53 | -}; | ||
| 54 | - | ||
| 55 | -TorchGatherGraphView CreateGraphView() { | ||
| 56 | - TorchGatherGraphView view; | ||
| 57 | - view.graph = std::make_shared<af::AscGraph>(kGraphName); | ||
| 58 | - view.output_sizes = {af::Symbol(kDim0), af::Symbol(kOutputDim1), af::Symbol(kDim2)}; | ||
| 59 | - view.output_strides = {af::Symbol(kOutputDim1 * kDim2), af::Symbol(kDim2), af::Symbol(1)}; | ||
| 60 | - view.index_strides = {af::Symbol(kIndexStride0), af::Symbol(kIndexStride1), af::Symbol(kIndexStride2)}; | ||
| 61 | - view.input_sizes = {af::Symbol(kDim0), af::Symbol(kInputDim1), af::Symbol(kDim2)}; | ||
| 62 | - view.input_strides = {af::Symbol(kEffectiveInputStride0), af::Symbol(kEffectiveInputStride1), | ||
| 63 | - af::Symbol(kEffectiveInputStride2)}; | ||
| 64 | - const auto a0 = view.graph->CreateAxis("a0", view.output_sizes[0]); | ||
| 65 | - const auto a1 = view.graph->CreateAxis("a1", view.output_sizes[1]); | ||
| 66 | - const auto a2 = view.graph->CreateAxis("a2", view.output_sizes[2]); | ||
| 67 | - view.axes = {a0.id, a1.id, a2.id}; | ||
| 68 | - return view; | ||
| 69 | -} | ||
| 70 | - | ||
| 71 | -std::shared_ptr<af::AscGraph> CreateSubGraph() { | ||
| 72 | - const auto view = CreateGraphView(); | ||
| 73 | - af::ascir_op::Data index("graph_hint/data", *view.graph); | ||
| 74 | - index.ir_attr.SetIndex(1); | ||
| 75 | - index.y.dtype = af::DT_INT64; | ||
| 76 | - af::ascir_op::Load index_load("graph_hint/load"); | ||
| 77 | - view.graph->AddNode(index_load); | ||
| 78 | - index_load.ir_attr.SetOffset(af::sym::kSymbolZero); | ||
| 79 | - index_load.x = index.y; | ||
| 80 | - | ||
| 81 | - const std::vector<af::Expression> index_source_sizes = {af::Symbol(1), af::Symbol(kOutputDim1), af::Symbol(1)}; | ||
| 82 | - const std::vector<af::Expression> index_source_strides = {af::ops::Zero, af::ops::One, af::ops::Zero}; | ||
| 83 | - SetView(index_load, view.axes, index_source_sizes, index_source_strides, af::DT_INT64); | ||
| 84 | - af::ascir_op::Broadcast index_first_broadcast("graph_hint/index_broadcast0"); | ||
| 85 | - view.graph->AddNode(index_first_broadcast); | ||
| 86 | - index_first_broadcast.attr.api.compute_type = af::ComputeType::kComputeBroadcast; | ||
| 87 | - index_first_broadcast.x = index_load.y; | ||
| 88 | - const std::vector<af::Expression> index_intermediate_sizes = {af::Symbol(kDim0), af::Symbol(kOutputDim1), | ||
| 89 | - af::Symbol(1)}; | ||
| 90 | - const std::vector<af::Expression> index_intermediate_strides = {af::Symbol(kOutputDim1), af::ops::One, af::ops::Zero}; | ||
| 91 | - SetView(index_first_broadcast, view.axes, index_intermediate_sizes, index_intermediate_strides, af::DT_INT64); | ||
| 92 | - af::ascir_op::Broadcast index_final_broadcast("graph_hint/index_broadcast1"); | ||
| 93 | - view.graph->AddNode(index_final_broadcast); | ||
| 94 | - index_final_broadcast.attr.api.compute_type = af::ComputeType::kComputeBroadcast; | ||
| 95 | - index_final_broadcast.x = index_first_broadcast.y; | ||
| 96 | - SetView(index_final_broadcast, view.axes, view.output_sizes, view.output_strides, af::DT_INT64); | ||
| 97 | - const auto index_output = index_final_broadcast.y; | ||
| 98 | - | ||
| 99 | - SetView(index_load, view.axes, view.output_sizes, view.index_strides, af::DT_INT64); | ||
| 100 | - | ||
| 101 | - | ||
| 102 | - af::ascir_op::Data data("graph_hint/data1", *view.graph); | ||
| 103 | - data.ir_attr.SetIndex(0); | ||
| 104 | - data.y.dtype = af::DT_FLOAT; | ||
| 105 | - af::ascir_op::Load data_load("graph_hint/load1"); | ||
| 106 | - view.graph->AddNode(data_load); | ||
| 107 | - data_load.ir_attr.SetOffset(af::sym::kSymbolZero); | ||
| 108 | - data_load.x = data.y; | ||
| 109 | - SetView(data_load, view.axes, view.input_sizes, view.input_strides, af::DT_FLOAT); | ||
| 110 | - | ||
| 111 | - af::ascir_op::IndirectLoad indirect_load("graph_hint/indirectload"); | ||
| 112 | - view.graph->AddNode(indirect_load); | ||
| 113 | - indirect_load.x1 = data_load.y; | ||
| 114 | - | ||
| 115 | - indirect_load.x2 = index_output; | ||
| 116 | - | ||
| 117 | - indirect_load.x2 = index_load.y; | ||
| 118 | - | ||
| 119 | - indirect_load.ir_attr.SetAxis(1); | ||
| 120 | - indirect_load.ir_attr.SetNegative_index_support(true); | ||
| 121 | - indirect_load.ir_attr.SetNeed_check_bound(true); | ||
| 122 | - SetView(indirect_load, view.axes, view.output_sizes, view.output_strides, af::DT_FLOAT); | ||
| 123 | - | ||
| 124 | - af::ascir_op::Store store("graph_hint/store"); | ||
| 125 | - view.graph->AddNode(store); | ||
| 126 | - store.ir_attr.SetOffset(af::sym::kSymbolZero); | ||
| 127 | - store.x = indirect_load.y; | ||
| 128 | - SetView(store, view.axes, view.output_sizes, view.output_strides, af::DT_FLOAT); | ||
| 129 | - af::ascir_op::Output output("graph_hint/output"); | ||
| 130 | - view.graph->AddNode(output); | ||
| 131 | - output.ir_attr.SetIndex(0); | ||
| 132 | - output.x = store.y; | ||
| 133 | - output.y.dtype = af::DT_FLOAT; | ||
| 134 | - return view.graph; | ||
| 135 | -} | ||
| 136 | - | ||
| 137 | -af::ComputeGraphPtr CreateGraph() { | ||
| 138 | - indirect_load_test::BackendGraph backend(kGraphName, "input0", "input1", af::DT_FLOAT); | ||
| 139 | - return backend.Finalize(CreateSubGraph(), "output0"); | ||
| 140 | -} | ||
| 141 | -} // namespace | ||
| 142 | - | ||
| 143 | -using TestIndirectLoadTorchGatherStridedE2e = indirect_load_test::BackendE2e; | ||
| 144 | - | ||
| 145 | -TEST_F(TestIndirectLoadTorchGatherStridedE2e, GeneratesKernelForInductorGraph) { | ||
| 146 | - const auto graph = CreateGraph(); | ||
| 147 | - ASSERT_NE(graph, nullptr); | ||
| 148 | - const auto expected_template = indirect_load_test::GetExpectedTemplate(kExpectSimt, kExpectSk); | ||
| 149 | - codegen::CodegenResult result; | ||
| 150 | - indirect_load_test::GenerateForTemplate(graph, {}, expected_template, result); | ||
| 151 | - EXPECT_NE(result.kernel.find(indirect_load_test::GetTemplateMarker(expected_template)), std::string::npos); | ||
| 152 | - if (!kExpectSk) { | ||
| 153 | - EXPECT_EQ(result.kernel.find("// IndirectLoad SK"), std::string::npos); | ||
| 154 | - } | ||
| 155 | - if (!kExpectSimt) { | ||
| 156 | - EXPECT_EQ(result.kernel.find("// IndirectLoad SIMT"), std::string::npos); | ||
| 157 | - } | ||
| 158 | - if (kExpectSimt || kExpectSk) { | ||
| 159 | - EXPECT_EQ(result.kernel.find("// IndirectLoad SIMD"), std::string::npos); | ||
| 160 | - } | ||
| 161 | - if (kExpectSimt) { | ||
| 162 | - | ||
| 163 | - EXPECT_NE(result.kernel.find("IndirectLoadSimtStridedPolicy<uint32_t, 3, 1, 7ULL, 2ULL>"), std::string::npos); | ||
| 164 | - | ||
| 165 | - EXPECT_NE(result.kernel.find("IndirectLoadSimtStridedPolicy<uint32_t, 3, 1, 7ULL, 7ULL>"), std::string::npos); | ||
| 166 | - | ||
| 167 | - EXPECT_EQ(result.kernel.find("x_axis_size"), std::string::npos); | ||
| 168 | - EXPECT_EQ(result.kernel.find("indirect_index < 0"), std::string::npos); | ||
| 169 | - } | ||
| 170 | - indirect_load_test::WriteGeneratedFiles(result); | ||
| 171 | -} | ||
Dautofuse/tests/v35/st/backend_e2e_v2/indirect_load_store_test/indirect_load_torch_gather_strided_cases.cmake+0-45
| @@ -1,45 +0,0 @@ | |||
| 1 | -function(add_indirect_load_torch_gather_strided_test test_name tiling_key expect_simt expect_sk | ||
| 2 | - input_stride0 input_stride1 input_stride2 index_stride0 index_stride1 index_stride2) | ||
| 3 | - set(case_workdir ${CMAKE_CURRENT_BINARY_DIR}/${test_name}) | ||
| 4 | - file(MAKE_DIRECTORY ${case_workdir}) | ||
| 5 | - do_backend_e2e_st_test(${test_name} | ||
| 6 | - WORKDIR ${case_workdir} | ||
| 7 | - CODEGEN indirect_load_torch_gather_strided_backend_generator.cpp | ||
| 8 | - TILING_KEY ${tiling_key} | ||
| 9 | - KERNEL_SRC | ||
| 10 | - indirect_load_torch_gather_strided_test_kernel.cpp | ||
| 11 | - indirect_load_torch_gather_strided_test_tiling.cpp | ||
| 12 | - autofuse_tiling_data.h | ||
| 13 | - TEST_SRC test_e2e_indirect_load_torch_gather_strided_kernel.cpp) | ||
| 14 | - set(case_definitions | ||
| 15 | - IL_EXPECT_TILING_KEY=${tiling_key} | ||
| 16 | - IL_EXPECT_SIMT=${expect_simt} | ||
| 17 | - IL_EXPECT_SK=${expect_sk} | ||
| 18 | - IL_INPUT_STRIDE0=${input_stride0} | ||
| 19 | - IL_INPUT_STRIDE1=${input_stride1} | ||
| 20 | - IL_INPUT_STRIDE2=${input_stride2} | ||
| 21 | - IL_INDEX_STRIDE0=${index_stride0} | ||
| 22 | - IL_INDEX_STRIDE1=${index_stride1} | ||
| 23 | - IL_INDEX_STRIDE2=${index_stride2}) | ||
| 24 | - target_compile_definitions(${test_name}_codegen_v2 PRIVATE ${case_definitions}) | ||
| 25 | - target_compile_definitions(${test_name}_e2e_v2 PRIVATE ${case_definitions}) | ||
| 26 | -endfunction() | ||
| 27 | - | ||
| 28 | -function(add_indirect_load_strided_template_tests case_name input_stride0 input_stride1 input_stride2 | ||
| 29 | - index_stride0 index_stride1 index_stride2) | ||
| 30 | - add_indirect_load_torch_gather_strided_test(${case_name}_simd_test 0 0 0 | ||
| 31 | - ${input_stride0} ${input_stride1} ${input_stride2} ${index_stride0} ${index_stride1} ${index_stride2}) | ||
| 32 | - add_indirect_load_torch_gather_strided_test(${case_name}_simt_test 0 1 0 | ||
| 33 | - ${input_stride0} ${input_stride1} ${input_stride2} ${index_stride0} ${index_stride1} ${index_stride2}) | ||
| 34 | - add_indirect_load_torch_gather_strided_test(${case_name}_sk_test 0 0 1 | ||
| 35 | - ${input_stride0} ${input_stride1} ${input_stride2} ${index_stride0} ${index_stride1} ${index_stride2}) | ||
| 36 | -endfunction() | ||
| 37 | - | ||
| 38 | -# Keep the SIMD/SK inner-gap coverage and use its SIMT variant for the common IndexSelect Broadcast layout. | ||
| 39 | -add_indirect_load_strided_template_tests(indirect_load_rank3_axis1_input_index_gap 384 10 1 192 10 1) | ||
| 40 | -add_indirect_load_strided_template_tests(indirect_load_rank3_axis1_input_index_outer_gap 192 5 1 128 5 1) | ||
| 41 | -set(indirect_load_index_select_defs IL_INDEX_SELECT_CASE=1) | ||
| 42 | -target_compile_definitions(indirect_load_rank3_axis1_input_index_gap_simt_test_codegen_v2 PRIVATE | ||
| 43 | - ${indirect_load_index_select_defs}) | ||
| 44 | -target_compile_definitions(indirect_load_rank3_axis1_input_index_gap_simt_test_e2e_v2 PRIVATE | ||
| 45 | - ${indirect_load_index_select_defs}) | ||
Dautofuse/tests/v35/st/backend_e2e_v2/indirect_load_store_test/test_e2e_indirect_load_broadcast_kernel.cpp+0-258
| @@ -1,258 +0,0 @@ | |||
| 1 | -/** | ||
| 2 | - * Copyright (c) 2026 Huawei Technologies Co., Ltd. | ||
| 3 | - * This program is free software, you can redistribute it and/or modify it under the terms and conditions of | ||
| 4 | - * CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 5 | - * Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 6 | - * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 7 | - * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 8 | - * See LICENSE in the root of the software repository for the full text of the License. | ||
| 9 | - */ | ||
| 10 | - | ||
| 11 | - | ||
| 12 | - | ||
| 13 | - | ||
| 14 | - | ||
| 15 | - | ||
| 16 | - | ||
| 17 | - | ||
| 18 | - | ||
| 19 | - | ||
| 20 | - | ||
| 21 | - | ||
| 22 | - | ||
| 23 | - | ||
| 24 | - | ||
| 25 | - | ||
| 26 | - | ||
| 27 | - | ||
| 28 | - | ||
| 29 | - | ||
| 30 | - | ||
| 31 | - | ||
| 32 | - | ||
| 33 | - | ||
| 34 | - | ||
| 35 | - | ||
| 36 | - | ||
| 37 | - | ||
| 38 | - | ||
| 39 | - | ||
| 40 | - | ||
| 41 | - | ||
| 42 | - | ||
| 43 | - | ||
| 44 | - | ||
| 45 | - | ||
| 46 | - | ||
| 47 | - | ||
| 48 | - | ||
| 49 | - | ||
| 50 | - | ||
| 51 | - | ||
| 52 | - | ||
| 53 | -extern "C" __global__ __aicore__ void indirect_load_aic_repro(GM_ADDR input, GM_ADDR index, GM_ADDR output, | ||
| 54 | - GM_ADDR workspace, GM_ADDR tiling); | ||
| 55 | - | ||
| 56 | -extern "C" __global__ __aicore__ void indirect_load_broadcast_test(GM_ADDR x, GM_ADDR index, GM_ADDR y, | ||
| 57 | - GM_ADDR workspace, GM_ADDR tiling); | ||
| 58 | - | ||
| 59 | - | ||
| 60 | -namespace { | ||
| 61 | - | ||
| 62 | -using DataType = float; | ||
| 63 | -using IndexType = int64_t; | ||
| 64 | -constexpr int32_t kInputRows = 100000; | ||
| 65 | -constexpr int32_t kRows = 1024; | ||
| 66 | -constexpr int32_t kColumns = 1024; | ||
| 67 | - | ||
| 68 | -void InitializeAicReproData(DataType *input, IndexType *index, DataType *expected) { | ||
| 69 | - for (int32_t row = 0; row < kInputRows; ++row) { | ||
| 70 | - for (int32_t column = 0; column < kColumns; ++column) { | ||
| 71 | - input[static_cast<int64_t>(row) * kColumns + column] = | ||
| 72 | - static_cast<DataType>((row % 97) * 0.25F + (column % 31) * 0.03125F); | ||
| 73 | - } | ||
| 74 | - } | ||
| 75 | - for (int32_t row = 0; row < kRows; ++row) { | ||
| 76 | - index[row] = static_cast<IndexType>((static_cast<int64_t>(row) * 97 + 13) % kInputRows); | ||
| 77 | - for (int32_t column = 0; column < kColumns; ++column) { | ||
| 78 | - const int64_t output_offset = static_cast<int64_t>(row) * kColumns + column; | ||
| 79 | - expected[output_offset] = input[index[row] * kColumns + column]; | ||
| 80 | - } | ||
| 81 | - } | ||
| 82 | -} | ||
| 83 | - | ||
| 84 | -using DataType = half; | ||
| 85 | -using IndexType = int64_t; | ||
| 86 | -constexpr std::array<int32_t, 4> kOutputShape = {IL_OUTPUT_S0, IL_OUTPUT_S1, IL_OUTPUT_S2, IL_OUTPUT_S3}; | ||
| 87 | -constexpr bool kInputBroadcast = IL_INPUT_BROADCAST; | ||
| 88 | -constexpr bool kIndexBroadcast = IL_INDEX_BROADCAST; | ||
| 89 | -constexpr bool kComplexBroadcast = IL_COMPLEX_BROADCAST; | ||
| 90 | -constexpr bool kComplexSimt = IL_COMPLEX_SIMT; | ||
| 91 | -constexpr bool kComplexInputBroadcast = IL_COMPLEX_INPUT_BROADCAST; | ||
| 92 | -constexpr bool kComplexIndexBroadcast = IL_COMPLEX_INDEX_BROADCAST; | ||
| 93 | -constexpr int32_t kBinaryElementKind = IL_BINARY_ELEMENT_KIND; | ||
| 94 | -constexpr bool kRetainBroadcast = IL_RETAIN_BROADCAST; | ||
| 95 | -constexpr bool kBroadcastPostReduce = IL_BROADCAST_POST_REDUCE; | ||
| 96 | -constexpr uint32_t kBroadcastAxesMask = IL_BROADCAST_AXES_MASK; | ||
| 97 | - | ||
| 98 | -constexpr std::array<int32_t, 4> MakeBroadcastSourceShape() { | ||
| 99 | - auto shape = kOutputShape; | ||
| 100 | - for (size_t dim = 0UL; dim < shape.size(); ++dim) { | ||
| 101 | - if ((kBroadcastAxesMask & (1U << dim)) != 0U) { | ||
| 102 | - shape[dim] = 1; | ||
| 103 | - } | ||
| 104 | - } | ||
| 105 | - return shape; | ||
| 106 | -} | ||
| 107 | - | ||
| 108 | -constexpr std::array<int32_t, 4> kBroadcastSourceShape = MakeBroadcastSourceShape(); | ||
| 109 | -constexpr std::array<int32_t, 4> kInputShape = kInputBroadcast && !kComplexSimt ? kBroadcastSourceShape : kOutputShape; | ||
| 110 | -constexpr std::array<int32_t, 4> kIndexShape = kIndexBroadcast ? kBroadcastSourceShape : kOutputShape; | ||
| 111 | -constexpr int32_t kInputElementCount = IL_HAS_INPUT_ELEMENT; | ||
| 112 | -constexpr int32_t kIndexElementCount = IL_HAS_INDEX_ELEMENT; | ||
| 113 | -constexpr bool kHasOutputRelu = IL_HAS_OUTPUT_RELU; | ||
| 114 | - | ||
| 115 | -template <size_t N> | ||
| 116 | -int32_t ElementCount(const std::array<int32_t, N> &shape) { | ||
| 117 | - int32_t count = 1; | ||
| 118 | - for (const int32_t dim : shape) { | ||
| 119 | - count *= dim; | ||
| 120 | - } | ||
| 121 | - return count; | ||
| 122 | -} | ||
| 123 | - | ||
| 124 | -int32_t ResultCount() { | ||
| 125 | - if constexpr (kBroadcastPostReduce) { | ||
| 126 | - return kOutputShape[0] * kOutputShape[1]; | ||
| 127 | - } | ||
| 128 | - return ElementCount(kOutputShape); | ||
| 129 | -} | ||
| 130 | - | ||
| 131 | -int32_t DenseOffset(const std::array<int32_t, 4> &coordinate, const std::array<int32_t, 4> &shape) { | ||
| 132 | - int32_t offset = 0; | ||
| 133 | - for (size_t dim = 0UL; dim < coordinate.size(); ++dim) { | ||
| 134 | - offset = offset * shape[dim] + coordinate[dim]; | ||
| 135 | - } | ||
| 136 | - return offset; | ||
| 137 | -} | ||
| 138 | - | ||
| 139 | -void InitializeData(DataType *x, IndexType *index, DataType *expected) { | ||
| 140 | - for (int32_t i = 0; i < ElementCount(kInputShape); ++i) { | ||
| 141 | - x[i] = static_cast<DataType>(static_cast<float>((i % 29) - 14) * 0.25F); | ||
| 142 | - } | ||
| 143 | - for (int32_t i = 0; i < ElementCount(kIndexShape); ++i) { | ||
| 144 | - const int32_t gathered_axis = (i * 3 + 1) % kInputShape[2]; | ||
| 145 | - index[i] = static_cast<IndexType>(kIndexElementCount == 0 || i % 2 == 0 ? gathered_axis : -gathered_axis); | ||
| 146 | - } | ||
| 147 | - for (int32_t i = 0; i < ElementCount(kOutputShape); ++i) { | ||
| 148 | - int32_t coordinate = i; | ||
| 149 | - const int32_t d = coordinate % kOutputShape[3]; | ||
| 150 | - coordinate /= kOutputShape[3]; | ||
| 151 | - const int32_t c = coordinate % kOutputShape[2]; | ||
| 152 | - coordinate /= kOutputShape[2]; | ||
| 153 | - const int32_t b = coordinate % kOutputShape[1]; | ||
| 154 | - coordinate /= kOutputShape[1]; | ||
| 155 | - const int32_t a = coordinate; | ||
| 156 | - std::array<int32_t, 4> index_coordinate = {a, b, c, d}; | ||
| 157 | - if (kIndexBroadcast) { | ||
| 158 | - for (size_t dim = 0UL; dim < index_coordinate.size(); ++dim) { | ||
| 159 | - if ((kBroadcastAxesMask & (1U << dim)) != 0U) { | ||
| 160 | - index_coordinate[dim] = 0; | ||
| 161 | - } | ||
| 162 | - } | ||
| 163 | - } | ||
| 164 | - const int32_t index_offset = DenseOffset(index_coordinate, kIndexShape); | ||
| 165 | - int64_t gathered_index = static_cast<int64_t>(index[index_offset]); | ||
| 166 | - if constexpr (kComplexIndexBroadcast) { | ||
| 167 | - if constexpr (kBinaryElementKind == 1) { | ||
| 168 | - gathered_index = 0; | ||
| 169 | - } else if constexpr (kBinaryElementKind == 3) { | ||
| 170 | - gathered_index = std::max(gathered_index, int64_t{0}); | ||
| 171 | - } | ||
| 172 | - } | ||
| 173 | - for (int32_t element = 0; element < kIndexElementCount; ++element) { | ||
| 174 | - gathered_index = std::abs(gathered_index); | ||
| 175 | - } | ||
| 176 | - const int32_t gathered_axis = static_cast<int32_t>(gathered_index); | ||
| 177 | - std::array<int32_t, 4> input_coordinate = {a, b, gathered_axis, d}; | ||
| 178 | - if (kInputBroadcast && !kComplexSimt) { | ||
| 179 | - for (size_t dim = 0UL; dim < input_coordinate.size(); ++dim) { | ||
| 180 | - if ((kBroadcastAxesMask & (1U << dim)) != 0U) { | ||
| 181 | - input_coordinate[dim] = 0; | ||
| 182 | - } | ||
| 183 | - } | ||
| 184 | - } | ||
| 185 | - const int32_t input_offset = DenseOffset(input_coordinate, kInputShape); | ||
| 186 | - float value = static_cast<float>(x[input_offset]); | ||
| 187 | - if constexpr (kComplexInputBroadcast) { | ||
| 188 | - if constexpr (kBinaryElementKind == 1) { | ||
| 189 | - value = 0.0F; | ||
| 190 | - } else if constexpr (kBinaryElementKind == 3) { | ||
| 191 | - value = std::max(value, 0.0F); | ||
| 192 | - } | ||
| 193 | - } | ||
| 194 | - if (kRetainBroadcast) { | ||
| 195 | - value += 1.5F; | ||
| 196 | - } else if (kComplexBroadcast && !kComplexSimt) { | ||
| 197 | - value = value * 2.0F + 1.5F; | ||
| 198 | - } else if (kComplexSimt) { | ||
| 199 | - value += 1.5F; | ||
| 200 | - } | ||
| 201 | - for (int32_t element = 0; element < kInputElementCount; ++element) { | ||
| 202 | - value = std::abs(value); | ||
| 203 | - } | ||
| 204 | - if (kHasOutputRelu) { | ||
| 205 | - value = std::max(value, 0.0F); | ||
| 206 | - } | ||
| 207 | - if constexpr (kBroadcastPostReduce) { | ||
| 208 | - const int32_t result_offset = a * kOutputShape[1] + b; | ||
| 209 | - expected[result_offset] = static_cast<DataType>(static_cast<float>(expected[result_offset]) + value); | ||
| 210 | - } else { | ||
| 211 | - expected[i] = static_cast<DataType>(value); | ||
| 212 | - } | ||
| 213 | - } | ||
| 214 | -} | ||
| 215 | - | ||
| 216 | -} // namespace | ||
| 217 | - | ||
| 218 | -TEST(E2EIndirectLoadBroadcast, GeneratedKernelMatchesReference) { | ||
| 219 | - | ||
| 220 | - constexpr int64_t input_count = static_cast<int64_t>(kInputRows) * kColumns; | ||
| 221 | - constexpr int64_t index_count = kRows; | ||
| 222 | - constexpr int64_t output_count = static_cast<int64_t>(kRows) * kColumns; | ||
| 223 | - indirect_load_test::KernelData<DataType, IndexType> buffers(input_count, index_count, output_count); | ||
| 224 | - ASSERT_TRUE(buffers.IsValid()); | ||
| 225 | - InitializeAicReproData(buffers.input.get(), buffers.index.get(), buffers.expected.data()); | ||
| 226 | - std::fill_n(buffers.output.get(), output_count, 0.0F); | ||
| 227 | - indirect_load_test::KernelTiling tiling; | ||
| 228 | - ASSERT_TRUE(tiling.IsValid()); | ||
| 229 | - | ||
| 230 | - AscendC::SetKernelMode(KernelMode::AIV_MODE); | ||
| 231 | - ICPU_RUN_KF(indirect_load_aic_repro, tiling.data.block_dim, reinterpret_cast<uint8_t *>(buffers.input.get()), | ||
| 232 | - reinterpret_cast<uint8_t *>(buffers.index.get()), reinterpret_cast<uint8_t *>(buffers.output.get()), | ||
| 233 | - tiling.workspace.get(), reinterpret_cast<uint8_t *>(&tiling.data)); | ||
| 234 | - for (int64_t i = 0; i < output_count; ++i) { | ||
| 235 | - EXPECT_FLOAT_EQ(buffers.output.get()[i], buffers.expected[static_cast<size_t>(i)]) << "offset=" << i; | ||
| 236 | - } | ||
| 237 | - | ||
| 238 | - const int32_t input_count = ElementCount(kInputShape); | ||
| 239 | - const int32_t index_count = ElementCount(kIndexShape); | ||
| 240 | - const int32_t output_count = ResultCount(); | ||
| 241 | - indirect_load_test::KernelData<DataType, IndexType> buffers(input_count, index_count, output_count); | ||
| 242 | - ASSERT_TRUE(buffers.IsValid()); | ||
| 243 | - InitializeData(buffers.input.get(), buffers.index.get(), buffers.expected.data()); | ||
| 244 | - std::fill_n(buffers.output.get(), output_count, static_cast<DataType>(0.0F)); | ||
| 245 | - indirect_load_test::KernelTiling tiling; | ||
| 246 | - ASSERT_TRUE(tiling.IsValid()); | ||
| 247 | - | ||
| 248 | - AscendC::SetKernelMode(KernelMode::AIV_MODE); | ||
| 249 | - ICPU_RUN_KF(indirect_load_broadcast_test, tiling.data.block_dim, reinterpret_cast<uint8_t *>(buffers.input.get()), | ||
| 250 | - reinterpret_cast<uint8_t *>(buffers.index.get()), reinterpret_cast<uint8_t *>(buffers.output.get()), | ||
| 251 | - tiling.workspace.get(), reinterpret_cast<uint8_t *>(&tiling.data)); | ||
| 252 | - for (int32_t i = 0; i < output_count; ++i) { | ||
| 253 | - EXPECT_NEAR(static_cast<float>(buffers.output.get()[i]), | ||
| 254 | - static_cast<float>(buffers.expected[static_cast<size_t>(i)]), 0.0625F) | ||
| 255 | - << "offset=" << i; | ||
| 256 | - } | ||
| 257 | - | ||
| 258 | -} | ||
Dautofuse/tests/v35/st/backend_e2e_v2/indirect_load_store_test/test_e2e_indirect_load_broadcast_where_kernel.cpp+0-80
| @@ -1,80 +0,0 @@ | |||
| 1 | -/** | ||
| 2 | - * Copyright (c) 2026 Huawei Technologies Co., Ltd. | ||
| 3 | - * This program is free software, you can redistribute it and/or modify it under the terms and conditions of | ||
| 4 | - * CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 5 | - * Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 6 | - * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 7 | - * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 8 | - * See LICENSE in the root of the software repository for the full text of the License. | ||
| 9 | - */ | ||
| 10 | - | ||
| 11 | - | ||
| 12 | - | ||
| 13 | - | ||
| 14 | - | ||
| 15 | - | ||
| 16 | - | ||
| 17 | - | ||
| 18 | -extern "C" __global__ __aicore__ void indirect_load_broadcast_index_where_simt_test(GM_ADDR input0, GM_ADDR input1, | ||
| 19 | - GM_ADDR input2, GM_ADDR output, | ||
| 20 | - GM_ADDR workspace, GM_ADDR tiling); | ||
| 21 | -extern "C" int64_t AutofuseTiling(AutofuseTilingData *, uint32_t *, uint32_t *, uint32_t, uint32_t); | ||
| 22 | - | ||
| 23 | -namespace { | ||
| 24 | -constexpr int32_t kRows = 6400; | ||
| 25 | -constexpr int32_t kColumns = 32; | ||
| 26 | -constexpr int32_t kTableRows = 315511; | ||
| 27 | - | ||
| 28 | -TEST(E2EIndirectLoadBroadcastWhere, GeneratedKernelMatchesReference) { | ||
| 29 | - const int64_t table_count = static_cast<int64_t>(kTableRows) * kColumns; | ||
| 30 | - const int64_t output_count = static_cast<int64_t>(kRows) * kColumns; | ||
| 31 | - auto *index0 = static_cast<int64_t *>(AscendC::GmAlloc(sizeof(int64_t) * kRows)); | ||
| 32 | - auto *table = static_cast<float *>(AscendC::GmAlloc(sizeof(float) * table_count)); | ||
| 33 | - auto *index2 = static_cast<int64_t *>(AscendC::GmAlloc(sizeof(int64_t) * kRows)); | ||
| 34 | - auto *output = static_cast<float *>(AscendC::GmAlloc(sizeof(float) * output_count)); | ||
| 35 | - ASSERT_NE(index0, nullptr); | ||
| 36 | - ASSERT_NE(table, nullptr); | ||
| 37 | - ASSERT_NE(index2, nullptr); | ||
| 38 | - ASSERT_NE(output, nullptr); | ||
| 39 | - | ||
| 40 | - for (int32_t row = 0; row < kTableRows; ++row) { | ||
| 41 | - for (int32_t column = 0; column < kColumns; ++column) { | ||
| 42 | - table[static_cast<int64_t>(row) * kColumns + column] = | ||
| 43 | - static_cast<float>((row % 97) * 0.25F + (column % 31) * 0.03125F); | ||
| 44 | - } | ||
| 45 | - } | ||
| 46 | - for (int32_t row = 0; row < kRows; ++row) { | ||
| 47 | - index0[row] = row % 2 == 0 ? -1 : static_cast<int64_t>((row * 17 + 3) % kTableRows); | ||
| 48 | - index2[row] = static_cast<int64_t>((row * 29 + 7) % kTableRows); | ||
| 49 | - } | ||
| 50 | - | ||
| 51 | - AutofuseTilingData tiling_data{}; | ||
| 52 | - uint32_t workspace_size = 0; | ||
| 53 | - uint32_t block_dim = 48; | ||
| 54 | - ASSERT_EQ(AutofuseTiling(&tiling_data, &workspace_size, &block_dim, 48U, 192U * 1024U), 0); | ||
| 55 | - void *workspace = workspace_size == 0U ? nullptr : AscendC::GmAlloc(workspace_size); | ||
| 56 | - ASSERT_TRUE(workspace_size == 0U || workspace != nullptr); | ||
| 57 | - AscendC::SetKernelMode(KernelMode::AIV_MODE); | ||
| 58 | - ICPU_RUN_KF(indirect_load_broadcast_index_where_simt_test, block_dim, reinterpret_cast<uint8_t *>(index0), | ||
| 59 | - reinterpret_cast<uint8_t *>(table), reinterpret_cast<uint8_t *>(index2), | ||
| 60 | - reinterpret_cast<uint8_t *>(output), reinterpret_cast<uint8_t *>(workspace), | ||
| 61 | - reinterpret_cast<uint8_t *>(&tiling_data)); | ||
| 62 | - | ||
| 63 | - for (int32_t row = 0; row < kRows; ++row) { | ||
| 64 | - const int64_t selected = index0[row] == -1 ? index2[row] : index0[row]; | ||
| 65 | - for (int32_t column = 0; column < kColumns; ++column) { | ||
| 66 | - const int64_t offset = static_cast<int64_t>(row) * kColumns + column; | ||
| 67 | - const float expected = table[selected * kColumns + column]; | ||
| 68 | - EXPECT_FLOAT_EQ(output[offset], expected) << "row=" << row << ", column=" << column; | ||
| 69 | - } | ||
| 70 | - } | ||
| 71 | - | ||
| 72 | - if (workspace != nullptr) { | ||
| 73 | - AscendC::GmFree(workspace); | ||
| 74 | - } | ||
| 75 | - AscendC::GmFree(index0); | ||
| 76 | - AscendC::GmFree(table); | ||
| 77 | - AscendC::GmFree(index2); | ||
| 78 | - AscendC::GmFree(output); | ||
| 79 | -} | ||
| 80 | -} // namespace | ||
Dautofuse/tests/v35/st/backend_e2e_v2/indirect_load_store_test/test_e2e_indirect_load_embedding_kernel.cpp+0-58
| @@ -1,58 +0,0 @@ | |||
| 1 | -/** | ||
| 2 | - * Copyright (c) 2026 Huawei Technologies Co., Ltd. | ||
| 3 | - * This program is free software, you can redistribute it and/or modify it under the terms and conditions of | ||
| 4 | - * CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 5 | - * Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 6 | - * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 7 | - * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 8 | - * See LICENSE in the root of the software repository for the full text of the License. | ||
| 9 | - */ | ||
| 10 | - | ||
| 11 | - | ||
| 12 | - | ||
| 13 | - | ||
| 14 | - | ||
| 15 | - | ||
| 16 | -extern "C" __global__ __aicore__ void indirect_load_embedding_test(GM_ADDR input, GM_ADDR index, GM_ADDR output, | ||
| 17 | - GM_ADDR workspace, GM_ADDR tiling); | ||
| 18 | - | ||
| 19 | -namespace { | ||
| 20 | -constexpr int32_t kInputRows = 100; | ||
| 21 | -constexpr int32_t kEmbeddingSize = 16; | ||
| 22 | -constexpr int32_t kIndexRows = 92; | ||
| 23 | - | ||
| 24 | -void InitializeData(float *input, int64_t *index, float *expected) { | ||
| 25 | - for (int32_t row = 0; row < kInputRows; ++row) { | ||
| 26 | - for (int32_t col = 0; col < kEmbeddingSize; ++col) { | ||
| 27 | - input[row * kEmbeddingSize + col] = static_cast<float>(row * 0.25F + col * 0.03125F); | ||
| 28 | - } | ||
| 29 | - } | ||
| 30 | - for (int32_t row = 0; row < kIndexRows; ++row) { | ||
| 31 | - index[row] = static_cast<int64_t>((row * 7 + 3) % kInputRows); | ||
| 32 | - for (int32_t col = 0; col < kEmbeddingSize; ++col) { | ||
| 33 | - expected[row * kEmbeddingSize + col] = input[index[row] * kEmbeddingSize + col]; | ||
| 34 | - } | ||
| 35 | - } | ||
| 36 | -} | ||
| 37 | -} // namespace | ||
| 38 | - | ||
| 39 | -TEST(E2EIndirectLoadEmbedding, GeneratedKernelMatchesReference) { | ||
| 40 | - constexpr int64_t input_count = static_cast<int64_t>(kInputRows) * kEmbeddingSize; | ||
| 41 | - constexpr int64_t index_count = kIndexRows; | ||
| 42 | - constexpr int64_t output_count = static_cast<int64_t>(kIndexRows) * kEmbeddingSize; | ||
| 43 | - indirect_load_test::KernelData<float, int64_t> buffers(input_count, index_count, output_count); | ||
| 44 | - ASSERT_TRUE(buffers.IsValid()); | ||
| 45 | - InitializeData(buffers.input.get(), buffers.index.get(), buffers.expected.data()); | ||
| 46 | - std::fill_n(buffers.output.get(), output_count, 0.0F); | ||
| 47 | - | ||
| 48 | - indirect_load_test::KernelTiling tiling; | ||
| 49 | - ASSERT_TRUE(tiling.IsValid()); | ||
| 50 | - AscendC::SetKernelMode(KernelMode::AIV_MODE); | ||
| 51 | - ICPU_RUN_KF(indirect_load_embedding_test, tiling.data.block_dim, reinterpret_cast<uint8_t *>(buffers.input.get()), | ||
| 52 | - reinterpret_cast<uint8_t *>(buffers.index.get()), reinterpret_cast<uint8_t *>(buffers.output.get()), | ||
| 53 | - tiling.workspace.get(), reinterpret_cast<uint8_t *>(&tiling.data)); | ||
| 54 | - | ||
| 55 | - for (int64_t i = 0; i < output_count; ++i) { | ||
| 56 | - EXPECT_FLOAT_EQ(buffers.output.get()[i], buffers.expected[static_cast<size_t>(i)]) << "offset=" << i; | ||
| 57 | - } | ||
| 58 | -} | ||
Mautofuse/tests/v35/st/backend_e2e_v2/indirect_load_store_test/test_e2e_indirect_load_store_kernel.cpp+891-0
| @@ -8,6 +8,80 @@ | |||
| 8 | * See LICENSE in the root of the software repository for the full text of the License. | 8 | * See LICENSE in the root of the software repository for the full text of the License. |
| 9 | */ | 9 | */ |
| 10 | 10 | ||
| 11 | + | ||
| 12 | + | ||
| 13 | + | ||
| 14 | + | ||
| 15 | + | ||
| 16 | + | ||
| 17 | + | ||
| 18 | + | ||
| 19 | + | ||
| 20 | + | ||
| 21 | + | ||
| 22 | + | ||
| 23 | + | ||
| 24 | +extern "C" int64_t AutofuseTiling(AutofuseTilingData *, uint32_t *, uint32_t *, uint32_t, uint32_t); | ||
| 25 | + | ||
| 26 | + | ||
| 27 | +namespace indirect_load_test { | ||
| 28 | +inline void GmFree(void *ptr) { | ||
| 29 | + AscendC::GmFree(ptr); | ||
| 30 | +} | ||
| 31 | + | ||
| 32 | +template <typename DataType, typename IndexType> | ||
| 33 | +struct KernelData { | ||
| 34 | + KernelData(int64_t input_count, int64_t index_count, int64_t output_count) | ||
| 35 | + : input(reinterpret_cast<DataType *>(AscendC::GmAlloc(input_count * sizeof(DataType))), GmFree), | ||
| 36 | + index(reinterpret_cast<IndexType *>(AscendC::GmAlloc(index_count * sizeof(IndexType))), GmFree), | ||
| 37 | + output(reinterpret_cast<DataType *>(AscendC::GmAlloc(output_count * sizeof(DataType))), GmFree), | ||
| 38 | + expected(static_cast<size_t>(output_count)) {} | ||
| 39 | + | ||
| 40 | + [[nodiscard]] bool IsValid() const { | ||
| 41 | + return input != nullptr && index != nullptr && output != nullptr; | ||
| 42 | + } | ||
| 43 | + | ||
| 44 | + std::unique_ptr<DataType, decltype(&GmFree)> input; | ||
| 45 | + std::unique_ptr<IndexType, decltype(&GmFree)> index; | ||
| 46 | + std::unique_ptr<DataType, decltype(&GmFree)> output; | ||
| 47 | + std::vector<DataType> expected; | ||
| 48 | +}; | ||
| 49 | + | ||
| 50 | + | ||
| 51 | +struct KernelTiling { | ||
| 52 | + KernelTiling() : workspace(nullptr, GmFree) { | ||
| 53 | + EXPECT_EQ(AutofuseTiling(&data, &workspace_size, &block_dim, 48U, 192U * 1024U), 0); | ||
| 54 | + EXPECT_GT(data.block_dim, 0U); | ||
| 55 | + if (workspace_size != 0U) { | ||
| 56 | + workspace.reset(reinterpret_cast<uint8_t *>(AscendC::GmAlloc(workspace_size))); | ||
| 57 | + } | ||
| 58 | + } | ||
| 59 | + | ||
| 60 | + [[nodiscard]] bool IsValid() const { | ||
| 61 | + return workspace_size == 0U || workspace != nullptr; | ||
| 62 | + } | ||
| 63 | + | ||
| 64 | + AutofuseTilingData data{}; | ||
| 65 | + uint32_t workspace_size = 0U; | ||
| 66 | + uint32_t block_dim = 48U; | ||
| 67 | + std::unique_ptr<uint8_t, decltype(&GmFree)> workspace; | ||
| 68 | +}; | ||
| 69 | + | ||
| 70 | +} // namespace indirect_load_test | ||
| 71 | + | ||
| 72 | + | ||
| 73 | + | ||
| 74 | + | ||
| 75 | +/** | ||
| 76 | + * Copyright (c) 2026 Huawei Technologies Co., Ltd. | ||
| 77 | + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of | ||
| 78 | + * CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 79 | + * Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 80 | + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 81 | + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 82 | + * See LICENSE in the root of the software repository for the full text of the License. | ||
| 83 | + */ | ||
| 84 | + | ||
| 11 | 85 | ||
| 12 | 86 | ||
| 13 | 87 | ||
| @@ -496,3 +570,820 @@ TEST(E2EIndirectLoadStore, GeneratedKernelMatchesReference) { | |||
| 496 | } | 570 | } |
| 497 | 571 | ||
| 498 | } | 572 | } |
| 573 | + | ||
| 574 | + | ||
| 575 | + | ||
| 576 | + | ||
| 577 | +/** | ||
| 578 | + * Copyright (c) 2026 Huawei Technologies Co., Ltd. | ||
| 579 | + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of | ||
| 580 | + * CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 581 | + * Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 582 | + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 583 | + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 584 | + * See LICENSE in the root of the software repository for the full text of the License. | ||
| 585 | + */ | ||
| 586 | + | ||
| 587 | + | ||
| 588 | + | ||
| 589 | + | ||
| 590 | + | ||
| 591 | + | ||
| 592 | + | ||
| 593 | + | ||
| 594 | + | ||
| 595 | + | ||
| 596 | + | ||
| 597 | + | ||
| 598 | + | ||
| 599 | + | ||
| 600 | + | ||
| 601 | + | ||
| 602 | + | ||
| 603 | + | ||
| 604 | + | ||
| 605 | + | ||
| 606 | + | ||
| 607 | + | ||
| 608 | + | ||
| 609 | + | ||
| 610 | + | ||
| 611 | + | ||
| 612 | + | ||
| 613 | + | ||
| 614 | + | ||
| 615 | + | ||
| 616 | + | ||
| 617 | + | ||
| 618 | + | ||
| 619 | + | ||
| 620 | + | ||
| 621 | + | ||
| 622 | + | ||
| 623 | + | ||
| 624 | + | ||
| 625 | + | ||
| 626 | + | ||
| 627 | + | ||
| 628 | + | ||
| 629 | + | ||
| 630 | +extern "C" __global__ __aicore__ void indirect_load_aic_repro(GM_ADDR input, GM_ADDR index, GM_ADDR output, | ||
| 631 | + GM_ADDR workspace, GM_ADDR tiling); | ||
| 632 | + | ||
| 633 | +extern "C" __global__ __aicore__ void indirect_load_broadcast_test(GM_ADDR x, GM_ADDR index, GM_ADDR y, | ||
| 634 | + GM_ADDR workspace, GM_ADDR tiling); | ||
| 635 | + | ||
| 636 | + | ||
| 637 | +namespace { | ||
| 638 | + | ||
| 639 | +using DataType = float; | ||
| 640 | +using IndexType = int64_t; | ||
| 641 | +constexpr int32_t kInputRows = 100000; | ||
| 642 | +constexpr int32_t kRows = 1024; | ||
| 643 | +constexpr int32_t kColumns = 1024; | ||
| 644 | + | ||
| 645 | +void InitializeAicReproData(DataType *input, IndexType *index, DataType *expected) { | ||
| 646 | + for (int32_t row = 0; row < kInputRows; ++row) { | ||
| 647 | + for (int32_t column = 0; column < kColumns; ++column) { | ||
| 648 | + input[static_cast<int64_t>(row) * kColumns + column] = | ||
| 649 | + static_cast<DataType>((row % 97) * 0.25F + (column % 31) * 0.03125F); | ||
| 650 | + } | ||
| 651 | + } | ||
| 652 | + for (int32_t row = 0; row < kRows; ++row) { | ||
| 653 | + index[row] = static_cast<IndexType>((static_cast<int64_t>(row) * 97 + 13) % kInputRows); | ||
| 654 | + for (int32_t column = 0; column < kColumns; ++column) { | ||
| 655 | + const int64_t output_offset = static_cast<int64_t>(row) * kColumns + column; | ||
| 656 | + expected[output_offset] = input[index[row] * kColumns + column]; | ||
| 657 | + } | ||
| 658 | + } | ||
| 659 | +} | ||
| 660 | + | ||
| 661 | +using DataType = half; | ||
| 662 | +using IndexType = int64_t; | ||
| 663 | +constexpr std::array<int32_t, 4> kOutputShape = {IL_OUTPUT_S0, IL_OUTPUT_S1, IL_OUTPUT_S2, IL_OUTPUT_S3}; | ||
| 664 | +constexpr bool kInputBroadcast = IL_INPUT_BROADCAST; | ||
| 665 | +constexpr bool kIndexBroadcast = IL_INDEX_BROADCAST; | ||
| 666 | +constexpr bool kComplexBroadcast = IL_COMPLEX_BROADCAST; | ||
| 667 | +constexpr bool kComplexSimt = IL_COMPLEX_SIMT; | ||
| 668 | +constexpr bool kComplexInputBroadcast = IL_COMPLEX_INPUT_BROADCAST; | ||
| 669 | +constexpr bool kComplexIndexBroadcast = IL_COMPLEX_INDEX_BROADCAST; | ||
| 670 | +constexpr int32_t kBinaryElementKind = IL_BINARY_ELEMENT_KIND; | ||
| 671 | +constexpr bool kRetainBroadcast = IL_RETAIN_BROADCAST; | ||
| 672 | +constexpr bool kBroadcastPostReduce = IL_BROADCAST_POST_REDUCE; | ||
| 673 | +constexpr bool kInputAbsBeforeBroadcast = IL_INPUT_ABS_BEFORE_BROADCAST; | ||
| 674 | +constexpr uint32_t kBroadcastAxesMask = IL_BROADCAST_AXES_MASK; | ||
| 675 | + | ||
| 676 | +constexpr std::array<int32_t, 4> MakeBroadcastSourceShape() { | ||
| 677 | + auto shape = kOutputShape; | ||
| 678 | + for (size_t dim = 0UL; dim < shape.size(); ++dim) { | ||
| 679 | + if ((kBroadcastAxesMask & (1U << dim)) != 0U) { | ||
| 680 | + shape[dim] = 1; | ||
| 681 | + } | ||
| 682 | + } | ||
| 683 | + return shape; | ||
| 684 | +} | ||
| 685 | + | ||
| 686 | +constexpr std::array<int32_t, 4> kBroadcastSourceShape = MakeBroadcastSourceShape(); | ||
| 687 | +constexpr std::array<int32_t, 4> kInputShape = kInputBroadcast && !kComplexSimt ? kBroadcastSourceShape : kOutputShape; | ||
| 688 | +constexpr std::array<int32_t, 4> kIndexShape = kIndexBroadcast ? kBroadcastSourceShape : kOutputShape; | ||
| 689 | +constexpr int32_t kInputElementCount = IL_HAS_INPUT_ELEMENT; | ||
| 690 | +constexpr int32_t kIndexElementCount = IL_HAS_INDEX_ELEMENT; | ||
| 691 | +constexpr bool kHasOutputRelu = IL_HAS_OUTPUT_RELU; | ||
| 692 | + | ||
| 693 | +template <size_t N> | ||
| 694 | +int32_t ElementCount(const std::array<int32_t, N> &shape) { | ||
| 695 | + int32_t count = 1; | ||
| 696 | + for (const int32_t dim : shape) { | ||
| 697 | + count *= dim; | ||
| 698 | + } | ||
| 699 | + return count; | ||
| 700 | +} | ||
| 701 | + | ||
| 702 | +int32_t ResultCount() { | ||
| 703 | + if constexpr (kBroadcastPostReduce) { | ||
| 704 | + return kOutputShape[0] * kOutputShape[1]; | ||
| 705 | + } | ||
| 706 | + return ElementCount(kOutputShape); | ||
| 707 | +} | ||
| 708 | + | ||
| 709 | +int32_t DenseOffset(const std::array<int32_t, 4> &coordinate, const std::array<int32_t, 4> &shape) { | ||
| 710 | + int32_t offset = 0; | ||
| 711 | + for (size_t dim = 0UL; dim < coordinate.size(); ++dim) { | ||
| 712 | + offset = offset * shape[dim] + coordinate[dim]; | ||
| 713 | + } | ||
| 714 | + return offset; | ||
| 715 | +} | ||
| 716 | + | ||
| 717 | +void InitializeData(DataType *x, IndexType *index, DataType *expected) { | ||
| 718 | + for (int32_t i = 0; i < ElementCount(kInputShape); ++i) { | ||
| 719 | + x[i] = static_cast<DataType>(static_cast<float>((i % 29) - 14) * 0.25F); | ||
| 720 | + } | ||
| 721 | + for (int32_t i = 0; i < ElementCount(kIndexShape); ++i) { | ||
| 722 | + const int32_t gathered_axis = (i * 3 + 1) % kInputShape[2]; | ||
| 723 | + index[i] = static_cast<IndexType>(kIndexElementCount == 0 || i % 2 == 0 ? gathered_axis : -gathered_axis); | ||
| 724 | + } | ||
| 725 | + for (int32_t i = 0; i < ElementCount(kOutputShape); ++i) { | ||
| 726 | + int32_t coordinate = i; | ||
| 727 | + const int32_t d = coordinate % kOutputShape[3]; | ||
| 728 | + coordinate /= kOutputShape[3]; | ||
| 729 | + const int32_t c = coordinate % kOutputShape[2]; | ||
| 730 | + coordinate /= kOutputShape[2]; | ||
| 731 | + const int32_t b = coordinate % kOutputShape[1]; | ||
| 732 | + coordinate /= kOutputShape[1]; | ||
| 733 | + const int32_t a = coordinate; | ||
| 734 | + std::array<int32_t, 4> index_coordinate = {a, b, c, d}; | ||
| 735 | + if (kIndexBroadcast) { | ||
| 736 | + for (size_t dim = 0UL; dim < index_coordinate.size(); ++dim) { | ||
| 737 | + if ((kBroadcastAxesMask & (1U << dim)) != 0U) { | ||
| 738 | + index_coordinate[dim] = 0; | ||
| 739 | + } | ||
| 740 | + } | ||
| 741 | + } | ||
| 742 | + const int32_t index_offset = DenseOffset(index_coordinate, kIndexShape); | ||
| 743 | + int64_t gathered_index = static_cast<int64_t>(index[index_offset]); | ||
| 744 | + if constexpr (kComplexIndexBroadcast) { | ||
| 745 | + if constexpr (kBinaryElementKind == 1) { | ||
| 746 | + gathered_index = 0; | ||
| 747 | + } else if constexpr (kBinaryElementKind == 3) { | ||
| 748 | + gathered_index = std::max(gathered_index, int64_t{0}); | ||
| 749 | + } | ||
| 750 | + } | ||
| 751 | + for (int32_t element = 0; element < kIndexElementCount; ++element) { | ||
| 752 | + gathered_index = std::abs(gathered_index); | ||
| 753 | + } | ||
| 754 | + const int32_t gathered_axis = static_cast<int32_t>(gathered_index); | ||
| 755 | + std::array<int32_t, 4> input_coordinate = {a, b, gathered_axis, d}; | ||
| 756 | + if (kInputBroadcast && !kComplexSimt) { | ||
| 757 | + for (size_t dim = 0UL; dim < input_coordinate.size(); ++dim) { | ||
| 758 | + if ((kBroadcastAxesMask & (1U << dim)) != 0U) { | ||
| 759 | + input_coordinate[dim] = 0; | ||
| 760 | + } | ||
| 761 | + } | ||
| 762 | + } | ||
| 763 | + const int32_t input_offset = DenseOffset(input_coordinate, kInputShape); | ||
| 764 | + float value = static_cast<float>(x[input_offset]); | ||
| 765 | + if constexpr (kInputAbsBeforeBroadcast) { | ||
| 766 | + value = std::abs(value); | ||
| 767 | + } | ||
| 768 | + if constexpr (kComplexInputBroadcast) { | ||
| 769 | + if constexpr (kBinaryElementKind == 1) { | ||
| 770 | + value = 0.0F; | ||
| 771 | + } else if constexpr (kBinaryElementKind == 3) { | ||
| 772 | + value = std::max(value, 0.0F); | ||
| 773 | + } | ||
| 774 | + } | ||
| 775 | + if (kRetainBroadcast) { | ||
| 776 | + value += 1.5F; | ||
| 777 | + } else if (kComplexBroadcast && !kComplexSimt) { | ||
| 778 | + value = value * 2.0F + 1.5F; | ||
| 779 | + } else if (kComplexSimt) { | ||
| 780 | + value += 1.5F; | ||
| 781 | + } | ||
| 782 | + for (int32_t element = 0; element < kInputElementCount; ++element) { | ||
| 783 | + value = std::abs(value); | ||
| 784 | + } | ||
| 785 | + if (kHasOutputRelu) { | ||
| 786 | + value = std::max(value, 0.0F); | ||
| 787 | + } | ||
| 788 | + if constexpr (kBroadcastPostReduce) { | ||
| 789 | + const int32_t result_offset = a * kOutputShape[1] + b; | ||
| 790 | + expected[result_offset] = static_cast<DataType>(static_cast<float>(expected[result_offset]) + value); | ||
| 791 | + } else { | ||
| 792 | + expected[i] = static_cast<DataType>(value); | ||
| 793 | + } | ||
| 794 | + } | ||
| 795 | +} | ||
| 796 | + | ||
| 797 | +} // namespace | ||
| 798 | + | ||
| 799 | +TEST(E2EIndirectLoadBroadcast, GeneratedKernelMatchesReference) { | ||
| 800 | + | ||
| 801 | + constexpr int64_t input_count = static_cast<int64_t>(kInputRows) * kColumns; | ||
| 802 | + constexpr int64_t index_count = kRows; | ||
| 803 | + constexpr int64_t output_count = static_cast<int64_t>(kRows) * kColumns; | ||
| 804 | + indirect_load_test::KernelData<DataType, IndexType> buffers(input_count, index_count, output_count); | ||
| 805 | + ASSERT_TRUE(buffers.IsValid()); | ||
| 806 | + InitializeAicReproData(buffers.input.get(), buffers.index.get(), buffers.expected.data()); | ||
| 807 | + std::fill_n(buffers.output.get(), output_count, 0.0F); | ||
| 808 | + indirect_load_test::KernelTiling tiling; | ||
| 809 | + ASSERT_TRUE(tiling.IsValid()); | ||
| 810 | + | ||
| 811 | + AscendC::SetKernelMode(KernelMode::AIV_MODE); | ||
| 812 | + ICPU_RUN_KF(indirect_load_aic_repro, tiling.data.block_dim, reinterpret_cast<uint8_t *>(buffers.input.get()), | ||
| 813 | + reinterpret_cast<uint8_t *>(buffers.index.get()), reinterpret_cast<uint8_t *>(buffers.output.get()), | ||
| 814 | + tiling.workspace.get(), reinterpret_cast<uint8_t *>(&tiling.data)); | ||
| 815 | + for (int64_t i = 0; i < output_count; ++i) { | ||
| 816 | + EXPECT_FLOAT_EQ(buffers.output.get()[i], buffers.expected[static_cast<size_t>(i)]) << "offset=" << i; | ||
| 817 | + } | ||
| 818 | + | ||
| 819 | + const int32_t input_count = ElementCount(kInputShape); | ||
| 820 | + const int32_t index_count = ElementCount(kIndexShape); | ||
| 821 | + const int32_t output_count = ResultCount(); | ||
| 822 | + indirect_load_test::KernelData<DataType, IndexType> buffers(input_count, index_count, output_count); | ||
| 823 | + ASSERT_TRUE(buffers.IsValid()); | ||
| 824 | + InitializeData(buffers.input.get(), buffers.index.get(), buffers.expected.data()); | ||
| 825 | + std::fill_n(buffers.output.get(), output_count, static_cast<DataType>(0.0F)); | ||
| 826 | + indirect_load_test::KernelTiling tiling; | ||
| 827 | + ASSERT_TRUE(tiling.IsValid()); | ||
| 828 | + | ||
| 829 | + AscendC::SetKernelMode(KernelMode::AIV_MODE); | ||
| 830 | + ICPU_RUN_KF(indirect_load_broadcast_test, tiling.data.block_dim, reinterpret_cast<uint8_t *>(buffers.input.get()), | ||
| 831 | + reinterpret_cast<uint8_t *>(buffers.index.get()), reinterpret_cast<uint8_t *>(buffers.output.get()), | ||
| 832 | + tiling.workspace.get(), reinterpret_cast<uint8_t *>(&tiling.data)); | ||
| 833 | + for (int32_t i = 0; i < output_count; ++i) { | ||
| 834 | + EXPECT_NEAR(static_cast<float>(buffers.output.get()[i]), | ||
| 835 | + static_cast<float>(buffers.expected[static_cast<size_t>(i)]), 0.0625F) | ||
| 836 | + << "offset=" << i; | ||
| 837 | + } | ||
| 838 | + | ||
| 839 | +} | ||
| 840 | + | ||
| 841 | + | ||
| 842 | + | ||
| 843 | + | ||
| 844 | +/** | ||
| 845 | + * Copyright (c) 2026 Huawei Technologies Co., Ltd. | ||
| 846 | + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of | ||
| 847 | + * CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 848 | + * Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 849 | + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 850 | + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 851 | + * See LICENSE in the root of the software repository for the full text of the License. | ||
| 852 | + */ | ||
| 853 | + | ||
| 854 | + | ||
| 855 | + | ||
| 856 | + | ||
| 857 | + | ||
| 858 | + | ||
| 859 | + | ||
| 860 | + | ||
| 861 | +extern "C" int64_t AutofuseTiling(AutofuseTilingData *, uint32_t *, uint32_t *, uint32_t, uint32_t); | ||
| 862 | + | ||
| 863 | + | ||
| 864 | + | ||
| 865 | +extern "C" __global__ __aicore__ void indirect_load_embedding_reduce_simt_test(GM_ADDR input0, GM_ADDR input1, | ||
| 866 | + GM_ADDR input2, GM_ADDR output, | ||
| 867 | + GM_ADDR workspace, GM_ADDR tiling); | ||
| 868 | + | ||
| 869 | +extern "C" __global__ __aicore__ void indirect_load_broadcast_index_where_simt_test(GM_ADDR input0, GM_ADDR input1, | ||
| 870 | + GM_ADDR input2, GM_ADDR output, | ||
| 871 | + GM_ADDR workspace, GM_ADDR tiling); | ||
| 872 | + | ||
| 873 | + | ||
| 874 | +extern "C" __global__ __aicore__ void indirect_load_add_il_reduce_test(GM_ADDR input0, GM_ADDR input1, GM_ADDR input2, | ||
| 875 | + GM_ADDR output, GM_ADDR workspace, | ||
| 876 | + GM_ADDR tiling); | ||
| 877 | + | ||
| 878 | + | ||
| 879 | +namespace { | ||
| 880 | + | ||
| 881 | + | ||
| 882 | +constexpr int32_t kEmbRows = 2; | ||
| 883 | +constexpr int32_t kEmbColumns = 2; | ||
| 884 | +constexpr int32_t kEmbReduceSize = 2; | ||
| 885 | +constexpr int32_t kEmbTableRows = 4; | ||
| 886 | + | ||
| 887 | +TEST(E2EIndirectLoadEmbeddingReduce, GeneratedKernelMatchesReference) { | ||
| 888 | + const int64_t index_count = static_cast<int64_t>(kEmbRows) * kEmbReduceSize; | ||
| 889 | + const int64_t table_count = static_cast<int64_t>(kEmbTableRows) * kEmbColumns; | ||
| 890 | + const int64_t output_count = static_cast<int64_t>(kEmbRows) * kEmbColumns; | ||
| 891 | + auto *index0 = static_cast<int64_t *>(AscendC::GmAlloc(sizeof(int64_t) * index_count)); | ||
| 892 | + auto *table = static_cast<float *>(AscendC::GmAlloc(sizeof(float) * table_count)); | ||
| 893 | + auto *unused_input = static_cast<int64_t *>(AscendC::GmAlloc(sizeof(int64_t) * index_count)); | ||
| 894 | + auto *output = static_cast<float *>(AscendC::GmAlloc(sizeof(float) * output_count)); | ||
| 895 | + ASSERT_NE(index0, nullptr); | ||
| 896 | + ASSERT_NE(table, nullptr); | ||
| 897 | + ASSERT_NE(unused_input, nullptr); | ||
| 898 | + ASSERT_NE(output, nullptr); | ||
| 899 | + | ||
| 900 | + for (int32_t row = 0; row < kEmbTableRows; ++row) { | ||
| 901 | + for (int32_t col = 0; col < kEmbColumns; ++col) { | ||
| 902 | + table[static_cast<int64_t>(row) * kEmbColumns + col] = static_cast<float>(row * kEmbColumns + col); | ||
| 903 | + } | ||
| 904 | + } | ||
| 905 | + for (int32_t p0 = 0; p0 < kEmbRows; ++p0) { | ||
| 906 | + for (int32_t p2 = 0; p2 < kEmbReduceSize; ++p2) { | ||
| 907 | + const int32_t pos = p0 * kEmbReduceSize + p2; | ||
| 908 | + index0[pos] = static_cast<int64_t>((p0 * kEmbReduceSize + p2) % kEmbTableRows); | ||
| 909 | + unused_input[pos] = 0; | ||
| 910 | + } | ||
| 911 | + } | ||
| 912 | + | ||
| 913 | + AutofuseTilingData tiling_data{}; | ||
| 914 | + uint32_t workspace_size = 0; | ||
| 915 | + uint32_t block_dim = 48; | ||
| 916 | + ASSERT_EQ(AutofuseTiling(&tiling_data, &workspace_size, &block_dim, 48U, 192U * 1024U), 0); | ||
| 917 | + void *workspace = workspace_size == 0U ? nullptr : AscendC::GmAlloc(workspace_size); | ||
| 918 | + ASSERT_TRUE(workspace_size == 0U || workspace != nullptr); | ||
| 919 | + AscendC::SetKernelMode(KernelMode::AIV_MODE); | ||
| 920 | + ICPU_RUN_KF(indirect_load_embedding_reduce_simt_test, block_dim, reinterpret_cast<uint8_t *>(index0), | ||
| 921 | + reinterpret_cast<uint8_t *>(table), reinterpret_cast<uint8_t *>(unused_input), | ||
| 922 | + reinterpret_cast<uint8_t *>(output), reinterpret_cast<uint8_t *>(workspace), | ||
| 923 | + reinterpret_cast<uint8_t *>(&tiling_data)); | ||
| 924 | + | ||
| 925 | + for (int32_t p0 = 0; p0 < kEmbRows; ++p0) { | ||
| 926 | + for (int32_t p1 = 0; p1 < kEmbColumns; ++p1) { | ||
| 927 | + float expected = 0.0F; | ||
| 928 | + for (int32_t p2 = 0; p2 < kEmbReduceSize; ++p2) { | ||
| 929 | + const int64_t idx = index0[p0 * kEmbReduceSize + p2]; | ||
| 930 | + expected += table[idx * kEmbColumns + p1] * (idx >= 0 ? 1.0F : 0.0F); | ||
| 931 | + } | ||
| 932 | + const int64_t offset = static_cast<int64_t>(p0) * kEmbColumns + p1; | ||
| 933 | + EXPECT_NEAR(output[offset], expected, 0.0625F) << "p0=" << p0 << ", p1=" << p1; | ||
| 934 | + } | ||
| 935 | + } | ||
| 936 | + | ||
| 937 | + if (workspace != nullptr) { | ||
| 938 | + AscendC::GmFree(workspace); | ||
| 939 | + } | ||
| 940 | + AscendC::GmFree(index0); | ||
| 941 | + AscendC::GmFree(table); | ||
| 942 | + AscendC::GmFree(unused_input); | ||
| 943 | + AscendC::GmFree(output); | ||
| 944 | +} | ||
| 945 | + | ||
| 946 | +constexpr int32_t kRows = 6400; | ||
| 947 | +constexpr int32_t kColumns = 32; | ||
| 948 | +constexpr int32_t kTableRows = 315511; | ||
| 949 | + | ||
| 950 | +TEST(E2EIndirectLoadBroadcastWhere, GeneratedKernelMatchesReference) { | ||
| 951 | + const int64_t table_count = static_cast<int64_t>(kTableRows) * kColumns; | ||
| 952 | + const int64_t output_count = static_cast<int64_t>(kRows) * kColumns; | ||
| 953 | + auto *index0 = static_cast<int64_t *>(AscendC::GmAlloc(sizeof(int64_t) * kRows)); | ||
| 954 | + auto *table = static_cast<float *>(AscendC::GmAlloc(sizeof(float) * table_count)); | ||
| 955 | + auto *index2 = static_cast<int64_t *>(AscendC::GmAlloc(sizeof(int64_t) * kRows)); | ||
| 956 | + auto *output = static_cast<float *>(AscendC::GmAlloc(sizeof(float) * output_count)); | ||
| 957 | + ASSERT_NE(index0, nullptr); | ||
| 958 | + ASSERT_NE(table, nullptr); | ||
| 959 | + ASSERT_NE(index2, nullptr); | ||
| 960 | + ASSERT_NE(output, nullptr); | ||
| 961 | + | ||
| 962 | + for (int32_t row = 0; row < kTableRows; ++row) { | ||
| 963 | + for (int32_t column = 0; column < kColumns; ++column) { | ||
| 964 | + table[static_cast<int64_t>(row) * kColumns + column] = | ||
| 965 | + static_cast<float>((row % 97) * 0.25F + (column % 31) * 0.03125F); | ||
| 966 | + } | ||
| 967 | + } | ||
| 968 | + for (int32_t row = 0; row < kRows; ++row) { | ||
| 969 | + index0[row] = row % 2 == 0 ? -1 : static_cast<int64_t>((row * 17 + 3) % kTableRows); | ||
| 970 | + index2[row] = static_cast<int64_t>((row * 29 + 7) % kTableRows); | ||
| 971 | + } | ||
| 972 | + | ||
| 973 | + AutofuseTilingData tiling_data{}; | ||
| 974 | + uint32_t workspace_size = 0; | ||
| 975 | + uint32_t block_dim = 48; | ||
| 976 | + ASSERT_EQ(AutofuseTiling(&tiling_data, &workspace_size, &block_dim, 48U, 192U * 1024U), 0); | ||
| 977 | + void *workspace = workspace_size == 0U ? nullptr : AscendC::GmAlloc(workspace_size); | ||
| 978 | + ASSERT_TRUE(workspace_size == 0U || workspace != nullptr); | ||
| 979 | + AscendC::SetKernelMode(KernelMode::AIV_MODE); | ||
| 980 | + ICPU_RUN_KF(indirect_load_broadcast_index_where_simt_test, block_dim, reinterpret_cast<uint8_t *>(index0), | ||
| 981 | + reinterpret_cast<uint8_t *>(table), reinterpret_cast<uint8_t *>(index2), | ||
| 982 | + reinterpret_cast<uint8_t *>(output), reinterpret_cast<uint8_t *>(workspace), | ||
| 983 | + reinterpret_cast<uint8_t *>(&tiling_data)); | ||
| 984 | + | ||
| 985 | + for (int32_t row = 0; row < kRows; ++row) { | ||
| 986 | + const int64_t selected = index0[row] == -1 ? index2[row] : index0[row]; | ||
| 987 | + for (int32_t column = 0; column < kColumns; ++column) { | ||
| 988 | + const int64_t offset = static_cast<int64_t>(row) * kColumns + column; | ||
| 989 | + const float expected = table[selected * kColumns + column]; | ||
| 990 | + EXPECT_FLOAT_EQ(output[offset], expected) << "row=" << row << ", column=" << column; | ||
| 991 | + } | ||
| 992 | + } | ||
| 993 | + | ||
| 994 | + if (workspace != nullptr) { | ||
| 995 | + AscendC::GmFree(workspace); | ||
| 996 | + } | ||
| 997 | + AscendC::GmFree(index0); | ||
| 998 | + AscendC::GmFree(table); | ||
| 999 | + AscendC::GmFree(index2); | ||
| 1000 | + AscendC::GmFree(output); | ||
| 1001 | +} | ||
| 1002 | + | ||
| 1003 | + | ||
| 1004 | +constexpr int32_t kAddIlReduceRows = 4; | ||
| 1005 | +constexpr int32_t kAddIlReduceColumns = 16; | ||
| 1006 | +constexpr int32_t kAddIlReduceTableRows = 8; | ||
| 1007 | + | ||
| 1008 | +TEST(E2EIndirectLoadAddIlReduce, GeneratedKernelMatchesReference) { | ||
| 1009 | + const int64_t table_count = static_cast<int64_t>(kAddIlReduceTableRows) * kAddIlReduceColumns; | ||
| 1010 | + auto *index0 = static_cast<int64_t *>(AscendC::GmAlloc(sizeof(int64_t) * kAddIlReduceRows)); | ||
| 1011 | + auto *table = static_cast<float *>(AscendC::GmAlloc(sizeof(float) * table_count)); | ||
| 1012 | + auto *offset = static_cast<int64_t *>(AscendC::GmAlloc(sizeof(int64_t) * kAddIlReduceRows)); | ||
| 1013 | + auto *output = static_cast<float *>(AscendC::GmAlloc(sizeof(float) * kAddIlReduceRows)); | ||
| 1014 | + ASSERT_NE(index0, nullptr); | ||
| 1015 | + ASSERT_NE(table, nullptr); | ||
| 1016 | + ASSERT_NE(offset, nullptr); | ||
| 1017 | + ASSERT_NE(output, nullptr); | ||
| 1018 | + | ||
| 1019 | + for (int32_t row = 0; row < kAddIlReduceTableRows; ++row) { | ||
| 1020 | + for (int32_t column = 0; column < kAddIlReduceColumns; ++column) { | ||
| 1021 | + table[row * kAddIlReduceColumns + column] = static_cast<float>(row * kAddIlReduceColumns + column); | ||
| 1022 | + } | ||
| 1023 | + } | ||
| 1024 | + for (int32_t row = 0; row < kAddIlReduceRows; ++row) { | ||
| 1025 | + index0[row] = static_cast<int64_t>((row * 3) % kAddIlReduceTableRows); | ||
| 1026 | + offset[row] = row % 2 == 0 ? 1 : -1; | ||
| 1027 | + } | ||
| 1028 | + | ||
| 1029 | + AutofuseTilingData tiling_data{}; | ||
| 1030 | + uint32_t workspace_size = 0; | ||
| 1031 | + uint32_t block_dim = 48; | ||
| 1032 | + ASSERT_EQ(AutofuseTiling(&tiling_data, &workspace_size, &block_dim, 48U, 192U * 1024U), 0); | ||
| 1033 | + void *workspace = workspace_size == 0U ? nullptr : AscendC::GmAlloc(workspace_size); | ||
| 1034 | + ASSERT_TRUE(workspace_size == 0U || workspace != nullptr); | ||
| 1035 | + AscendC::SetKernelMode(KernelMode::AIV_MODE); | ||
| 1036 | + ICPU_RUN_KF(indirect_load_add_il_reduce_test, block_dim, reinterpret_cast<uint8_t *>(index0), | ||
| 1037 | + reinterpret_cast<uint8_t *>(table), reinterpret_cast<uint8_t *>(offset), | ||
| 1038 | + reinterpret_cast<uint8_t *>(output), reinterpret_cast<uint8_t *>(workspace), | ||
| 1039 | + reinterpret_cast<uint8_t *>(&tiling_data)); | ||
| 1040 | + | ||
| 1041 | + for (int32_t row = 0; row < kAddIlReduceRows; ++row) { | ||
| 1042 | + // 与图语义一致：FLOAT 域做 Add 后 Cast 回 INT64 作为 gather 行号。 | ||
| 1043 | + const int64_t selected = static_cast<int64_t>(static_cast<float>(index0[row]) + static_cast<float>(offset[row])); | ||
| 1044 | + float expected = 0.0F; | ||
| 1045 | + for (int32_t column = 0; column < kAddIlReduceColumns; ++column) { | ||
| 1046 | + expected += table[selected * kAddIlReduceColumns + column]; | ||
| 1047 | + } | ||
| 1048 | + EXPECT_NEAR(output[row], expected, 0.0625F) << "row=" << row; | ||
| 1049 | + } | ||
| 1050 | + | ||
| 1051 | + if (workspace != nullptr) { | ||
| 1052 | + AscendC::GmFree(workspace); | ||
| 1053 | + } | ||
| 1054 | + AscendC::GmFree(index0); | ||
| 1055 | + AscendC::GmFree(table); | ||
| 1056 | + AscendC::GmFree(offset); | ||
| 1057 | + AscendC::GmFree(output); | ||
| 1058 | +} | ||
| 1059 | + | ||
| 1060 | +} // namespace | ||
| 1061 | + | ||
| 1062 | + | ||
| 1063 | + | ||
| 1064 | + | ||
| 1065 | +/** | ||
| 1066 | + * Copyright (c) 2026 Huawei Technologies Co., Ltd. | ||
| 1067 | + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of | ||
| 1068 | + * CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 1069 | + * Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 1070 | + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 1071 | + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 1072 | + * See LICENSE in the root of the software repository for the full text of the License. | ||
| 1073 | + */ | ||
| 1074 | + | ||
| 1075 | + | ||
| 1076 | + | ||
| 1077 | + | ||
| 1078 | + | ||
| 1079 | + | ||
| 1080 | +extern "C" __global__ __aicore__ void indirect_load_stride_zero_test(GM_ADDR x, GM_ADDR index, GM_ADDR y, | ||
| 1081 | + GM_ADDR workspace, GM_ADDR tiling); | ||
| 1082 | + | ||
| 1083 | +namespace { | ||
| 1084 | +using DataType = half; | ||
| 1085 | +using IndexType = int64_t; | ||
| 1086 | +constexpr std::array<int64_t, 4> kShape = {4, 5, 4, 16}; | ||
| 1087 | +constexpr uint32_t kInputZeroStrideMask = IL_INPUT_ZERO_STRIDE_MASK; | ||
| 1088 | +constexpr uint32_t kIndexZeroStrideMask = IL_INDEX_ZERO_STRIDE_MASK; | ||
| 1089 | +constexpr bool kHasInputElement = IL_HAS_INPUT_ELEMENT > 0; | ||
| 1090 | +constexpr bool kHasIndexElement = IL_HAS_INDEX_ELEMENT > 0; | ||
| 1091 | + | ||
| 1092 | +std::array<int64_t, 4> MakeStrides(uint32_t zero_stride_mask) { | ||
| 1093 | + std::array<int64_t, 4> strides{}; | ||
| 1094 | + int64_t stride = 1; | ||
| 1095 | + for (size_t index = kShape.size(); index > 0UL; --index) { | ||
| 1096 | + const size_t dim = index - 1UL; | ||
| 1097 | + if ((zero_stride_mask & (1U << dim)) == 0U) { | ||
| 1098 | + strides[dim] = stride; | ||
| 1099 | + stride *= kShape[dim]; | ||
| 1100 | + } | ||
| 1101 | + } | ||
| 1102 | + return strides; | ||
| 1103 | +} | ||
| 1104 | + | ||
| 1105 | +int64_t StorageSpan(const std::array<int64_t, 4> &strides) { | ||
| 1106 | + int64_t span = 1; | ||
| 1107 | + for (size_t dim = 0; dim < kShape.size(); ++dim) { | ||
| 1108 | + span += (kShape[dim] - 1) * strides[dim]; | ||
| 1109 | + } | ||
| 1110 | + return span; | ||
| 1111 | +} | ||
| 1112 | + | ||
| 1113 | +int64_t Offset(const std::array<int64_t, 4> &coordinate, const std::array<int64_t, 4> &strides) { | ||
| 1114 | + int64_t offset = 0; | ||
| 1115 | + for (size_t dim = 0; dim < coordinate.size(); ++dim) { | ||
| 1116 | + offset += coordinate[dim] * strides[dim]; | ||
| 1117 | + } | ||
| 1118 | + return offset; | ||
| 1119 | +} | ||
| 1120 | + | ||
| 1121 | +void InitializeData(DataType *x, IndexType *index, DataType *expected, const std::array<int64_t, 4> &input_strides, | ||
| 1122 | + const std::array<int64_t, 4> &index_strides) { | ||
| 1123 | + const int64_t input_count = StorageSpan(input_strides); | ||
| 1124 | + const int64_t index_count = StorageSpan(index_strides); | ||
| 1125 | + for (int64_t i = 0; i < input_count; ++i) { | ||
| 1126 | + x[i] = static_cast<DataType>(static_cast<float>((i % 37) - 18) * 0.25F); | ||
| 1127 | + } | ||
| 1128 | + for (int64_t i = 0; i < index_count; ++i) { | ||
| 1129 | + const int64_t gathered_axis = (i * 3 + 1) % kShape[2]; | ||
| 1130 | + index[i] = kHasIndexElement && (i % 2 == 1) ? -gathered_axis : gathered_axis; | ||
| 1131 | + } | ||
| 1132 | + | ||
| 1133 | + const int64_t output_count = kShape[0] * kShape[1] * kShape[2] * kShape[3]; | ||
| 1134 | + for (int64_t output_offset = 0; output_offset < output_count; ++output_offset) { | ||
| 1135 | + int64_t linear = output_offset; | ||
| 1136 | + const int64_t d = linear % kShape[3]; | ||
| 1137 | + linear /= kShape[3]; | ||
| 1138 | + const int64_t c = linear % kShape[2]; | ||
| 1139 | + linear /= kShape[2]; | ||
| 1140 | + const int64_t b = linear % kShape[1]; | ||
| 1141 | + const int64_t a = linear / kShape[1]; | ||
| 1142 | + std::array<int64_t, 4> coordinate = {a, b, c, d}; | ||
| 1143 | + int64_t gathered_axis = index[Offset(coordinate, index_strides)]; | ||
| 1144 | + if (kHasIndexElement) { | ||
| 1145 | + gathered_axis = std::abs(gathered_axis); | ||
| 1146 | + } | ||
| 1147 | + coordinate[2] = gathered_axis; | ||
| 1148 | + float value = static_cast<float>(x[Offset(coordinate, input_strides)]); | ||
| 1149 | + if (kHasInputElement) { | ||
| 1150 | + value = std::abs(value); | ||
| 1151 | + } | ||
| 1152 | + expected[output_offset] = static_cast<DataType>(std::max(value, 0.0F)); | ||
| 1153 | + } | ||
| 1154 | +} | ||
| 1155 | +} // namespace | ||
| 1156 | + | ||
| 1157 | +TEST(E2EIndirectLoadStrideZero, GeneratedKernelMatchesPhysicalStrideReference) { | ||
| 1158 | + const auto dense_strides = MakeStrides(0U); | ||
| 1159 | + const auto input_strides = MakeStrides(kInputZeroStrideMask); | ||
| 1160 | + const auto index_strides = MakeStrides(kIndexZeroStrideMask); | ||
| 1161 | + const int64_t input_count = StorageSpan(input_strides); | ||
| 1162 | + const int64_t index_count = StorageSpan(index_strides); | ||
| 1163 | + const int64_t output_count = StorageSpan(dense_strides); | ||
| 1164 | + indirect_load_test::KernelData<DataType, IndexType> buffers(input_count, index_count, output_count); | ||
| 1165 | + ASSERT_TRUE(buffers.IsValid()); | ||
| 1166 | + InitializeData(buffers.input.get(), buffers.index.get(), buffers.expected.data(), input_strides, index_strides); | ||
| 1167 | + std::fill_n(buffers.output.get(), output_count, static_cast<DataType>(0.0F)); | ||
| 1168 | + indirect_load_test::KernelTiling tiling; | ||
| 1169 | + ASSERT_TRUE(tiling.IsValid()); | ||
| 1170 | + | ||
| 1171 | + AscendC::SetKernelMode(KernelMode::AIV_MODE); | ||
| 1172 | + ICPU_RUN_KF(indirect_load_stride_zero_test, tiling.data.block_dim, reinterpret_cast<uint8_t *>(buffers.input.get()), | ||
| 1173 | + reinterpret_cast<uint8_t *>(buffers.index.get()), reinterpret_cast<uint8_t *>(buffers.output.get()), | ||
| 1174 | + tiling.workspace.get(), reinterpret_cast<uint8_t *>(&tiling.data)); | ||
| 1175 | + for (int64_t i = 0; i < output_count; ++i) { | ||
| 1176 | + EXPECT_NEAR(static_cast<float>(buffers.output.get()[i]), | ||
| 1177 | + static_cast<float>(buffers.expected[static_cast<size_t>(i)]), 0.0625F) | ||
| 1178 | + << "offset=" << i; | ||
| 1179 | + } | ||
| 1180 | +} | ||
| 1181 | + | ||
| 1182 | + | ||
| 1183 | + | ||
| 1184 | + | ||
| 1185 | +/** | ||
| 1186 | + * Copyright (c) 2026 Huawei Technologies Co., Ltd. | ||
| 1187 | + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of | ||
| 1188 | + * CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 1189 | + * Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 1190 | + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 1191 | + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 1192 | + * See LICENSE in the root of the software repository for the full text of the License. | ||
| 1193 | + */ | ||
| 1194 | + | ||
| 1195 | + | ||
| 1196 | + | ||
| 1197 | + | ||
| 1198 | + | ||
| 1199 | + | ||
| 1200 | + | ||
| 1201 | + | ||
| 1202 | + | ||
| 1203 | + | ||
| 1204 | + | ||
| 1205 | +extern "C" __global__ __aicore__ void indirect_load_torch_gather_strided_test(GM_ADDR data, GM_ADDR index, | ||
| 1206 | + GM_ADDR output, GM_ADDR workspace, | ||
| 1207 | + GM_ADDR tiling); | ||
| 1208 | +extern "C" int64_t AutofuseTiling(AutofuseTilingData *, uint32_t *, uint32_t *, uint32_t, uint32_t); | ||
| 1209 | + | ||
| 1210 | +namespace { | ||
| 1211 | +constexpr int32_t kInputStride0 = IL_INPUT_STRIDE0; | ||
| 1212 | +constexpr int32_t kInputStride1 = IL_INPUT_STRIDE1; | ||
| 1213 | +constexpr int32_t kInputStride2 = IL_INPUT_STRIDE2; | ||
| 1214 | +constexpr int32_t kIndexStride0 = IL_INDEX_STRIDE0; | ||
| 1215 | +constexpr int32_t kIndexStride1 = IL_INDEX_STRIDE1; | ||
| 1216 | +constexpr int32_t kIndexStride2 = IL_INDEX_STRIDE2; | ||
| 1217 | + | ||
| 1218 | +constexpr int32_t kEffectiveInputStride0 = 138; | ||
| 1219 | +constexpr int32_t kEffectiveInputStride1 = 23; | ||
| 1220 | +constexpr int32_t kEffectiveInputStride2 = 1; | ||
| 1221 | + | ||
| 1222 | +constexpr int32_t kEffectiveInputStride0 = kInputStride0; | ||
| 1223 | +constexpr int32_t kEffectiveInputStride1 = kInputStride1; | ||
| 1224 | +constexpr int32_t kEffectiveInputStride2 = kInputStride2; | ||
| 1225 | + | ||
| 1226 | + | ||
| 1227 | +constexpr int32_t kDim0 = 30; | ||
| 1228 | +constexpr int32_t kInputDim1 = 6; | ||
| 1229 | +constexpr int32_t kOutputDim1 = 3; | ||
| 1230 | +constexpr int32_t kDim2 = 23; | ||
| 1231 | + | ||
| 1232 | +constexpr int32_t kDim0 = 8; | ||
| 1233 | +constexpr int32_t kInputDim1 = 32; | ||
| 1234 | +constexpr int32_t kOutputDim1 = 16; | ||
| 1235 | +constexpr int32_t kDim2 = 5; | ||
| 1236 | + | ||
| 1237 | + | ||
| 1238 | +constexpr int32_t kInputStorageSize = kDim0 * kInputDim1 * kDim2; | ||
| 1239 | +constexpr int32_t kIndexStorageSize = kOutputDim1; | ||
| 1240 | +constexpr int32_t kOutputSize = kDim0 * kOutputDim1 * kDim2; | ||
| 1241 | + | ||
| 1242 | +constexpr int32_t kInputStorageSize = (8 - 1) * kInputStride0 + (32 - 1) * kInputStride1 + (5 - 1) * kInputStride2 + 1; | ||
| 1243 | +constexpr int32_t kIndexStorageSize = (8 - 1) * kIndexStride0 + (16 - 1) * kIndexStride1 + (5 - 1) * kIndexStride2 + 1; | ||
| 1244 | +constexpr int32_t kOutputSize = 8 * 16 * 5; | ||
| 1245 | + | ||
| 1246 | + | ||
| 1247 | +void Initialize(float *data, int64_t *index, std::vector<float> &expected) { | ||
| 1248 | + for (int32_t i = 0; i < kInputStorageSize; ++i) { | ||
| 1249 | + data[i] = 1.0F + static_cast<float>((i * 37) % 997) / 997.0F; | ||
| 1250 | + } | ||
| 1251 | + std::fill_n(index, kIndexStorageSize, int64_t{0}); | ||
| 1252 | + for (int32_t a0 = 0; a0 < kDim0; ++a0) { | ||
| 1253 | + for (int32_t a1 = 0; a1 < kOutputDim1; ++a1) { | ||
| 1254 | + for (int32_t a2 = 0; a2 < kDim2; ++a2) { | ||
| 1255 | + const int32_t output_offset = kOutputDim1 * kDim2 * a0 + kDim2 * a1 + a2; | ||
| 1256 | + | ||
| 1257 | + const int32_t index_offset = a1; | ||
| 1258 | + const int64_t index_value = (a1 * 2 + 1) % kInputDim1; | ||
| 1259 | + | ||
| 1260 | + const int32_t index_offset = kIndexStride0 * a0 + kIndexStride1 * a1 + kIndexStride2 * a2; | ||
| 1261 | + const int64_t index_value = (a0 * 17 + a1 * 7 + a2 * 3) % 32; | ||
| 1262 | + | ||
| 1263 | + index[index_offset] = index_value; | ||
| 1264 | + expected[static_cast<size_t>(output_offset)] = | ||
| 1265 | + data[kEffectiveInputStride0 * a0 + kEffectiveInputStride1 * index_value + kEffectiveInputStride2 * a2]; | ||
| 1266 | + } | ||
| 1267 | + } | ||
| 1268 | + } | ||
| 1269 | +} | ||
| 1270 | +} // namespace | ||
| 1271 | + | ||
| 1272 | +TEST(E2EIndirectLoadTorchGatherStrided, GeneratedKernelMatchesReference) { | ||
| 1273 | + const auto gm_free = [](void *ptr) { AscendC::GmFree(ptr); }; | ||
| 1274 | + std::unique_ptr<float, decltype(gm_free)> data( | ||
| 1275 | + reinterpret_cast<float *>(AscendC::GmAlloc(kInputStorageSize * sizeof(float))), gm_free); | ||
| 1276 | + std::unique_ptr<int64_t, decltype(gm_free)> index( | ||
| 1277 | + reinterpret_cast<int64_t *>(AscendC::GmAlloc(kIndexStorageSize * sizeof(int64_t))), gm_free); | ||
| 1278 | + std::unique_ptr<float, decltype(gm_free)> output( | ||
| 1279 | + reinterpret_cast<float *>(AscendC::GmAlloc(kOutputSize * sizeof(float))), gm_free); | ||
| 1280 | + ASSERT_NE(data, nullptr); | ||
| 1281 | + ASSERT_NE(index, nullptr); | ||
| 1282 | + ASSERT_NE(output, nullptr); | ||
| 1283 | + | ||
| 1284 | + std::vector<float> expected(kOutputSize); | ||
| 1285 | + Initialize(data.get(), index.get(), expected); | ||
| 1286 | + std::fill_n(output.get(), kOutputSize, 0.0F); | ||
| 1287 | + | ||
| 1288 | + AutofuseTilingData tiling_data{}; | ||
| 1289 | + uint32_t workspace_size = 0U; | ||
| 1290 | + uint32_t block_dim = 48U; | ||
| 1291 | + ASSERT_EQ(AutofuseTiling(&tiling_data, &workspace_size, &block_dim, 48U, 192U * 1024U), 0); | ||
| 1292 | + | ||
| 1293 | + ASSERT_EQ(tiling_data.get_tiling_key(), static_cast<uint32_t>(IL_EXPECT_TILING_KEY)); | ||
| 1294 | + | ||
| 1295 | + ASSERT_EQ(tiling_data.graph0_tiling_key, static_cast<uint32_t>(IL_EXPECT_TILING_KEY)); | ||
| 1296 | + | ||
| 1297 | + ASSERT_GT(tiling_data.block_dim, 0U); | ||
| 1298 | + std::unique_ptr<uint8_t, decltype(gm_free)> workspace(nullptr, gm_free); | ||
| 1299 | + if (workspace_size > 0U) { | ||
| 1300 | + workspace.reset(reinterpret_cast<uint8_t *>(AscendC::GmAlloc(workspace_size))); | ||
| 1301 | + ASSERT_NE(workspace, nullptr); | ||
| 1302 | + std::fill_n(workspace.get(), workspace_size, uint8_t{0}); | ||
| 1303 | + } | ||
| 1304 | + | ||
| 1305 | + AscendC::SetKernelMode(KernelMode::AIV_MODE); | ||
| 1306 | + ICPU_RUN_KF(indirect_load_torch_gather_strided_test, tiling_data.block_dim, reinterpret_cast<uint8_t *>(data.get()), | ||
| 1307 | + reinterpret_cast<uint8_t *>(index.get()), reinterpret_cast<uint8_t *>(output.get()), workspace.get(), | ||
| 1308 | + reinterpret_cast<uint8_t *>(&tiling_data)); | ||
| 1309 | + int32_t mismatch_count = 0; | ||
| 1310 | + int32_t first_mismatch = -1; | ||
| 1311 | + for (int32_t i = 0; i < kOutputSize; ++i) { | ||
| 1312 | + if (output.get()[i] != expected[static_cast<size_t>(i)]) { | ||
| 1313 | + if (first_mismatch < 0) { | ||
| 1314 | + first_mismatch = i; | ||
| 1315 | + } | ||
| 1316 | + ++mismatch_count; | ||
| 1317 | + } | ||
| 1318 | + } | ||
| 1319 | + if (mismatch_count > 0) { | ||
| 1320 | + ADD_FAILURE() << "mismatch count=" << mismatch_count << ", first mismatch at offset=" << first_mismatch | ||
| 1321 | + << ", actual=" << output.get()[first_mismatch] | ||
| 1322 | + << ", expected=" << expected[static_cast<size_t>(first_mismatch)]; | ||
| 1323 | + } | ||
| 1324 | +} | ||
| 1325 | + | ||
| 1326 | + | ||
| 1327 | + | ||
| 1328 | + | ||
| 1329 | +/** | ||
| 1330 | + * Copyright (c) 2026 Huawei Technologies Co., Ltd. | ||
| 1331 | + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of | ||
| 1332 | + * CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 1333 | + * Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 1334 | + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 1335 | + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 1336 | + * See LICENSE in the root of the software repository for the full text of the License. | ||
| 1337 | + */ | ||
| 1338 | + | ||
| 1339 | + | ||
| 1340 | + | ||
| 1341 | + | ||
| 1342 | +extern "C" __global__ __aicore__ void indirect_load_embedding_test(GM_ADDR input, GM_ADDR index, GM_ADDR output, | ||
| 1343 | + GM_ADDR workspace, GM_ADDR tiling); | ||
| 1344 | + | ||
| 1345 | +namespace { | ||
| 1346 | +constexpr int32_t kInputRows = 64; | ||
| 1347 | +constexpr int32_t kEmbeddingSize = 32; | ||
| 1348 | +constexpr int32_t kIndexRows = 32; | ||
| 1349 | + | ||
| 1350 | +void InitializeData(float *input, int32_t *index, float *expected) { | ||
| 1351 | + for (int32_t row = 0; row < kInputRows; ++row) { | ||
| 1352 | + for (int32_t col = 0; col < kEmbeddingSize; ++col) { | ||
| 1353 | + input[row * kEmbeddingSize + col] = static_cast<float>(row * kEmbeddingSize + col); | ||
| 1354 | + } | ||
| 1355 | + } | ||
| 1356 | + for (int32_t row = 0; row < kIndexRows; ++row) { | ||
| 1357 | + index[row] = static_cast<int32_t>((row * 7 + 3) % kInputRows); | ||
| 1358 | + float sum = 0.0F; | ||
| 1359 | + for (int32_t col = 0; col < kEmbeddingSize; ++col) { | ||
| 1360 | + sum += 2.0F * (input[index[row] * kEmbeddingSize + col] + 0.1F); | ||
| 1361 | + } | ||
| 1362 | + expected[row] = sum; | ||
| 1363 | + } | ||
| 1364 | +} | ||
| 1365 | +} // namespace | ||
| 1366 | + | ||
| 1367 | +TEST(E2EIndirectLoadEmbedding, GeneratedKernelMatchesReference) { | ||
| 1368 | + constexpr int64_t input_count = static_cast<int64_t>(kInputRows) * kEmbeddingSize; | ||
| 1369 | + constexpr int64_t index_count = kIndexRows; | ||
| 1370 | + constexpr int64_t output_count = kIndexRows; | ||
| 1371 | + indirect_load_test::KernelData<float, int32_t> buffers(input_count, index_count, output_count); | ||
| 1372 | + ASSERT_TRUE(buffers.IsValid()); | ||
| 1373 | + InitializeData(buffers.input.get(), buffers.index.get(), buffers.expected.data()); | ||
| 1374 | + std::fill_n(buffers.output.get(), output_count, 0.0F); | ||
| 1375 | + | ||
| 1376 | + indirect_load_test::KernelTiling tiling; | ||
| 1377 | + ASSERT_TRUE(tiling.IsValid()); | ||
| 1378 | + AscendC::SetKernelMode(KernelMode::AIV_MODE); | ||
| 1379 | + ICPU_RUN_KF(indirect_load_embedding_test, tiling.data.block_dim, reinterpret_cast<uint8_t *>(buffers.input.get()), | ||
| 1380 | + reinterpret_cast<uint8_t *>(buffers.index.get()), reinterpret_cast<uint8_t *>(buffers.output.get()), | ||
| 1381 | + tiling.workspace.get(), reinterpret_cast<uint8_t *>(&tiling.data)); | ||
| 1382 | + | ||
| 1383 | + // 设备侧 ReduceSum 走向量/树形归约，与 CPU 串行累加顺序不同，浮点结果存在数 ULP 差异，故用绝对容差比较。 | ||
| 1384 | + for (int64_t i = 0; i < output_count; ++i) { | ||
| 1385 | + EXPECT_NEAR(buffers.output.get()[i], buffers.expected[static_cast<size_t>(i)], 0.0625F) << "offset=" << i; | ||
| 1386 | + } | ||
| 1387 | +} | ||
| 1388 | + | ||
| 1389 | + | ||
Dautofuse/tests/v35/st/backend_e2e_v2/indirect_load_store_test/test_e2e_indirect_load_stride_zero_kernel.cpp+0-118
| @@ -1,118 +0,0 @@ | |||
| 1 | -/** | ||
| 2 | - * Copyright (c) 2026 Huawei Technologies Co., Ltd. | ||
| 3 | - * This program is free software, you can redistribute it and/or modify it under the terms and conditions of | ||
| 4 | - * CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 5 | - * Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 6 | - * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 7 | - * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 8 | - * See LICENSE in the root of the software repository for the full text of the License. | ||
| 9 | - */ | ||
| 10 | - | ||
| 11 | - | ||
| 12 | - | ||
| 13 | - | ||
| 14 | - | ||
| 15 | - | ||
| 16 | - | ||
| 17 | - | ||
| 18 | -extern "C" __global__ __aicore__ void indirect_load_stride_zero_test(GM_ADDR x, GM_ADDR index, GM_ADDR y, | ||
| 19 | - GM_ADDR workspace, GM_ADDR tiling); | ||
| 20 | - | ||
| 21 | -namespace { | ||
| 22 | -using DataType = half; | ||
| 23 | -using IndexType = int64_t; | ||
| 24 | -constexpr std::array<int64_t, 4> kShape = {4, 5, 4, 16}; | ||
| 25 | -constexpr uint32_t kInputZeroStrideMask = IL_INPUT_ZERO_STRIDE_MASK; | ||
| 26 | -constexpr uint32_t kIndexZeroStrideMask = IL_INDEX_ZERO_STRIDE_MASK; | ||
| 27 | -constexpr bool kHasInputElement = IL_HAS_INPUT_ELEMENT > 0; | ||
| 28 | -constexpr bool kHasIndexElement = IL_HAS_INDEX_ELEMENT > 0; | ||
| 29 | - | ||
| 30 | -std::array<int64_t, 4> MakeStrides(uint32_t zero_stride_mask) { | ||
| 31 | - std::array<int64_t, 4> strides{}; | ||
| 32 | - int64_t stride = 1; | ||
| 33 | - for (size_t index = kShape.size(); index > 0UL; --index) { | ||
| 34 | - const size_t dim = index - 1UL; | ||
| 35 | - if ((zero_stride_mask & (1U << dim)) == 0U) { | ||
| 36 | - strides[dim] = stride; | ||
| 37 | - stride *= kShape[dim]; | ||
| 38 | - } | ||
| 39 | - } | ||
| 40 | - return strides; | ||
| 41 | -} | ||
| 42 | - | ||
| 43 | -int64_t StorageSpan(const std::array<int64_t, 4> &strides) { | ||
| 44 | - int64_t span = 1; | ||
| 45 | - for (size_t dim = 0; dim < kShape.size(); ++dim) { | ||
| 46 | - span += (kShape[dim] - 1) * strides[dim]; | ||
| 47 | - } | ||
| 48 | - return span; | ||
| 49 | -} | ||
| 50 | - | ||
| 51 | -int64_t Offset(const std::array<int64_t, 4> &coordinate, const std::array<int64_t, 4> &strides) { | ||
| 52 | - int64_t offset = 0; | ||
| 53 | - for (size_t dim = 0; dim < coordinate.size(); ++dim) { | ||
| 54 | - offset += coordinate[dim] * strides[dim]; | ||
| 55 | - } | ||
| 56 | - return offset; | ||
| 57 | -} | ||
| 58 | - | ||
| 59 | -void InitializeData(DataType *x, IndexType *index, DataType *expected, const std::array<int64_t, 4> &input_strides, | ||
| 60 | - const std::array<int64_t, 4> &index_strides) { | ||
| 61 | - const int64_t input_count = StorageSpan(input_strides); | ||
| 62 | - const int64_t index_count = StorageSpan(index_strides); | ||
| 63 | - for (int64_t i = 0; i < input_count; ++i) { | ||
| 64 | - x[i] = static_cast<DataType>(static_cast<float>((i % 37) - 18) * 0.25F); | ||
| 65 | - } | ||
| 66 | - for (int64_t i = 0; i < index_count; ++i) { | ||
| 67 | - const int64_t gathered_axis = (i * 3 + 1) % kShape[2]; | ||
| 68 | - index[i] = kHasIndexElement && (i % 2 == 1) ? -gathered_axis : gathered_axis; | ||
| 69 | - } | ||
| 70 | - | ||
| 71 | - const int64_t output_count = kShape[0] * kShape[1] * kShape[2] * kShape[3]; | ||
| 72 | - for (int64_t output_offset = 0; output_offset < output_count; ++output_offset) { | ||
| 73 | - int64_t linear = output_offset; | ||
| 74 | - const int64_t d = linear % kShape[3]; | ||
| 75 | - linear /= kShape[3]; | ||
| 76 | - const int64_t c = linear % kShape[2]; | ||
| 77 | - linear /= kShape[2]; | ||
| 78 | - const int64_t b = linear % kShape[1]; | ||
| 79 | - const int64_t a = linear / kShape[1]; | ||
| 80 | - std::array<int64_t, 4> coordinate = {a, b, c, d}; | ||
| 81 | - int64_t gathered_axis = index[Offset(coordinate, index_strides)]; | ||
| 82 | - if (kHasIndexElement) { | ||
| 83 | - gathered_axis = std::abs(gathered_axis); | ||
| 84 | - } | ||
| 85 | - coordinate[2] = gathered_axis; | ||
| 86 | - float value = static_cast<float>(x[Offset(coordinate, input_strides)]); | ||
| 87 | - if (kHasInputElement) { | ||
| 88 | - value = std::abs(value); | ||
| 89 | - } | ||
| 90 | - expected[output_offset] = static_cast<DataType>(std::max(value, 0.0F)); | ||
| 91 | - } | ||
| 92 | -} | ||
| 93 | -} // namespace | ||
| 94 | - | ||
| 95 | -TEST(E2EIndirectLoadStrideZero, GeneratedKernelMatchesPhysicalStrideReference) { | ||
| 96 | - const auto dense_strides = MakeStrides(0U); | ||
| 97 | - const auto input_strides = MakeStrides(kInputZeroStrideMask); | ||
| 98 | - const auto index_strides = MakeStrides(kIndexZeroStrideMask); | ||
| 99 | - const int64_t input_count = StorageSpan(input_strides); | ||
| 100 | - const int64_t index_count = StorageSpan(index_strides); | ||
| 101 | - const int64_t output_count = StorageSpan(dense_strides); | ||
| 102 | - indirect_load_test::KernelData<DataType, IndexType> buffers(input_count, index_count, output_count); | ||
| 103 | - ASSERT_TRUE(buffers.IsValid()); | ||
| 104 | - InitializeData(buffers.input.get(), buffers.index.get(), buffers.expected.data(), input_strides, index_strides); | ||
| 105 | - std::fill_n(buffers.output.get(), output_count, static_cast<DataType>(0.0F)); | ||
| 106 | - indirect_load_test::KernelTiling tiling; | ||
| 107 | - ASSERT_TRUE(tiling.IsValid()); | ||
| 108 | - | ||
| 109 | - AscendC::SetKernelMode(KernelMode::AIV_MODE); | ||
| 110 | - ICPU_RUN_KF(indirect_load_stride_zero_test, tiling.data.block_dim, reinterpret_cast<uint8_t *>(buffers.input.get()), | ||
| 111 | - reinterpret_cast<uint8_t *>(buffers.index.get()), reinterpret_cast<uint8_t *>(buffers.output.get()), | ||
| 112 | - tiling.workspace.get(), reinterpret_cast<uint8_t *>(&tiling.data)); | ||
| 113 | - for (int64_t i = 0; i < output_count; ++i) { | ||
| 114 | - EXPECT_NEAR(static_cast<float>(buffers.output.get()[i]), | ||
| 115 | - static_cast<float>(buffers.expected[static_cast<size_t>(i)]), 0.0625F) | ||
| 116 | - << "offset=" << i; | ||
| 117 | - } | ||
| 118 | -} | ||
Dautofuse/tests/v35/st/backend_e2e_v2/indirect_load_store_test/test_e2e_indirect_load_torch_gather_strided_kernel.cpp+0-140
| @@ -1,140 +0,0 @@ | |||
| 1 | -/** | ||
| 2 | - * Copyright (c) 2026 Huawei Technologies Co., Ltd. | ||
| 3 | - * This program is free software, you can redistribute it and/or modify it under the terms and conditions of | ||
| 4 | - * CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 5 | - * Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 6 | - * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 7 | - * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 8 | - * See LICENSE in the root of the software repository for the full text of the License. | ||
| 9 | - */ | ||
| 10 | - | ||
| 11 | - | ||
| 12 | - | ||
| 13 | - | ||
| 14 | - | ||
| 15 | - | ||
| 16 | - | ||
| 17 | - | ||
| 18 | - | ||
| 19 | - | ||
| 20 | - | ||
| 21 | -extern "C" __global__ __aicore__ void indirect_load_torch_gather_strided_test(GM_ADDR data, GM_ADDR index, | ||
| 22 | - GM_ADDR output, GM_ADDR workspace, | ||
| 23 | - GM_ADDR tiling); | ||
| 24 | -extern "C" int64_t AutofuseTiling(AutofuseTilingData *, uint32_t *, uint32_t *, uint32_t, uint32_t); | ||
| 25 | - | ||
| 26 | -namespace { | ||
| 27 | -constexpr int32_t kInputStride0 = IL_INPUT_STRIDE0; | ||
| 28 | -constexpr int32_t kInputStride1 = IL_INPUT_STRIDE1; | ||
| 29 | -constexpr int32_t kInputStride2 = IL_INPUT_STRIDE2; | ||
| 30 | -constexpr int32_t kIndexStride0 = IL_INDEX_STRIDE0; | ||
| 31 | -constexpr int32_t kIndexStride1 = IL_INDEX_STRIDE1; | ||
| 32 | -constexpr int32_t kIndexStride2 = IL_INDEX_STRIDE2; | ||
| 33 | - | ||
| 34 | -constexpr int32_t kEffectiveInputStride0 = 138; | ||
| 35 | -constexpr int32_t kEffectiveInputStride1 = 23; | ||
| 36 | -constexpr int32_t kEffectiveInputStride2 = 1; | ||
| 37 | - | ||
| 38 | -constexpr int32_t kEffectiveInputStride0 = kInputStride0; | ||
| 39 | -constexpr int32_t kEffectiveInputStride1 = kInputStride1; | ||
| 40 | -constexpr int32_t kEffectiveInputStride2 = kInputStride2; | ||
| 41 | - | ||
| 42 | - | ||
| 43 | -constexpr int32_t kDim0 = 30; | ||
| 44 | -constexpr int32_t kInputDim1 = 6; | ||
| 45 | -constexpr int32_t kOutputDim1 = 3; | ||
| 46 | -constexpr int32_t kDim2 = 23; | ||
| 47 | - | ||
| 48 | -constexpr int32_t kDim0 = 8; | ||
| 49 | -constexpr int32_t kInputDim1 = 32; | ||
| 50 | -constexpr int32_t kOutputDim1 = 16; | ||
| 51 | -constexpr int32_t kDim2 = 5; | ||
| 52 | - | ||
| 53 | - | ||
| 54 | -constexpr int32_t kInputStorageSize = kDim0 * kInputDim1 * kDim2; | ||
| 55 | -constexpr int32_t kIndexStorageSize = kOutputDim1; | ||
| 56 | -constexpr int32_t kOutputSize = kDim0 * kOutputDim1 * kDim2; | ||
| 57 | - | ||
| 58 | -constexpr int32_t kInputStorageSize = (8 - 1) * kInputStride0 + (32 - 1) * kInputStride1 + (5 - 1) * kInputStride2 + 1; | ||
| 59 | -constexpr int32_t kIndexStorageSize = (8 - 1) * kIndexStride0 + (16 - 1) * kIndexStride1 + (5 - 1) * kIndexStride2 + 1; | ||
| 60 | -constexpr int32_t kOutputSize = 8 * 16 * 5; | ||
| 61 | - | ||
| 62 | - | ||
| 63 | -void Initialize(float *data, int64_t *index, std::vector<float> &expected) { | ||
| 64 | - for (int32_t i = 0; i < kInputStorageSize; ++i) { | ||
| 65 | - data[i] = 1.0F + static_cast<float>((i * 37) % 997) / 997.0F; | ||
| 66 | - } | ||
| 67 | - std::fill_n(index, kIndexStorageSize, int64_t{0}); | ||
| 68 | - for (int32_t a0 = 0; a0 < kDim0; ++a0) { | ||
| 69 | - for (int32_t a1 = 0; a1 < kOutputDim1; ++a1) { | ||
| 70 | - for (int32_t a2 = 0; a2 < kDim2; ++a2) { | ||
| 71 | - const int32_t output_offset = kOutputDim1 * kDim2 * a0 + kDim2 * a1 + a2; | ||
| 72 | - | ||
| 73 | - const int32_t index_offset = a1; | ||
| 74 | - const int64_t index_value = (a1 * 2 + 1) % kInputDim1; | ||
| 75 | - | ||
| 76 | - const int32_t index_offset = kIndexStride0 * a0 + kIndexStride1 * a1 + kIndexStride2 * a2; | ||
| 77 | - const int64_t index_value = (a0 * 17 + a1 * 7 + a2 * 3) % 32; | ||
| 78 | - | ||
| 79 | - index[index_offset] = index_value; | ||
| 80 | - expected[static_cast<size_t>(output_offset)] = | ||
| 81 | - data[kEffectiveInputStride0 * a0 + kEffectiveInputStride1 * index_value + kEffectiveInputStride2 * a2]; | ||
| 82 | - } | ||
| 83 | - } | ||
| 84 | - } | ||
| 85 | -} | ||
| 86 | -} // namespace | ||
| 87 | - | ||
| 88 | -TEST(E2EIndirectLoadTorchGatherStrided, GeneratedKernelMatchesReference) { | ||
| 89 | - const auto gm_free = [](void *ptr) { AscendC::GmFree(ptr); }; | ||
| 90 | - std::unique_ptr<float, decltype(gm_free)> data( | ||
| 91 | - reinterpret_cast<float *>(AscendC::GmAlloc(kInputStorageSize * sizeof(float))), gm_free); | ||
| 92 | - std::unique_ptr<int64_t, decltype(gm_free)> index( | ||
| 93 | - reinterpret_cast<int64_t *>(AscendC::GmAlloc(kIndexStorageSize * sizeof(int64_t))), gm_free); | ||
| 94 | - std::unique_ptr<float, decltype(gm_free)> output( | ||
| 95 | - reinterpret_cast<float *>(AscendC::GmAlloc(kOutputSize * sizeof(float))), gm_free); | ||
| 96 | - ASSERT_NE(data, nullptr); | ||
| 97 | - ASSERT_NE(index, nullptr); | ||
| 98 | - ASSERT_NE(output, nullptr); | ||
| 99 | - | ||
| 100 | - std::vector<float> expected(kOutputSize); | ||
| 101 | - Initialize(data.get(), index.get(), expected); | ||
| 102 | - std::fill_n(output.get(), kOutputSize, 0.0F); | ||
| 103 | - | ||
| 104 | - AutofuseTilingData tiling_data{}; | ||
| 105 | - uint32_t workspace_size = 0U; | ||
| 106 | - uint32_t block_dim = 48U; | ||
| 107 | - ASSERT_EQ(AutofuseTiling(&tiling_data, &workspace_size, &block_dim, 48U, 192U * 1024U), 0); | ||
| 108 | - | ||
| 109 | - ASSERT_EQ(tiling_data.get_tiling_key(), static_cast<uint32_t>(IL_EXPECT_TILING_KEY)); | ||
| 110 | - | ||
| 111 | - ASSERT_EQ(tiling_data.graph0_tiling_key, static_cast<uint32_t>(IL_EXPECT_TILING_KEY)); | ||
| 112 | - | ||
| 113 | - ASSERT_GT(tiling_data.block_dim, 0U); | ||
| 114 | - std::unique_ptr<uint8_t, decltype(gm_free)> workspace(nullptr, gm_free); | ||
| 115 | - if (workspace_size > 0U) { | ||
| 116 | - workspace.reset(reinterpret_cast<uint8_t *>(AscendC::GmAlloc(workspace_size))); | ||
| 117 | - ASSERT_NE(workspace, nullptr); | ||
| 118 | - std::fill_n(workspace.get(), workspace_size, uint8_t{0}); | ||
| 119 | - } | ||
| 120 | - | ||
| 121 | - AscendC::SetKernelMode(KernelMode::AIV_MODE); | ||
| 122 | - ICPU_RUN_KF(indirect_load_torch_gather_strided_test, tiling_data.block_dim, reinterpret_cast<uint8_t *>(data.get()), | ||
| 123 | - reinterpret_cast<uint8_t *>(index.get()), reinterpret_cast<uint8_t *>(output.get()), workspace.get(), | ||
| 124 | - reinterpret_cast<uint8_t *>(&tiling_data)); | ||
| 125 | - int32_t mismatch_count = 0; | ||
| 126 | - int32_t first_mismatch = -1; | ||
| 127 | - for (int32_t i = 0; i < kOutputSize; ++i) { | ||
| 128 | - if (output.get()[i] != expected[static_cast<size_t>(i)]) { | ||
| 129 | - if (first_mismatch < 0) { | ||
| 130 | - first_mismatch = i; | ||
| 131 | - } | ||
| 132 | - ++mismatch_count; | ||
| 133 | - } | ||
| 134 | - } | ||
| 135 | - if (mismatch_count > 0) { | ||
| 136 | - ADD_FAILURE() << "mismatch count=" << mismatch_count << ", first mismatch at offset=" << first_mismatch | ||
| 137 | - << ", actual=" << output.get()[first_mismatch] | ||
| 138 | - << ", expected=" << expected[static_cast<size_t>(first_mismatch)]; | ||
| 139 | - } | ||
| 140 | -} | ||
| @@ -324,15 +324,17 @@ __aicore__ inline bool TryIndirectLoadSimdEmbedding(const LocalTensor<X> &x, con | |||
| 324 | sizeof(X) <= AscendC::ONE_BLK_SIZE) { | 324 | sizeof(X) <= AscendC::ONE_BLK_SIZE) { |
| 325 | const int64_t embedding_size = shape[1]; | 325 | const int64_t embedding_size = shape[1]; |
| 326 | const int64_t block_elements = static_cast<int64_t>(AscendC::ONE_BLK_SIZE / sizeof(X)); | 326 | const int64_t block_elements = static_cast<int64_t>(AscendC::ONE_BLK_SIZE / sizeof(X)); |
| 327 | + const int64_t input_row_stride = shape[2]; | ||
| 328 | + const int64_t index_row_stride = shape[4]; | ||
| 327 | const bool full_rows = embedding_size > 0 && embedding_size % block_elements == 0 && shape[2] == embedding_size && | 329 | const bool full_rows = embedding_size > 0 && embedding_size % block_elements == 0 && shape[2] == embedding_size && |
| 328 | - shape[3] == 1 && shape[4] == 1 && shape[5] == 0 && output_offset % embedding_size == 0 && | 330 | + shape[3] == 1 && index_row_stride > 0 && shape[5] == 0 && |
| 329 | - actual_size % embedding_size == 0; | 331 | + output_offset % embedding_size == 0 && actual_size % embedding_size == 0; |
| 330 | if (full_rows) { | 332 | if (full_rows) { |
| 331 | const int64_t first_row = output_offset / embedding_size; | 333 | const int64_t first_row = output_offset / embedding_size; |
| 332 | const uint32_t row_count = actual_size / static_cast<uint32_t>(embedding_size); | 334 | const uint32_t row_count = actual_size / static_cast<uint32_t>(embedding_size); |
| 333 | for (uint32_t row = 0; row < row_count; ++row) { | 335 | for (uint32_t row = 0; row < row_count; ++row) { |
| 334 | - const int64_t index_value = static_cast<int64_t>(index.GetValue(first_row + row)); | 336 | + const int64_t index_value = static_cast<int64_t>(index.GetValue((first_row + row) * index_row_stride)); |
| 335 | - const int64_t source_offset = index_value * embedding_size; | 337 | + const int64_t source_offset = index_value * input_row_stride; |
| 336 | AscendC::DataCopy(y[row * embedding_size], x[source_offset], static_cast<uint32_t>(embedding_size)); | 338 | AscendC::DataCopy(y[row * embedding_size], x[source_offset], static_cast<uint32_t>(embedding_size)); |
| 337 | } | 339 | } |
| 338 | return true; | 340 | return true; |
| @@ -264,7 +264,7 @@ __simt_callee__ __aicore__ inline Y IndirectLoadSimtCompute(__gm__ X *x, Context | |||
| 264 | if constexpr (AddressPolicy::kUsesInputAxis) { | 264 | if constexpr (AddressPolicy::kUsesInputAxis) { |
| 265 | input_offset += indirect_index * address_policy.input_axis_stride; | 265 | input_offset += indirect_index * address_policy.input_axis_stride; |
| 266 | } | 266 | } |
| 267 | - return FusedBody::Output(x[input_offset], output_index, context); | 267 | + return FusedBody::Output(x[input_offset], output_index, address.index_offset, context); |
| 268 | } | 268 | } |
| 269 | 269 | ||
| 270 | template <typename X, typename Y, typename FusedBody, typename Context, uint32_t ThreadNum, typename AddressPolicy> | 270 | template <typename X, typename Y, typename FusedBody, typename Context, uint32_t ThreadNum, typename AddressPolicy> |
| @@ -22,11 +22,26 @@ | |||
| 22 | 22 | ||
| 23 | 23 | ||
| 24 | 24 | ||
| 25 | + | ||
| 25 | 26 | ||
| 26 | 27 | ||
| 27 | namespace codegen { | 28 | namespace codegen { |
| 28 | namespace { | 29 | namespace { |
| 30 | +constexpr size_t kIndirectLoadInputCount = 2UL; | ||
| 31 | +constexpr size_t kIndirectLoadOutputCount = 1UL; | ||
| 32 | +constexpr char kSimtContextNamePrefix[] = "IndirectLoadSimtContext_"; | ||
| 33 | +constexpr char kSimtBodyNamePrefix[] = "IndirectLoadSimtBody_"; | ||
| 34 | +constexpr char kGlobalTensorNamePrefix[] = "global_"; | ||
| 35 | +constexpr char kSimtGmFieldNamePrefix[] = "gm_"; | ||
| 36 | +constexpr char kSimtValueNamePrefix[] = "v_"; | ||
| 37 | + | ||
| 29 | struct LogicalTensorInfo { | 38 | struct LogicalTensorInfo { |
| 39 | + LogicalTensorInfo() = default; | ||
| 40 | + explicit LogicalTensorInfo(const ascgen_utils::indirect_load::LogicalTensorView &view) | ||
| 41 | + : sizes(view.sizes), strides(view.strides) {} | ||
| 42 | + LogicalTensorInfo(const std::vector<ascir::SizeExpr> &sizes_in, const std::vector<ascir::SizeExpr> &strides_in) | ||
| 43 | + : sizes(sizes_in), strides(strides_in) {} | ||
| 44 | + | ||
| 30 | std::vector<ascir::SizeExpr> sizes; | 45 | std::vector<ascir::SizeExpr> sizes; |
| 31 | std::vector<ascir::SizeExpr> strides; | 46 | std::vector<ascir::SizeExpr> strides; |
| 32 | }; | 47 | }; |
| @@ -82,29 +97,18 @@ bool IsAxisDerivedFrom(const TPipe &tpipe, ascir::AxisId axis_id, ascir::AxisId | |||
| 82 | return false; | 97 | return false; |
| 83 | } | 98 | } |
| 84 | 99 | ||
| 85 | -LogicalTensorInfo BuildLogicalTensorInfo(const ascgen_utils::indirect_load::LogicalTensorView &view, | ||
| 86 | - const TPipe &tpipe) { | ||
| 87 | - (void)tpipe; | ||
| 88 | - LogicalTensorInfo info; | ||
| 89 | - info.sizes = view.sizes; | ||
| 90 | - info.strides = view.strides; | ||
| 91 | - return info; | ||
| 92 | -} | ||
| 93 | - | ||
| 94 | af::Status BuildTensorWindowInfo(const ascgen_utils::indirect_load::IndirectLoadTensorLayout &layout, | 100 | af::Status BuildTensorWindowInfo(const ascgen_utils::indirect_load::IndirectLoadTensorLayout &layout, |
| 95 | - const Tensor &tensor, size_t axis_pos, const TPipe &tpipe, LogicalTensorInfo &info) { | 101 | + const Tensor &tensor, size_t axis_pos, LogicalTensorInfo &info) { |
| 96 | GE_ASSERT_TRUE(layout.axis_ids.size() == layout.sizes.size() && layout.sizes.size() == layout.strides.size(), | 102 | GE_ASSERT_TRUE(layout.axis_ids.size() == layout.sizes.size() && layout.sizes.size() == layout.strides.size(), |
| 97 | "IndirectLoad tensor window layout rank mismatch."); | 103 | "IndirectLoad tensor window layout rank mismatch."); |
| 98 | GE_ASSERT_TRUE(axis_pos < layout.sizes.size(), "IndirectLoad tensor window axis is out of range."); | 104 | GE_ASSERT_TRUE(axis_pos < layout.sizes.size(), "IndirectLoad tensor window axis is out of range."); |
| 99 | GE_ASSERT_TRUE(tensor.vectorized_axis.size() == tensor.vectorized_strides.size(), | 105 | GE_ASSERT_TRUE(tensor.vectorized_axis.size() == tensor.vectorized_strides.size(), |
| 100 | "IndirectLoad tensor vectorized axis/stride rank mismatch."); | 106 | "IndirectLoad tensor vectorized axis/stride rank mismatch."); |
| 101 | - info = BuildLogicalTensorInfo(layout, tpipe); | 107 | + info = LogicalTensorInfo(layout); |
| 102 | - // Dense and zero-stride-compact layouts keep their logical row-major/alias | 108 | + // Dense layouts keep their logical row-major strides after the local window is built. For a zero-stride-compact |
| 103 | - // strides after the local window is built. A merged vectorized axis may | 109 | + // view, preserve zero-stride axes but derive non-zero window strides from the physical vectorized tensor view; |
| 104 | - // expose a unit vectorized stride for every source axis; using it directly | 110 | + // bitwidth-changing producers can introduce padding between logical elements. |
| 105 | - // would collapse, for example, the [s6, s7] index axes to [1, 1]. | 111 | + if (layout.kind == ascgen_utils::indirect_load::IndirectLoadLayoutKind::kDense) { |
| 106 | - if (layout.kind == ascgen_utils::indirect_load::IndirectLoadLayoutKind::kDense || | ||
| 107 | - layout.kind == ascgen_utils::indirect_load::IndirectLoadLayoutKind::kZeroStrideCompact) { | ||
| 108 | return af::SUCCESS; | 112 | return af::SUCCESS; |
| 109 | } | 113 | } |
| 110 | af::Expression compact_stride = af::sym::kSymbolOne; | 114 | af::Expression compact_stride = af::sym::kSymbolOne; |
| @@ -130,64 +134,48 @@ af::Status BuildTensorWindowInfo(const ascgen_utils::indirect_load::IndirectLoad | |||
| 130 | return af::SUCCESS; | 134 | return af::SUCCESS; |
| 131 | } | 135 | } |
| 132 | 136 | ||
| 133 | -bool TryGetNonNegativeConst(const af::Expression &expr, uint64_t &value) { | 137 | +bool TryBuildStaticSpan(const af::Expression &size_expr, uint64_t stride, uint64_t &span) { |
| 134 | - int64_t signed_value = 0L; | 138 | + int64_t size = 0L; |
| 135 | - if (!expr.IsConstExpr() || !expr.GetConstValue(signed_value) || signed_value < 0L) { | 139 | + if (!size_expr.GetConstValue(size) || size < 0L) { |
| 136 | return false; | 140 | return false; |
| 137 | } | 141 | } |
| 138 | - value = static_cast<uint64_t>(signed_value); | 142 | + return !ge::MulOverflow(static_cast<uint64_t>(size), stride, span); |
| 139 | - return true; | ||
| 140 | } | 143 | } |
| 141 | 144 | ||
| 142 | -bool CheckedMul(uint64_t lhs, uint64_t rhs, uint64_t &result) { | 145 | +bool TryAccumulateStaticOffset(const af::Expression &size_expr, const af::Expression &stride_expr, uint64_t &offset) { |
| 143 | - if (lhs != 0U && rhs > std::numeric_limits<uint64_t>::max() / lhs) { | 146 | + int64_t size = 0L; |
| 147 | + int64_t stride = 0L; | ||
| 148 | + if (!size_expr.GetConstValue(size) || size <= 0L || !stride_expr.GetConstValue(stride) || stride < 0L) { | ||
| 144 | return false; | 149 | return false; |
| 145 | } | 150 | } |
| 146 | - result = lhs * rhs; | 151 | + uint64_t dim_offset = 0U; |
| 147 | - return true; | 152 | + return !ge::MulOverflow(static_cast<uint64_t>(size - 1L), static_cast<uint64_t>(stride), dim_offset) && |
| 148 | -} | 153 | + !ge::AddOverflow(offset, dim_offset, offset); |
| 149 | - | ||
| 150 | -bool CheckedAdd(uint64_t lhs, uint64_t rhs, uint64_t &result) { | ||
| 151 | - if (rhs > std::numeric_limits<uint64_t>::max() - lhs) { | ||
| 152 | - return false; | ||
| 153 | - } | ||
| 154 | - result = lhs + rhs; | ||
| 155 | - return true; | ||
| 156 | } | 154 | } |
| 157 | 155 | ||
| 158 | bool TryGetStaticSpans(const LogicalTensorInfo &input, const LogicalTensorInfo &output, size_t axis, | 156 | bool TryGetStaticSpans(const LogicalTensorInfo &input, const LogicalTensorInfo &output, size_t axis, |
| 159 | SimtCodegenPlan &plan) { | 157 | SimtCodegenPlan &plan) { |
| 160 | uint64_t inner = 1U; | 158 | uint64_t inner = 1U; |
| 161 | for (size_t i = axis + 1U; i < output.sizes.size(); ++i) { | 159 | for (size_t i = axis + 1U; i < output.sizes.size(); ++i) { |
| 162 | - uint64_t size = 0U; | 160 | + if (!TryBuildStaticSpan(output.sizes[i], inner, inner)) { |
| 163 | - if (!TryGetNonNegativeConst(output.sizes[i], size) || !CheckedMul(inner, size, inner)) { | ||
| 164 | return false; | 161 | return false; |
| 165 | } | 162 | } |
| 166 | } | 163 | } |
| 167 | - uint64_t output_axis_size = 0U; | 164 | + int64_t input_stride = 0L; |
| 168 | - uint64_t input_axis_size = 0U; | 165 | + if (!input.strides[axis].GetConstValue(input_stride) || input_stride < 0L || |
| 169 | - uint64_t input_stride = 0U; | 166 | + !TryBuildStaticSpan(output.sizes[axis], inner, plan.output_axis_span_value) || |
| 170 | - if (!TryGetNonNegativeConst(output.sizes[axis], output_axis_size) || | 167 | + !TryBuildStaticSpan(input.sizes[axis], static_cast<uint64_t>(input_stride), plan.input_axis_span_value)) { |
| 171 | - !TryGetNonNegativeConst(input.sizes[axis], input_axis_size) || | ||
| 172 | - !TryGetNonNegativeConst(input.strides[axis], input_stride) || | ||
| 173 | - !CheckedMul(output_axis_size, inner, plan.output_axis_span_value) || | ||
| 174 | - !CheckedMul(input_axis_size, input_stride, plan.input_axis_span_value)) { | ||
| 175 | return false; | 168 | return false; |
| 176 | } | 169 | } |
| 177 | plan.inner_span_value = inner; | 170 | plan.inner_span_value = inner; |
| 178 | - plan.input_axis_stride_value = input_stride; | 171 | + plan.input_axis_stride_value = static_cast<uint64_t>(input_stride); |
| 179 | return true; | 172 | return true; |
| 180 | } | 173 | } |
| 181 | 174 | ||
| 182 | bool TryGetMaxElementOffset(const LogicalTensorInfo &tensor, uint64_t &max_offset) { | 175 | bool TryGetMaxElementOffset(const LogicalTensorInfo &tensor, uint64_t &max_offset) { |
| 183 | max_offset = 0U; | 176 | max_offset = 0U; |
| 184 | for (size_t i = 0U; i < tensor.sizes.size(); ++i) { | 177 | for (size_t i = 0U; i < tensor.sizes.size(); ++i) { |
| 185 | - uint64_t size = 0U; | 178 | + if (!TryAccumulateStaticOffset(tensor.sizes[i], tensor.strides[i], max_offset)) { |
| 186 | - uint64_t stride = 0U; | ||
| 187 | - uint64_t dim_offset = 0U; | ||
| 188 | - if (!TryGetNonNegativeConst(tensor.sizes[i], size) || size == 0U || | ||
| 189 | - !TryGetNonNegativeConst(tensor.strides[i], stride) || !CheckedMul(size - 1U, stride, dim_offset) || | ||
| 190 | - !CheckedAdd(max_offset, dim_offset, max_offset)) { | ||
| 191 | return false; | 179 | return false; |
| 192 | } | 180 | } |
| 193 | } | 181 | } |
| @@ -237,8 +225,8 @@ bool CanUseUint32Offsets(const LogicalTensorInfo &input, const LogicalTensorInfo | |||
| 237 | 225 | ||
| 238 | bool CanUseUint32Divisors(const LogicalTensorInfo &index) { | 226 | bool CanUseUint32Divisors(const LogicalTensorInfo &index) { |
| 239 | for (const af::Expression &size_expr : index.sizes) { | 227 | for (const af::Expression &size_expr : index.sizes) { |
| 240 | - uint64_t size = 0U; | 228 | + int64_t size = 0L; |
| 241 | - if (!TryGetNonNegativeConst(size_expr, size) || size > static_cast<uint64_t>(INT32_MAX)) { | 229 | + if (!size_expr.GetConstValue(size) || size < 0L || size > static_cast<int64_t>(INT32_MAX)) { |
| 242 | return false; | 230 | return false; |
| 243 | } | 231 | } |
| 244 | } | 232 | } |
| @@ -421,32 +409,25 @@ af::Status CheckIndirectLoadShape(const ascgen_utils::indirect_load::TemplateLog | |||
| 421 | const auto &input = logical_view.input; | 409 | const auto &input = logical_view.input; |
| 422 | const auto &index = logical_view.index; | 410 | const auto &index = logical_view.index; |
| 423 | const auto &output = logical_view.output; | 411 | const auto &output = logical_view.output; |
| 424 | - GE_ASSERT_TRUE(input.sizes.size() == index.sizes.size(), | 412 | + GE_ASSERT_TRUE(input.sizes.size() == index.sizes.size(), "Invalid IndirectLoad logical rank, input:%zu, index:%zu.", |
| 425 | - "IndirectLoad expects input and index with the same logical rank."); | 413 | + input.sizes.size(), index.sizes.size()); |
| 426 | - GE_ASSERT_TRUE(index.sizes.size() == output.sizes.size(), | 414 | + GE_ASSERT_TRUE(index.sizes.size() == output.sizes.size(), "Invalid IndirectLoad logical rank, index:%zu, output:%zu.", |
| 427 | - "IndirectLoad index and output must have the same logical rank."); | 415 | + index.sizes.size(), output.sizes.size()); |
| 428 | GE_ASSERT_TRUE(input.kind != ascgen_utils::indirect_load::IndirectLoadLayoutKind::kUnsupported && | 416 | GE_ASSERT_TRUE(input.kind != ascgen_utils::indirect_load::IndirectLoadLayoutKind::kUnsupported && |
| 429 | index.kind != ascgen_utils::indirect_load::IndirectLoadLayoutKind::kUnsupported, | 417 | index.kind != ascgen_utils::indirect_load::IndirectLoadLayoutKind::kUnsupported, |
| 430 | "IndirectLoad input or index layout is unsupported."); | 418 | "IndirectLoad input or index layout is unsupported."); |
| 431 | GE_ASSERT_TRUE(input.sizes.size() == input.strides.size() && index.sizes.size() == index.strides.size(), | 419 | GE_ASSERT_TRUE(input.sizes.size() == input.strides.size() && index.sizes.size() == index.strides.size(), |
| 432 | "IndirectLoad logical sizes/strides rank mismatch."); | 420 | "IndirectLoad logical sizes/strides rank mismatch."); |
| 433 | - GE_ASSERT_SUCCESS(CheckDenseStrides({output.sizes, output.strides}, "output logical view")); | 421 | + GE_ASSERT_SUCCESS(CheckDenseStrides(LogicalTensorInfo(output.sizes, output.strides), "output logical view")); |
| 434 | GE_ASSERT_TRUE(output_tensor.axis_size.size() == output_tensor.axis_strides.size(), | 422 | GE_ASSERT_TRUE(output_tensor.axis_size.size() == output_tensor.axis_strides.size(), |
| 435 | "IndirectLoad output tensor sizes/strides rank mismatch."); | 423 | "IndirectLoad output tensor sizes/strides rank mismatch."); |
| 436 | - GE_ASSERT_SUCCESS(CheckDenseStrides({output_tensor.axis_size, output_tensor.axis_strides}, "output tensor")); | 424 | + GE_ASSERT_SUCCESS( |
| 425 | + CheckDenseStrides(LogicalTensorInfo(output_tensor.axis_size, output_tensor.axis_strides), "output tensor")); | ||
| 437 | return af::SUCCESS; | 426 | return af::SUCCESS; |
| 438 | } | 427 | } |
| 439 | 428 | ||
| 440 | using SimtNodeSet = std::unordered_set<const af::AscNode *>; | 429 | using SimtNodeSet = std::unordered_set<const af::AscNode *>; |
| 441 | 430 | ||
| 442 | -bool IsSimtGmInput(const af::AscNodePtr &node) { | ||
| 443 | - return af::ops::IsOps<af::ascir_op::Load>(node); | ||
| 444 | -} | ||
| 445 | - | ||
| 446 | -bool IsSimtScalarInput(const af::AscNodePtr &node) { | ||
| 447 | - return af::ops::IsOps<af::ascir_op::Scalar>(node); | ||
| 448 | -} | ||
| 449 | - | ||
| 450 | af::Status CollectSimtBackwardNodes(const af::AscNodePtr &root, const ascir::NodeView &indirect_load, | 431 | af::Status CollectSimtBackwardNodes(const af::AscNodePtr &root, const ascir::NodeView &indirect_load, |
| 451 | SimtNodeSet &nodes) { | 432 | SimtNodeSet &nodes) { |
| 452 | std::vector<af::AscNodePtr> pending = {root}; | 433 | std::vector<af::AscNodePtr> pending = {root}; |
| @@ -455,7 +436,7 @@ af::Status CollectSimtBackwardNodes(const af::AscNodePtr &root, const ascir::Nod | |||
| 455 | if (current == nullptr || current == indirect_load || !nodes.emplace(current.get()).second) { | 436 | if (current == nullptr || current == indirect_load || !nodes.emplace(current.get()).second) { |
| 456 | continue; | 437 | continue; |
| 457 | } | 438 | } |
| 458 | - if (IsSimtGmInput(current) || IsSimtScalarInput(current)) { | 439 | + if (af::ops::IsOps<af::ascir_op::Load>(current) || af::ops::IsOps<af::ascir_op::Scalar>(current)) { |
| 459 | continue; | 440 | continue; |
| 460 | } | 441 | } |
| 461 | GE_ASSERT_TRUE(current->inputs.Size() > 0UL, "IndirectLoad SIMT node[%s] has no input.", current->GetNamePtr()); | 442 | GE_ASSERT_TRUE(current->inputs.Size() > 0UL, "IndirectLoad SIMT node[%s] has no input.", current->GetNamePtr()); |
| @@ -479,10 +460,12 @@ af::AscNodePtr FindSimtOutputStore(const ascir::NodeView &indirect_load) { | |||
| 479 | } | 460 | } |
| 480 | 461 | ||
| 481 | af::Status ValidateSimtRegionNode(const af::AscNodePtr &node) { | 462 | af::Status ValidateSimtRegionNode(const af::AscNodePtr &node) { |
| 482 | - if (IsSimtGmInput(node) || IsSimtScalarInput(node) || af::ops::IsOps<af::ascir_op::Store>(node)) { | 463 | + if (af::ops::IsOps<af::ascir_op::Load>(node) || af::ops::IsOps<af::ascir_op::Scalar>(node) || |
| 464 | + af::ops::IsOps<af::ascir_op::Store>(node)) { | ||
| 483 | return af::SUCCESS; | 465 | return af::SUCCESS; |
| 484 | } | 466 | } |
| 485 | - GE_ASSERT_TRUE(ascgen_utils::indirect_load::IsSimtInlineTransform(node), | 467 | + GE_ASSERT_TRUE(ascgen_utils::indirect_load::GetTemplateRole(node) == |
| 468 | + ascgen_utils::indirect_load::TemplateRole::kSimtInlineTransform, | ||
| 486 | "IndirectLoad SIMT node[%s] has no inline-transform role.", node->GetNamePtr()); | 469 | "IndirectLoad SIMT node[%s] has no inline-transform role.", node->GetNamePtr()); |
| 487 | GE_ASSERT_TRUE(!af::ops::IsOps<af::ascir_op::VectorFunc>(node), | 470 | GE_ASSERT_TRUE(!af::ops::IsOps<af::ascir_op::VectorFunc>(node), |
| 488 | "IndirectLoad SIMT transform must use scalar emission, node:%s", node->GetNamePtr()); | 471 | "IndirectLoad SIMT transform must use scalar emission, node:%s", node->GetNamePtr()); |
| @@ -524,7 +507,8 @@ af::Status CollectSimtRegionMetadata(const ascir::NodeView &indirect_load, std:: | |||
| 524 | GE_ASSERT_SUCCESS(ValidateSimtRegionNode(node)); | 507 | GE_ASSERT_SUCCESS(ValidateSimtRegionNode(node)); |
| 525 | output_nodes.emplace_back(node); | 508 | output_nodes.emplace_back(node); |
| 526 | } | 509 | } |
| 527 | - if (IsSimtGmInput(node) && (index_set.count(node.get()) != 0UL || output_set.count(node.get()) != 0UL)) { | 510 | + if (af::ops::IsOps<af::ascir_op::Load>(node) && |
| 511 | + (index_set.count(node.get()) != 0UL || output_set.count(node.get()) != 0UL)) { | ||
| 528 | AppendSimtGmTensor(node, gm_tensors); | 512 | AppendSimtGmTensor(node, gm_tensors); |
| 529 | } | 513 | } |
| 530 | } | 514 | } |
| @@ -545,18 +529,13 @@ af::Status CollectSimtMetadata(const ascir::NodeView &indirect_load, const af::A | |||
| 545 | } | 529 | } |
| 546 | GE_ASSERT_SUCCESS(ValidateSimtRegionNode(node)); | 530 | GE_ASSERT_SUCCESS(ValidateSimtRegionNode(node)); |
| 547 | nodes.emplace_back(node); | 531 | nodes.emplace_back(node); |
| 548 | - if (IsSimtGmInput(node)) { | 532 | + if (af::ops::IsOps<af::ascir_op::Load>(node)) { |
| 549 | AppendSimtGmTensor(node, gm_tensors); | 533 | AppendSimtGmTensor(node, gm_tensors); |
| 550 | } | 534 | } |
| 551 | } | 535 | } |
| 552 | return af::SUCCESS; | 536 | return af::SUCCESS; |
| 553 | } | 537 | } |
| 554 | 538 | ||
| 555 | -void RegisterSimtGmInput(const af::AscNodePtr &node, std::map<ascir::TensorId, std::string> &values) { | ||
| 556 | - const auto output = node->outputs()[0]; | ||
| 557 | - values[output->attr.mem.tensor_id] = "context.gm_" + std::to_string(output->attr.mem.tensor_id) + "[output_index]"; | ||
| 558 | -} | ||
| 559 | - | ||
| 560 | af::Status EmitSimtScalarInput(const af::AscNodePtr &node, std::map<ascir::TensorId, std::string> &values, | 539 | af::Status EmitSimtScalarInput(const af::AscNodePtr &node, std::map<ascir::TensorId, std::string> &values, |
| 561 | std::stringstream &ss) { | 540 | std::stringstream &ss) { |
| 562 | std::string value; | 541 | std::string value; |
| @@ -567,7 +546,7 @@ af::Status EmitSimtScalarInput(const af::AscNodePtr &node, std::map<ascir::Tenso | |||
| 567 | GE_ASSERT_SUCCESS(Tensor::DtypeName(output->attr.dtype, dtype)); | 546 | GE_ASSERT_SUCCESS(Tensor::DtypeName(output->attr.dtype, dtype)); |
| 568 | std::string processed_value; | 547 | std::string processed_value; |
| 569 | GE_ASSERT_SUCCESS(ascgen_utils::ScalarValuePreProcess(value, dtype, processed_value)); | 548 | GE_ASSERT_SUCCESS(ascgen_utils::ScalarValuePreProcess(value, dtype, processed_value)); |
| 570 | - const std::string variable = "v_" + std::to_string(output->attr.mem.tensor_id); | 549 | + const std::string variable = kSimtValueNamePrefix + std::to_string(output->attr.mem.tensor_id); |
| 571 | ss << " " << dtype << " " << variable << " = static_cast<" << dtype << ">(" << processed_value << ");" | 550 | ss << " " << dtype << " " << variable << " = static_cast<" << dtype << ">(" << processed_value << ");" |
| 572 | << std::endl; | 551 | << std::endl; |
| 573 | values[output->attr.mem.tensor_id] = variable; | 552 | values[output->attr.mem.tensor_id] = variable; |
| @@ -587,7 +566,7 @@ af::Status EmitSimtTransform(const af::AscNodePtr &node, std::map<ascir::TensorI | |||
| 587 | const auto output = node->outputs()[0]; | 566 | const auto output = node->outputs()[0]; |
| 588 | std::string output_dtype; | 567 | std::string output_dtype; |
| 589 | GE_ASSERT_SUCCESS(Tensor::DtypeName(output->attr.dtype, output_dtype)); | 568 | GE_ASSERT_SUCCESS(Tensor::DtypeName(output->attr.dtype, output_dtype)); |
| 590 | - const std::string variable = "v_" + std::to_string(output->attr.mem.tensor_id); | 569 | + const std::string variable = kSimtValueNamePrefix + std::to_string(output->attr.mem.tensor_id); |
| 591 | ss << " " << output_dtype << " " << variable << " = " << expr << ";" << std::endl; | 570 | ss << " " << output_dtype << " " << variable << " = " << expr << ";" << std::endl; |
| 592 | values[output->attr.mem.tensor_id] = variable; | 571 | values[output->attr.mem.tensor_id] = variable; |
| 593 | return af::SUCCESS; | 572 | return af::SUCCESS; |
| @@ -595,13 +574,17 @@ af::Status EmitSimtTransform(const af::AscNodePtr &node, std::map<ascir::TensorI | |||
| 595 | 574 | ||
| 596 | af::Status GenerateSimtEvaluatorBody(const std::vector<af::AscNodePtr> &nodes, | 575 | af::Status GenerateSimtEvaluatorBody(const std::vector<af::AscNodePtr> &nodes, |
| 597 | std::map<ascir::TensorId, std::string> &values, ascir::TensorId result_tensor_id, | 576 | std::map<ascir::TensorId, std::string> &values, ascir::TensorId result_tensor_id, |
| 598 | - std::stringstream &ss) { | 577 | + std::stringstream &ss, const SimtNodeSet *index_nodes = nullptr) { |
| 599 | for (const af::AscNodePtr &node : nodes) { | 578 | for (const af::AscNodePtr &node : nodes) { |
| 600 | - if (IsSimtGmInput(node)) { | 579 | + if (af::ops::IsOps<af::ascir_op::Load>(node)) { |
| 601 | - RegisterSimtGmInput(node, values); | 580 | + const auto output = node->outputs()[0]; |
| 581 | + const bool is_index_node = index_nodes != nullptr && index_nodes->count(node.get()) != 0UL; | ||
| 582 | + const char *offset = is_index_node ? "index_offset" : "output_index"; | ||
| 583 | + values[output->attr.mem.tensor_id] = "context." + std::string(kSimtGmFieldNamePrefix) + | ||
| 584 | + std::to_string(output->attr.mem.tensor_id) + "[" + offset + "]"; | ||
| 602 | continue; | 585 | continue; |
| 603 | } | 586 | } |
| 604 | - if (IsSimtScalarInput(node)) { | 587 | + if (af::ops::IsOps<af::ascir_op::Scalar>(node)) { |
| 605 | GE_ASSERT_SUCCESS(EmitSimtScalarInput(node, values, ss)); | 588 | GE_ASSERT_SUCCESS(EmitSimtScalarInput(node, values, ss)); |
| 606 | continue; | 589 | continue; |
| 607 | } | 590 | } |
| @@ -617,23 +600,23 @@ af::Status GenerateSimtEvaluatorBody(const std::vector<af::AscNodePtr> &nodes, | |||
| 617 | return af::SUCCESS; | 600 | return af::SUCCESS; |
| 618 | } | 601 | } |
| 619 | 602 | ||
| 620 | -af::Status GenerateSimtIndexEvaluator(const std::string &index_dtype, const std::string &offset_type, | 603 | +af::Status GenSimtIndexEvaluator(const std::string &index_dtype, const std::string &offset_type, |
| 621 | - ascir::TensorId result_tensor_id, const std::vector<af::AscNodePtr> &nodes, | 604 | + ascir::TensorId result_tensor_id, const std::vector<af::AscNodePtr> &nodes, |
| 622 | - std::stringstream &ss) { | 605 | + std::stringstream &ss) { |
| 623 | std::map<ascir::TensorId, std::string> values; | 606 | std::map<ascir::TensorId, std::string> values; |
| 624 | ss << " __simt_callee__ __aicore__ inline static " << index_dtype << " Index(" << offset_type | 607 | ss << " __simt_callee__ __aicore__ inline static " << index_dtype << " Index(" << offset_type |
| 625 | << " output_index, const Context &context) {" << std::endl; | 608 | << " output_index, const Context &context) {" << std::endl; |
| 626 | return GenerateSimtEvaluatorBody(nodes, values, result_tensor_id, ss); | 609 | return GenerateSimtEvaluatorBody(nodes, values, result_tensor_id, ss); |
| 627 | } | 610 | } |
| 628 | 611 | ||
| 629 | -af::Status GenerateSimtOutputEvaluator(const std::string &output_dtype, const std::string &input_dtype, | 612 | +af::Status GenSimtOutputEvaluator(const std::string &output_dtype, const std::string &input_dtype, |
| 630 | - const std::string &offset_type, ascir::TensorId value_tensor_id, | 613 | + const std::string &offset_type, ascir::TensorId value_tensor_id, |
| 631 | - ascir::TensorId result_tensor_id, const std::vector<af::AscNodePtr> &nodes, | 614 | + ascir::TensorId result_tensor_id, const std::vector<af::AscNodePtr> &nodes, |
| 632 | - std::stringstream &ss) { | 615 | + const SimtNodeSet &index_nodes, std::stringstream &ss) { |
| 633 | std::map<ascir::TensorId, std::string> values{{value_tensor_id, "value"}}; | 616 | std::map<ascir::TensorId, std::string> values{{value_tensor_id, "value"}}; |
| 634 | ss << " __simt_callee__ __aicore__ inline static " << output_dtype << " Output(" << input_dtype << " value, " | 617 | ss << " __simt_callee__ __aicore__ inline static " << output_dtype << " Output(" << input_dtype << " value, " |
| 635 | - << offset_type << " output_index, const Context &context) {" << std::endl; | 618 | + << offset_type << " output_index, " << offset_type << " index_offset, const Context &context) {" << std::endl; |
| 636 | - return GenerateSimtEvaluatorBody(nodes, values, result_tensor_id, ss); | 619 | + return GenerateSimtEvaluatorBody(nodes, values, result_tensor_id, ss, &index_nodes); |
| 637 | } | 620 | } |
| 638 | 621 | ||
| 639 | af::Status CalcVectorizedElementCount(const Tensor &tensor, af::Expression &element_count) { | 622 | af::Status CalcVectorizedElementCount(const Tensor &tensor, af::Expression &element_count) { |
| @@ -652,7 +635,8 @@ af::Status GenerateSimtContextInitializer(const std::string &context_name, const | |||
| 652 | const SimtGmTensor &gm_tensor = gm_tensors[i]; | 635 | const SimtGmTensor &gm_tensor = gm_tensors[i]; |
| 653 | std::string dtype; | 636 | std::string dtype; |
| 654 | GE_ASSERT_SUCCESS(Tensor::DtypeName(gm_tensor.dtype, dtype)); | 637 | GE_ASSERT_SUCCESS(Tensor::DtypeName(gm_tensor.dtype, dtype)); |
| 655 | - ss << (i == 0UL ? "" : ", ") << "(__gm__ " << dtype << " *)global_" << gm_tensor.gm_tensor_id << ".GetPhyAddr()"; | 638 | + ss << (i == 0UL ? "" : ", ") << "(__gm__ " << dtype << " *)" << kGlobalTensorNamePrefix << gm_tensor.gm_tensor_id |
| 639 | + << ".GetPhyAddr()"; | ||
| 656 | } | 640 | } |
| 657 | ss << "};" << std::endl; | 641 | ss << "};" << std::endl; |
| 658 | return af::SUCCESS; | 642 | return af::SUCCESS; |
| @@ -688,8 +672,10 @@ Status IndirectLoadRegApiCall::ParseAttr(const ascir::NodeView &node) { | |||
| 688 | } | 672 | } |
| 689 | 673 | ||
| 690 | Status IndirectLoadRegApiCall::ParseSimtAttr(const ascir::NodeView &node) { | 674 | Status IndirectLoadRegApiCall::ParseSimtAttr(const ascir::NodeView &node) { |
| 691 | - GE_ASSERT_TRUE(node->inputs.Size() == 2UL && node->outputs().size() == 1UL, | 675 | + GE_ASSERT_TRUE(node->inputs.Size() == kIndirectLoadInputCount, "Invalid IndirectLoad SIMT input number:%zu.", |
| 692 | - "IndirectLoad SIMT expects 2 inputs and 1 output."); | 676 | + node->inputs.Size()); |
| 677 | + GE_ASSERT_TRUE(node->outputs().size() == kIndirectLoadOutputCount, "Invalid IndirectLoad SIMT output number:%zu.", | ||
| 678 | + node->outputs().size()); | ||
| 693 | GE_ASSERT_TRUE(logical_view_.input.sizes.size() < 64UL, "IndirectLoad SIMT rank must be smaller than 64."); | 679 | GE_ASSERT_TRUE(logical_view_.input.sizes.size() < 64UL, "IndirectLoad SIMT rank must be smaller than 64."); |
| 694 | const auto node_inputs = node->inputs(); | 680 | const auto node_inputs = node->inputs(); |
| 695 | index_result_tensor_id_ = node_inputs[ascgen_utils::indirect_load::kIndexTensorIndex]->attr.mem.tensor_id; | 681 | index_result_tensor_id_ = node_inputs[ascgen_utils::indirect_load::kIndexTensorIndex]->attr.mem.tensor_id; |
| @@ -705,16 +691,19 @@ Status IndirectLoadRegApiCall::ParseSimtAttr(const ascir::NodeView &node) { | |||
| 705 | ascgen_utils::indirect_load::GetInputProducer(node, ascgen_utils::indirect_load::kIndexTensorIndex); | 691 | ascgen_utils::indirect_load::GetInputProducer(node, ascgen_utils::indirect_load::kIndexTensorIndex); |
| 706 | GE_ASSERT_SUCCESS(CollectSimtMetadata(node, index_root, index_nodes_, simt_gm_tensors_)); | 692 | GE_ASSERT_SUCCESS(CollectSimtMetadata(node, index_root, index_nodes_, simt_gm_tensors_)); |
| 707 | GE_ASSERT_SUCCESS(CollectSimtMetadata(node, output_root, output_nodes_, simt_gm_tensors_)); | 693 | GE_ASSERT_SUCCESS(CollectSimtMetadata(node, output_root, output_nodes_, simt_gm_tensors_)); |
| 708 | - GE_ASSERT_TRUE(outputs.size() == 1UL, "IndirectLoad SIMT expects one output."); | 694 | + GE_ASSERT_TRUE(outputs.size() == kIndirectLoadOutputCount, "Invalid IndirectLoad SIMT output number:%zu.", |
| 695 | + outputs.size()); | ||
| 709 | outputs[0].id = output_result_tensor_id_; | 696 | outputs[0].id = output_result_tensor_id_; |
| 710 | return af::SUCCESS; | 697 | return af::SUCCESS; |
| 711 | } | 698 | } |
| 712 | af::AscNodePtr store; | 699 | af::AscNodePtr store; |
| 713 | GE_ASSERT_SUCCESS(CollectSimtRegionMetadata(node, index_nodes_, output_nodes_, simt_gm_tensors_, store)); | 700 | GE_ASSERT_SUCCESS(CollectSimtRegionMetadata(node, index_nodes_, output_nodes_, simt_gm_tensors_, store)); |
| 714 | - GE_ASSERT_TRUE(store->inputs.Size() == 1UL && store->outputs().size() == 1UL, | 701 | + GE_ASSERT_TRUE(store->inputs.Size() == 1UL, "Invalid IndirectLoad SIMT Store input number:%zu.", |
| 715 | - "IndirectLoad SIMT Store expects 1 input and 1 output."); | 702 | + store->inputs.Size()); |
| 703 | + GE_ASSERT_TRUE(store->outputs().size() == 1UL, "Invalid IndirectLoad SIMT Store output number:%zu.", | ||
| 704 | + store->outputs().size()); | ||
| 716 | output_result_tensor_id_ = store->inputs()[0]->attr.mem.tensor_id; | 705 | output_result_tensor_id_ = store->inputs()[0]->attr.mem.tensor_id; |
| 717 | - output_gm_tensor_ = "global_" + std::to_string(store->outputs()[0]->attr.mem.tensor_id); | 706 | + output_gm_tensor_ = kGlobalTensorNamePrefix + std::to_string(store->outputs()[0]->attr.mem.tensor_id); |
| 718 | output_dtype_ = store->outputs()[0]->attr.dtype; | 707 | output_dtype_ = store->outputs()[0]->attr.dtype; |
| 719 | GE_ASSERT_TRUE(store->inputs()[0]->attr.dtype == output_dtype_, | 708 | GE_ASSERT_TRUE(store->inputs()[0]->attr.dtype == output_dtype_, |
| 720 | "IndirectLoad SIMT output transform dtype[%d] does not match Store dtype[%d].", | 709 | "IndirectLoad SIMT output transform dtype[%d] does not match Store dtype[%d].", |
| @@ -729,13 +718,13 @@ Status IndirectLoadRegApiCall::GenerateFuncDefinition(const TPipe &tpipe, const | |||
| 729 | return af::SUCCESS; | 718 | return af::SUCCESS; |
| 730 | } | 719 | } |
| 731 | 720 | ||
| 732 | - GE_ASSERT_TRUE(inputs.size() == 2U, "IndirectLoad SIMT expects 2 inputs."); | 721 | + GE_ASSERT_TRUE(inputs.size() == kIndirectLoadInputCount, "Invalid IndirectLoad SIMT input number:%zu.", |
| 722 | + inputs.size()); | ||
| 733 | const Tensor *input_tensor = tpipe.GetTensor(inputs[ascgen_utils::indirect_load::kInputTensorIndex]->id); | 723 | const Tensor *input_tensor = tpipe.GetTensor(inputs[ascgen_utils::indirect_load::kInputTensorIndex]->id); |
| 734 | GE_ASSERT_NOTNULL(input_tensor, "IndirectLoad SIMT input tensor is missing."); | 724 | GE_ASSERT_NOTNULL(input_tensor, "IndirectLoad SIMT input tensor is missing."); |
| 735 | - const Tensor &input = *input_tensor; | 725 | + const LogicalTensorInfo input_info(logical_view_.input); |
| 736 | - const LogicalTensorInfo input_info = BuildLogicalTensorInfo(logical_view_.input, tpipe); | 726 | + LogicalTensorInfo index_info(logical_view_.index); |
| 737 | - LogicalTensorInfo index_info = BuildLogicalTensorInfo(logical_view_.index, tpipe); | 727 | + const LogicalTensorInfo output_info(logical_view_.output); |
| 738 | - const LogicalTensorInfo output_info = BuildLogicalTensorInfo(logical_view_.output, tpipe); | ||
| 739 | const bool index_broadcast_strided = ApplySimtIndexPhysicalStrides(index_nodes_, index_info); | 728 | const bool index_broadcast_strided = ApplySimtIndexPhysicalStrides(index_nodes_, index_info); |
| 740 | const bool strided = logical_view_.input.kind != ascgen_utils::indirect_load::IndirectLoadLayoutKind::kDense || | 729 | const bool strided = logical_view_.input.kind != ascgen_utils::indirect_load::IndirectLoadLayoutKind::kDense || |
| 741 | logical_view_.index.kind != ascgen_utils::indirect_load::IndirectLoadLayoutKind::kDense || | 730 | logical_view_.index.kind != ascgen_utils::indirect_load::IndirectLoadLayoutKind::kDense || |
| @@ -743,10 +732,10 @@ Status IndirectLoadRegApiCall::GenerateFuncDefinition(const TPipe &tpipe, const | |||
| 743 | const SimtCodegenPlan plan = | 732 | const SimtCodegenPlan plan = |
| 744 | BuildSimtCodegenPlan(input_info, index_info, output_info, static_cast<size_t>(axis_), strided); | 733 | BuildSimtCodegenPlan(input_info, index_info, output_info, static_cast<size_t>(axis_), strided); |
| 745 | std::string input_dtype; | 734 | std::string input_dtype; |
| 746 | - GE_ASSERT_SUCCESS(Tensor::DtypeName(input.dtype, input_dtype)); | 735 | + GE_ASSERT_SUCCESS(Tensor::DtypeName(input_tensor->dtype, input_dtype)); |
| 747 | const std::string valid_node_name = ascgen_utils::GenValidName(node_name); | 736 | const std::string valid_node_name = ascgen_utils::GenValidName(node_name); |
| 748 | - const std::string context_name = "IndirectLoadSimtContext_" + valid_node_name; | 737 | + const std::string context_name = kSimtContextNamePrefix + valid_node_name; |
| 749 | - const std::string body_name = "IndirectLoadSimtBody_" + valid_node_name; | 738 | + const std::string body_name = kSimtBodyNamePrefix + valid_node_name; |
| 750 | std::string index_dtype; | 739 | std::string index_dtype; |
| 751 | std::string output_dtype; | 740 | std::string output_dtype; |
| 752 | GE_ASSERT_SUCCESS(Tensor::DtypeName(index_dtype_, index_dtype)); | 741 | GE_ASSERT_SUCCESS(Tensor::DtypeName(index_dtype_, index_dtype)); |
| @@ -761,15 +750,18 @@ Status IndirectLoadRegApiCall::GenerateFuncDefinition(const TPipe &tpipe, const | |||
| 761 | for (const SimtGmTensor &tensor : simt_gm_tensors_) { | 750 | for (const SimtGmTensor &tensor : simt_gm_tensors_) { |
| 762 | std::string dtype; | 751 | std::string dtype; |
| 763 | GE_ASSERT_SUCCESS(Tensor::DtypeName(tensor.dtype, dtype)); | 752 | GE_ASSERT_SUCCESS(Tensor::DtypeName(tensor.dtype, dtype)); |
| 764 | - ss << " __gm__ " << dtype << " *gm_" << tensor.value_tensor_id << ";" << std::endl; | 753 | + ss << " __gm__ " << dtype << " *" << kSimtGmFieldNamePrefix << tensor.value_tensor_id << ";" << std::endl; |
| 765 | } | 754 | } |
| 766 | ss << "};" << std::endl; | 755 | ss << "};" << std::endl; |
| 767 | ss << "struct " << body_name << " {" << std::endl; | 756 | ss << "struct " << body_name << " {" << std::endl; |
| 768 | ss << " using Context = " << context_name << ";" << std::endl; | 757 | ss << " using Context = " << context_name << ";" << std::endl; |
| 769 | - GE_ASSERT_SUCCESS( | 758 | + SimtNodeSet index_node_set; |
| 770 | - GenerateSimtIndexEvaluator(index_dtype, plan.offset_type, index_result_tensor_id_, index_nodes_, ss)); | 759 | + for (const af::AscNodePtr &node : index_nodes_) { |
| 771 | - GE_ASSERT_SUCCESS(GenerateSimtOutputEvaluator(output_dtype, input_dtype, plan.offset_type, simt_value_tensor_id_, | 760 | + index_node_set.insert(node.get()); |
| 772 | - output_result_tensor_id_, output_nodes_, ss)); | 761 | + } |
| 762 | + GE_ASSERT_SUCCESS(GenSimtIndexEvaluator(index_dtype, plan.offset_type, index_result_tensor_id_, index_nodes_, ss)); | ||
| 763 | + GE_ASSERT_SUCCESS(GenSimtOutputEvaluator(output_dtype, input_dtype, plan.offset_type, simt_value_tensor_id_, | ||
| 764 | + output_result_tensor_id_, output_nodes_, index_node_set, ss)); | ||
| 773 | ss << "};" << std::endl; | 765 | ss << "};" << std::endl; |
| 774 | return af::SUCCESS; | 766 | return af::SUCCESS; |
| 775 | } | 767 | } |
| @@ -778,7 +770,8 @@ Status IndirectLoadRegApiCall::Generate(const TPipe &tpipe, const std::vector<as | |||
| 778 | const std::vector<std::reference_wrapper<const Tensor>> &inputs, | 770 | const std::vector<std::reference_wrapper<const Tensor>> &inputs, |
| 779 | const std::vector<std::reference_wrapper<const Tensor>> &outputs, | 771 | const std::vector<std::reference_wrapper<const Tensor>> &outputs, |
| 780 | std::string &result) const { | 772 | std::string &result) const { |
| 781 | - GE_ASSERT_TRUE(inputs.size() == 2U && outputs.size() == 1U, "IndirectLoad expects 2 inputs and 1 output."); | 773 | + GE_ASSERT_TRUE(inputs.size() == kIndirectLoadInputCount, "Invalid IndirectLoad input number:%zu.", inputs.size()); |
| 774 | + GE_ASSERT_TRUE(outputs.size() == kIndirectLoadOutputCount, "Invalid IndirectLoad output number:%zu.", outputs.size()); | ||
| 782 | GE_ASSERT_TRUE( | 775 | GE_ASSERT_TRUE( |
| 783 | template_id_ == ascir::TemplateId::kIndirectLoadSK || template_id_ == ascir::TemplateId::kIndirectLoadSimd, | 776 | template_id_ == ascir::TemplateId::kIndirectLoadSK || template_id_ == ascir::TemplateId::kIndirectLoadSimd, |
| 784 | "IndirectLoad tensor-based Generate only supports SK and SIMD."); | 777 | "IndirectLoad tensor-based Generate only supports SK and SIMD."); |
| @@ -800,7 +793,8 @@ Status IndirectLoadRegApiCall::Generate(const TPipe &tpipe, const std::vector<as | |||
| 800 | if (template_id_ != ascir::TemplateId::kIndirectLoadSimt) { | 793 | if (template_id_ != ascir::TemplateId::kIndirectLoadSimt) { |
| 801 | return ApiCall::Generate(tpipe, current_axis, result); | 794 | return ApiCall::Generate(tpipe, current_axis, result); |
| 802 | } | 795 | } |
| 803 | - GE_ASSERT_TRUE(inputs.size() == 2U, "IndirectLoad SIMT expects 2 inputs."); | 796 | + GE_ASSERT_TRUE(inputs.size() == kIndirectLoadInputCount, "Invalid IndirectLoad SIMT input number:%zu.", |
| 797 | + inputs.size()); | ||
| 804 | const Tensor *input_tensor = tpipe.GetTensor(inputs[ascgen_utils::indirect_load::kInputTensorIndex]->id); | 798 | const Tensor *input_tensor = tpipe.GetTensor(inputs[ascgen_utils::indirect_load::kInputTensorIndex]->id); |
| 805 | GE_ASSERT_NOTNULL(input_tensor, "IndirectLoad SIMT input tensor is missing."); | 799 | GE_ASSERT_NOTNULL(input_tensor, "IndirectLoad SIMT input tensor is missing."); |
| 806 | return GenerateSimt(tpipe, current_axis, *input_tensor, result); | 800 | return GenerateSimt(tpipe, current_axis, *input_tensor, result); |
| @@ -817,9 +811,9 @@ Status IndirectLoadRegApiCall::GenerateSk(const TPipe &tpipe, const std::vector< | |||
| 817 | GE_ASSERT_TRUE(tmp_iter != tmp_buf_id.end(), "IndirectLoad SK requires an API-level tmp buffer."); | 811 | GE_ASSERT_TRUE(tmp_iter != tmp_buf_id.end(), "IndirectLoad SK requires an API-level tmp buffer."); |
| 818 | 812 | ||
| 819 | const size_t axis_pos = static_cast<size_t>(axis_); | 813 | const size_t axis_pos = static_cast<size_t>(axis_); |
| 820 | - const LogicalTensorInfo x_info = BuildLogicalTensorInfo(logical_view_.input, tpipe); | 814 | + const LogicalTensorInfo x_info(logical_view_.input); |
| 821 | LogicalTensorInfo index_info; | 815 | LogicalTensorInfo index_info; |
| 822 | - GE_ASSERT_SUCCESS(BuildTensorWindowInfo(logical_view_.index, index, axis_pos, tpipe, index_info)); | 816 | + GE_ASSERT_SUCCESS(BuildTensorWindowInfo(logical_view_.index, index, axis_pos, index_info)); |
| 823 | std::string x_dtype_name; | 817 | std::string x_dtype_name; |
| 824 | std::string index_dtype_name; | 818 | std::string index_dtype_name; |
| 825 | GE_ASSERT_SUCCESS(Tensor::DtypeName(x.dtype, x_dtype_name)); | 819 | GE_ASSERT_SUCCESS(Tensor::DtypeName(x.dtype, x_dtype_name)); |
| @@ -852,8 +846,8 @@ Status IndirectLoadRegApiCall::GenerateSimd(const TPipe &tpipe, const std::vecto | |||
| 852 | const size_t axis_pos = static_cast<size_t>(axis_); | 846 | const size_t axis_pos = static_cast<size_t>(axis_); |
| 853 | LogicalTensorInfo input_info; | 847 | LogicalTensorInfo input_info; |
| 854 | LogicalTensorInfo index_info; | 848 | LogicalTensorInfo index_info; |
| 855 | - GE_ASSERT_SUCCESS(BuildTensorWindowInfo(logical_view_.input, input, axis_pos, tpipe, input_info)); | 849 | + GE_ASSERT_SUCCESS(BuildTensorWindowInfo(logical_view_.input, input, axis_pos, input_info)); |
| 856 | - GE_ASSERT_SUCCESS(BuildTensorWindowInfo(logical_view_.index, index, axis_pos, tpipe, index_info)); | 850 | + GE_ASSERT_SUCCESS(BuildTensorWindowInfo(logical_view_.index, index, axis_pos, index_info)); |
| 857 | const bool requires_strided_api = | 851 | const bool requires_strided_api = |
| 858 | logical_view_.input.kind != ascgen_utils::indirect_load::IndirectLoadLayoutKind::kDense || | 852 | logical_view_.input.kind != ascgen_utils::indirect_load::IndirectLoadLayoutKind::kDense || |
| 859 | logical_view_.index.kind != ascgen_utils::indirect_load::IndirectLoadLayoutKind::kDense; | 853 | logical_view_.index.kind != ascgen_utils::indirect_load::IndirectLoadLayoutKind::kDense; |
| @@ -899,16 +893,16 @@ Status IndirectLoadRegApiCall::GenerateSimd(const TPipe &tpipe, const std::vecto | |||
| 899 | Status IndirectLoadRegApiCall::GenerateSimtInvocation(const TPipe &tpipe, const std::string &input_dtype, | 893 | Status IndirectLoadRegApiCall::GenerateSimtInvocation(const TPipe &tpipe, const std::string &input_dtype, |
| 900 | const std::string &output_dtype, const std::string &outer_tb_var, | 894 | const std::string &output_dtype, const std::string &outer_tb_var, |
| 901 | std::stringstream &ss) const { | 895 | std::stringstream &ss) const { |
| 902 | - const LogicalTensorInfo input_info = BuildLogicalTensorInfo(logical_view_.input, tpipe); | 896 | + const LogicalTensorInfo input_info(logical_view_.input); |
| 903 | - LogicalTensorInfo index_info = BuildLogicalTensorInfo(logical_view_.index, tpipe); | 897 | + LogicalTensorInfo index_info(logical_view_.index); |
| 904 | - const LogicalTensorInfo output_info = BuildLogicalTensorInfo(logical_view_.output, tpipe); | 898 | + const LogicalTensorInfo output_info(logical_view_.output); |
| 905 | const bool index_broadcast_strided = ApplySimtIndexPhysicalStrides(index_nodes_, index_info); | 899 | const bool index_broadcast_strided = ApplySimtIndexPhysicalStrides(index_nodes_, index_info); |
| 906 | const bool strided = logical_view_.input.kind != ascgen_utils::indirect_load::IndirectLoadLayoutKind::kDense || | 900 | const bool strided = logical_view_.input.kind != ascgen_utils::indirect_load::IndirectLoadLayoutKind::kDense || |
| 907 | logical_view_.index.kind != ascgen_utils::indirect_load::IndirectLoadLayoutKind::kDense || | 901 | logical_view_.index.kind != ascgen_utils::indirect_load::IndirectLoadLayoutKind::kDense || |
| 908 | index_broadcast_strided; | 902 | index_broadcast_strided; |
| 909 | const SimtCodegenPlan plan = | 903 | const SimtCodegenPlan plan = |
| 910 | BuildSimtCodegenPlan(input_info, index_info, output_info, static_cast<size_t>(axis_), strided); | 904 | BuildSimtCodegenPlan(input_info, index_info, output_info, static_cast<size_t>(axis_), strided); |
| 911 | - const std::string body_name = "IndirectLoadSimtBody_" + ascgen_utils::GenValidName(node_name); | 905 | + const std::string body_name = kSimtBodyNamePrefix + ascgen_utils::GenValidName(node_name); |
| 912 | const Tensor *output_tensor = has_post_reduce_ ? tpipe.GetTensor(output_result_tensor_id_) : nullptr; | 906 | const Tensor *output_tensor = has_post_reduce_ ? tpipe.GetTensor(output_result_tensor_id_) : nullptr; |
| 913 | af::Expression output_element_count = af::ops::One; | 907 | af::Expression output_element_count = af::ops::One; |
| 914 | if (has_post_reduce_) { | 908 | if (has_post_reduce_) { |
| @@ -952,7 +946,7 @@ Status IndirectLoadRegApiCall::GenerateSimt(const TPipe &tpipe, const std::vecto | |||
| 952 | GE_ASSERT_SUCCESS(Tensor::DtypeName(input.dtype, input_dtype)); | 946 | GE_ASSERT_SUCCESS(Tensor::DtypeName(input.dtype, input_dtype)); |
| 953 | GE_ASSERT_SUCCESS(Tensor::DtypeName(output_dtype_, output_dtype)); | 947 | GE_ASSERT_SUCCESS(Tensor::DtypeName(output_dtype_, output_dtype)); |
| 954 | const std::string valid_node_name = ascgen_utils::GenValidName(node_name); | 948 | const std::string valid_node_name = ascgen_utils::GenValidName(node_name); |
| 955 | - const std::string context_name = "IndirectLoadSimtContext_" + valid_node_name; | 949 | + const std::string context_name = kSimtContextNamePrefix + valid_node_name; |
| 956 | std::stringstream ss; | 950 | std::stringstream ss; |
| 957 | ss << "// IndirectLoad SIMT" << std::endl; | 951 | ss << "// IndirectLoad SIMT" << std::endl; |
| 958 | ss << "{" << std::endl; | 952 | ss << "{" << std::endl; |
| @@ -576,7 +576,7 @@ void VectorFuncPartitioner::RefineEnableVFFlag(const af::AscNodePtr &node, bool | |||
| 576 | } | 576 | } |
| 577 | 577 | ||
| 578 | // 4. IndirectLoad template nodes with custom emit paths should not participate in regular VF fusion. | 578 | // 4. IndirectLoad template nodes with custom emit paths should not participate in regular VF fusion. |
| 579 | - if (ascgen_utils::indirect_load::ShouldDisableRegularVectorFunc(node)) { | 579 | + if (ascgen_utils::indirect_load::GetTemplateBehavior(node).uses_direct_gm_pipeline) { |
| 580 | enable_vf = false; | 580 | enable_vf = false; |
| 581 | GELOGD("Node [%s] is IndirectLoad custom emit node, disable VF support.", node->GetNamePtr()); | 581 | GELOGD("Node [%s] is IndirectLoad custom emit node, disable VF support.", node->GetNamePtr()); |
| 582 | return; | 582 | return; |
| @@ -39,7 +39,7 @@ af::Status LoadToNddmaTemplate::Generate(const af::AscGraph &origin_graph, | |||
| 39 | if (!af::ops::IsOps<af::ascir_op::Load>(node)) { | 39 | if (!af::ops::IsOps<af::ascir_op::Load>(node)) { |
| 40 | continue; | 40 | continue; |
| 41 | } | 41 | } |
| 42 | - if (ascgen_utils::indirect_load::ShouldDisableRegularVectorFunc(node)) { | 42 | + if (ascgen_utils::indirect_load::GetTemplateBehavior(node).uses_direct_gm_pipeline) { |
| 43 | continue; | 43 | continue; |
| 44 | } | 44 | } |
| 45 | DiscontinuityInfo info; | 45 | DiscontinuityInfo info; |
| @@ -247,7 +247,7 @@ af::Status NddmaTemplate::Generate([[maybe_unused]] const af::AscGraph &origin_g | |||
| 247 | for (const auto &node : new_case.GetAllNodes()) { | 247 | for (const auto &node : new_case.GetAllNodes()) { |
| 248 | GE_CHECK_NOTNULL(node); | 248 | GE_CHECK_NOTNULL(node); |
| 249 | if ((!af::ops::IsOps<af::ascir_op::Load>(node) && !af::ops::IsOps<af::ascir_op::Nddma>(node)) || | 249 | if ((!af::ops::IsOps<af::ascir_op::Load>(node) && !af::ops::IsOps<af::ascir_op::Nddma>(node)) || |
| 250 | - ascgen_utils::indirect_load::ShouldDisableRegularVectorFunc(node)) { | 250 | + ascgen_utils::indirect_load::GetTemplateBehavior(node).uses_direct_gm_pipeline) { |
| 251 | continue; | 251 | continue; |
| 252 | } | 252 | } |
| 253 | if (node->GetOutAllNodes().size() > 1UL) { | 253 | if (node->GetOutAllNodes().size() > 1UL) { |
| @@ -785,7 +785,6 @@ build_backend() { | |||
| 785 | indirect_load_rank3_axis1_pow2_gather_e2e_v2 \ | 785 | indirect_load_rank3_axis1_pow2_gather_e2e_v2 \ |
| 786 | indirect_load_rank3_axis1_pow2_simt_e2e_v2 \ | 786 | indirect_load_rank3_axis1_pow2_simt_e2e_v2 \ |
| 787 | indirect_load_rank3_axis1_float_int32_gather_e2e_v2 \ | 787 | indirect_load_rank3_axis1_float_int32_gather_e2e_v2 \ |
| 788 | - indirect_load_rank3_axis1_float_int64_gather_exact_e2e_v2 \ | ||
| 789 | indirect_load_rank3_axis1_float_int64_gather_e2e_v2 \ | 788 | indirect_load_rank3_axis1_float_int64_gather_e2e_v2 \ |
| 790 | indirect_load_rank4_axis1_float_int64_strided_post_simd_e2e_v2 \ | 789 | indirect_load_rank4_axis1_float_int64_strided_post_simd_e2e_v2 \ |
| 791 | indirect_load_rank4_axis1_direct_index_abs_exp2_sum_simd_e2e_v2 \ | 790 | indirect_load_rank4_axis1_direct_index_abs_exp2_sum_simd_e2e_v2 \ |
| @@ -825,9 +824,7 @@ build_backend() { | |||
| 825 | indirect_load_rank3_axis1_input_index_gap_simd_test_e2e_v2 \ | 824 | indirect_load_rank3_axis1_input_index_gap_simd_test_e2e_v2 \ |
| 826 | indirect_load_rank3_axis1_input_index_gap_simt_test_e2e_v2 \ | 825 | indirect_load_rank3_axis1_input_index_gap_simt_test_e2e_v2 \ |
| 827 | indirect_load_rank3_axis1_input_index_gap_sk_test_e2e_v2 \ | 826 | indirect_load_rank3_axis1_input_index_gap_sk_test_e2e_v2 \ |
| 828 | - indirect_load_rank3_axis1_input_index_outer_gap_simd_test_e2e_v2 \ | ||
| 829 | indirect_load_rank3_axis1_input_index_outer_gap_simt_test_e2e_v2 \ | 827 | indirect_load_rank3_axis1_input_index_outer_gap_simt_test_e2e_v2 \ |
| 830 | - indirect_load_rank3_axis1_input_index_outer_gap_sk_test_e2e_v2 \ | ||
| 831 | indirect_load_rank3_axis1_torch_gather_frontend_e2e_v2 \ | 828 | indirect_load_rank3_axis1_torch_gather_frontend_e2e_v2 \ |
| 832 | load_where_x2_x3_is_ubscalar_store_test_e2e_v2 \ | 829 | load_where_x2_x3_is_ubscalar_store_test_e2e_v2 \ |
| 833 | gather_reduce_store_test_e2e_v2 \ | 830 | gather_reduce_store_test_e2e_v2 \ |