Star708
263
代码
代码
Issues65
Pull Requests158
Actions
项目讨论
Wiki
分析
Star708
263
已合并
feat: Inductor支持PGO(复用TF流程) #1541
zhang_shengjie创建于 7月28日
feat: Inductor支持PGO(复用TF流程) #1541
已合并
zhang_shengjie创建于 7月28日
从zhang_shengjie/graph-autofusionfeature/inductor-mspti-pgo-a合入到cann/graph-autofusiondevelop
共 39 个文件变更+5288-516
Mautofuse/att/gen_tiling_impl.cpp+9-0
@@ -47,6 +47,14 @@ void PgoEnvConfigInit(TilingCodeGenConfig &generator_config) {
47 }47 }
48}48}
49 49 
50+void OverridePgoConfigByOptions(TilingCodeGenConfig &generator_config,
51+ const std::map<std::string, std::string> &options) {
52+ const auto iter = options.find(kInternalEnableAutofusePgo);
53+ if (iter != options.cend()) {
54+ generator_config.enable_autofuse_pgo = (iter->second == "true");
55+ }
56+}
57+ 
50af::Status InitializeConfigByEnvOrIni(TilingCodeGenConfig &generator_config) {58af::Status InitializeConfigByEnvOrIni(TilingCodeGenConfig &generator_config) {
51 // ATT的配置初始化，当前前端不支持传入配置文件的目录，所以这里直接使用默认的配置文件路径59 // ATT的配置初始化，当前前端不支持传入配置文件的目录，所以这里直接使用默认的配置文件路径
52 const auto res = AutoFuseConfig::MutableAttStrategyConfig().Init();60 const auto res = AutoFuseConfig::MutableAttStrategyConfig().Init();
@@ -188,6 +196,7 @@ bool GenTilingImplAutoFuseV3(const std::string &op_name, const ascir::FusedSched
188 generator_config.is_inductor_scene = is_inductor_scene;196 generator_config.is_inductor_scene = is_inductor_scene;
189 generator_config.is_cube = ascgen_utils::IsCubeFusedScheduled(fused_schedule_result);197 generator_config.is_cube = ascgen_utils::IsCubeFusedScheduled(fused_schedule_result);
190 InitializeConfigByEnvOrIni(generator_config);198 InitializeConfigByEnvOrIni(generator_config);
199+ OverridePgoConfigByOptions(generator_config, options);
191 TilingCodeGenerator generator;200 TilingCodeGenerator generator;
192 FusedParsedScheduleResult fused_parsed_schedule_result;201 FusedParsedScheduleResult fused_parsed_schedule_result;
193 GE_ASSERT_SUCCESS(GetModelInfoMap(fused_schedule_result, options, fused_parsed_schedule_result));202 GE_ASSERT_SUCCESS(GetModelInfoMap(fused_schedule_result, options, fused_parsed_schedule_result));
Mautofuse/att/gen_tiling_impl.h+2-0
@@ -16,6 +16,8 @@
16#include "ascendc_ir/ascendc_ir_core/ascendc_ir.h"16#include "ascendc_ir/ascendc_ir_core/ascendc_ir.h"
17 17 
18namespace att {18namespace att {
19+inline constexpr char kInternalEnableAutofusePgo[] = "_internal_enable_autofuse_pgo";
20+ 
19extern "C" {21extern "C" {
20/**22/**
21 * @brief 生成Tiling函数23 * @brief 生成Tiling函数
Mautofuse/att/generator/solver_pass_gen/axes_reorder_solver/axes_reorder_solver_gen.cpp+1-1
@@ -864,7 +864,7 @@ std::string AxesReorderSolverGen::GenPGOSolverClassImpl() {
864 codes += " bool CalUsedCoreNum(double &used_core_num) override;\n";864 codes += " bool CalUsedCoreNum(double &used_core_num) override;\n";
865 codes += " bool CalRealUsedCoreNum(int64_t &used_corenum) override;\n";865 codes += " bool CalRealUsedCoreNum(int64_t &used_corenum) override;\n";
866 codes += " bool SatisfyThresholdUBSize() override {return false;};\n";866 codes += " bool SatisfyThresholdUBSize() override {return false;};\n";
867- codes += " bool SatisfyUBSizeCacheLine(uint32_t idx) override {return false;};\n";867+ codes += " bool SatisfyUBSizeCacheLine(uint32_t) override {return false;};\n";
868 codes += " double GetPerf() override {return 0;};\n";868 codes += " double GetPerf() override {return 0;};\n";
869 codes += "};\n";869 codes += "};\n";
870 codes += "\n";870 codes += "\n";
Mautofuse/att/generator/tiling_code_gen_impl.cpp+15-7
@@ -463,8 +463,10 @@ std::string WrapAtomicHeaderBody(autofuse::GeneratedHeaderId header_id, const st
463 " &tiling_data);\nnamespace optiling {\nstruct PgoTensorArgs;\nstruct SearchConfig;\n" + body +463 " &tiling_data);\nnamespace optiling {\nstruct PgoTensorArgs;\nstruct SearchConfig;\n" + body +
464 "} // namespace optiling\n";464 "} // namespace optiling\n";
465 }465 }
466- return "struct AutofuseTilingDataPerf;\nnamespace optiling {\nstruct " + tiling_data_type_name +466+ const std::string global_tiling_decl = is_autofuse ? "struct AutofuseTilingData;\n" : "";
467- ";\nstruct PgoTensorArgs;\nstruct SearchConfig;\n" + body + "} // namespace optiling\n";467+ return global_tiling_decl + "struct AutofuseTilingDataPerf;\nnamespace optiling {\nstruct " +
468+ tiling_data_type_name + ";\nstruct PgoTensorArgs;\nstruct SearchConfig;\n" + body +
469+ "} // namespace optiling\n";
468 }470 }
469 std::string prefix;471 std::string prefix;
470 if (header_id == autofuse::GeneratedHeaderId::kSolver) {472 if (header_id == autofuse::GeneratedHeaderId::kSolver) {
@@ -1031,6 +1033,8 @@ void TilingCodeGenImpl::GenPgoConfigDefs(ge::CodePrinter &pgo_header) {
1031 pgo_header.AddLine(" ProfilingCallback single_callback;");1033 pgo_header.AddLine(" ProfilingCallback single_callback;");
1032 pgo_header.AddLine(" ProfilingBatchCallback batch_callback;");1034 pgo_header.AddLine(" ProfilingBatchCallback batch_callback;");
1033 pgo_header.AddLine(" PgoTensorArgs *tensor_args = nullptr;");1035 pgo_header.AddLine(" PgoTensorArgs *tensor_args = nullptr;");
1036+ pgo_header.AddLine(" std::vector<AutofuseTilingDataPerf> *measured_candidates = nullptr;");
1037+ pgo_header.AddLine(" void *stream = nullptr;");
1034 pgo_header.AddLine(" int32_t pgo_algorithm = 1; // 0 for pruning, 1 for core num");1038 pgo_header.AddLine(" int32_t pgo_algorithm = 1; // 0 for pruning, 1 for core num");
1035 pgo_header.AddLine(" bool need_change_solver_run = false;");1039 pgo_header.AddLine(" bool need_change_solver_run = false;");
1036 pgo_header.AddLine(" size_t pgo_threshold_index = 0;");1040 pgo_header.AddLine(" size_t pgo_threshold_index = 0;");
@@ -2930,8 +2934,10 @@ void TilingCodeGenImpl::GenPGOSearchTilingKeyUniqGroupBatch() {
2930 tiling_func_.AddLine(" workspaceSize += 16 * 1024 * 1024;");2934 tiling_func_.AddLine(" workspaceSize += 16 * 1024 * 1024;");
2931 tiling_func_.AddLine(" if (PgoConfig::Instance().batch_callback != nullptr) {");2935 tiling_func_.AddLine(" if (PgoConfig::Instance().batch_callback != nullptr) {");
2932 tiling_func_.AddLine(2936 tiling_func_.AddLine(
2933- " PgoConfig::Instance().batch_callback(PgoConfig::Instance().tensor_args, stream, "2937+ " if (PgoConfig::Instance().batch_callback(PgoConfig::Instance().tensor_args, stream, "
2934- "workspaceSize, &tiling_data_list);");2938+ "workspaceSize, &tiling_data_list) != 0) {");
2939+ tiling_func_.AddLine(" return false;");
2940+ tiling_func_.AddLine(" }");
2935 tiling_func_.AddLine(" }");2941 tiling_func_.AddLine(" }");
2936 tiling_func_.AddLine(" for (const auto &tiling_data_perf : tiling_data_list) {");2942 tiling_func_.AddLine(" for (const auto &tiling_data_perf : tiling_data_list) {");
2937 tiling_func_.AddLine(" if (best_perf > tiling_data_perf.best_perf) {");2943 tiling_func_.AddLine(" if (best_perf > tiling_data_perf.best_perf) {");
@@ -3967,8 +3973,10 @@ af::Status TilingCodeGenImpl::GenPGOGetScheduleResultPerGroup(
3967 tiling_func_.AddLine(" workspaceSize += 16 * 1024 * 1024;");3973 tiling_func_.AddLine(" workspaceSize += 16 * 1024 * 1024;");
3968 tiling_func_.AddLine(" if (PgoConfig::Instance().batch_callback) {");3974 tiling_func_.AddLine(" if (PgoConfig::Instance().batch_callback) {");
3969 tiling_func_.AddLine(3975 tiling_func_.AddLine(
3970- " PgoConfig::Instance().batch_callback(PgoConfig::Instance().tensor_args, stream, "3976+ " if (PgoConfig::Instance().batch_callback(PgoConfig::Instance().tensor_args, stream, "
3971- "workspaceSize, &tiling_data_list_tmp);");3977+ "workspaceSize, &tiling_data_list_tmp) != 0) {");
3978+ tiling_func_.AddLine(" return false;");
3979+ tiling_func_.AddLine(" }");
3972 tiling_func_.AddLine(" }");3980 tiling_func_.AddLine(" }");
3973 tiling_func_.AddLine(" for (size_t candidate_index = " + candidate_begin_name +3981 tiling_func_.AddLine(" for (size_t candidate_index = " + candidate_begin_name +
3974 "; candidate_index < tiling_data_list_tmp.size(); ++candidate_index) {");3982 "; candidate_index < tiling_data_list_tmp.size(); ++candidate_index) {");
@@ -5074,7 +5082,7 @@ af::Status TilingCodeGenImpl::GenPGOReuseGroupTilingWrapper() {
5074 // Gen PGOProfileReuseGroup: reuse group does not search, only profiles by copying from primary group5082 // Gen PGOProfileReuseGroup: reuse group does not search, only profiles by copying from primary group
5075 std::string pgo_profile_sig =5083 std::string pgo_profile_sig =
5076 std::string("bool PGOProfileReuseGroup(std::vector<AutofuseTilingDataPerf>& tiling_data_list, ") +5084 std::string("bool PGOProfileReuseGroup(std::vector<AutofuseTilingDataPerf>& tiling_data_list, ") +
5077- config_.tiling_data_type_name + "* output_tiling_data, void* stream, uint32_t workspaceSize, double& best_perf)";5085+ "AutofuseTilingData* output_tiling_data, void* stream, uint32_t workspaceSize, double& best_perf)";
5078 AddAtomicHeaderLine(autofuse::GeneratedHeaderId::kApi, pgo_profile_sig + ";");5086 AddAtomicHeaderLine(autofuse::GeneratedHeaderId::kApi, pgo_profile_sig + ";");
5079 tiling_func_.AddLine(pgo_profile_sig + " {");5087 tiling_func_.AddLine(pgo_profile_sig + " {");
5080 tiling_func_.AddLine(" double cur_perf = DBL_MAX;");5088 tiling_func_.AddLine(" double cur_perf = DBL_MAX;");
Mautofuse/codegen/codegen.cpp+23-1
@@ -390,10 +390,32 @@ Status Codegen::Generate(const ascir::FusedScheduledResult &fused_schedule_resul
390// inductor路径仍返回单个host_tiling字符串，通过注释marker保留拆分边界390// inductor路径仍返回单个host_tiling字符串，通过注释marker保留拆分边界
391Status Codegen::GenerateForInductor(const ascir::FusedScheduledResult &fused_schedule_result,391Status Codegen::GenerateForInductor(const ascir::FusedScheduledResult &fused_schedule_result,
392 CodegenResult &result) const {392 CodegenResult &result) const {
393+ const auto generate_tiling_without_pgo = [&]() {
394+ if (!tiling_lib_.IsInductorPgoEnabled() || ascgen_utils::IsCubeFusedScheduled(fused_schedule_result) ||
395+ !ascgen_utils::IsStaticSchedResult(fused_schedule_result)) {
396+ return af::FAILED;
397+ }
398+ GELOGW("Inductor PGO codegen failed, fallback to non-PGO codegen for static non-CV kernel");
399+ Codegen fallback(*this);
400+ fallback.tiling_lib_.DisableInductorPgo();
401+ return fallback.GenerateForInductor(fused_schedule_result, result);
402+ };
393 GE_CHK_STATUS_RET(GenerateKernel(fused_schedule_result, result.kernel, true), "Codegen generate kernel failed");403 GE_CHK_STATUS_RET(GenerateKernel(fused_schedule_result, result.kernel, true), "Codegen generate kernel failed");
394 result.tiling_data = GenerateTilingData(fused_schedule_result, true);404 result.tiling_data = GenerateTilingData(fused_schedule_result, true);
395 std::map<std::string, std::string> tiling_file_name_to_content;405 std::map<std::string, std::string> tiling_file_name_to_content;
396- GE_CHK_STATUS_RET(GenerateTilingForInductor(fused_schedule_result, tiling_file_name_to_content));406+ const auto tiling_ret = GenerateTilingForInductor(fused_schedule_result, tiling_file_name_to_content);
407+ if (tiling_ret != af::SUCCESS) {
408+ if (generate_tiling_without_pgo() == af::SUCCESS) {
409+ return af::SUCCESS;
410+ }
411+ GE_CHK_STATUS_RET(tiling_ret, "Codegen generate tiling failed");
412+ }
413+ const auto pgo_runner = tiling_file_name_to_content.find(kPgoRunnerIdentify);
414+ if (pgo_runner != tiling_file_name_to_content.end()) {
415+ GE_CHK_BOOL_RET_STATUS(!pgo_runner->second.empty() && !result.kernel.empty(), af::FAILED,
416+ "Inductor PGO runner or device source is empty");
417+ tiling_file_name_to_content[kPgoDeviceSourceIdentify] = result.kernel;
418+ }
397 GE_CHK_STATUS_RET(CombineTilingsWithSplitMarkers(tiling_file_name_to_content, result.tiling));419 GE_CHK_STATUS_RET(CombineTilingsWithSplitMarkers(tiling_file_name_to_content, result.tiling));
398 return af::SUCCESS;420 return af::SUCCESS;
399}421}
Mautofuse/codegen/codegen_tiling.cpp+14-24
@@ -16,11 +16,9 @@
16#include <initializer_list>16#include <initializer_list>
17#include <string>17#include <string>
18#include <cstdlib>18#include <cstdlib>
19-#include <fstream>
20#include <set>19#include <set>
20+#include <fstream>
21#include <securec.h>21#include <securec.h>
22-#include "runtime/base.h"
23-#include "runtime/dev.h"
24 22 
25#include "dlfcn.h"23#include "dlfcn.h"
26 24 
@@ -33,8 +31,6 @@
33#include "graph/symbolizer/symbolic_utils.h"31#include "graph/symbolizer/symbolic_utils.h"
34#include "autofuse_config/auto_fuse_config.h"32#include "autofuse_config/auto_fuse_config.h"
35#include "graph/ge_context.h"33#include "graph/ge_context.h"
36-#include "common/platform_context.h"
37-#include "graph/utils/type_utils.h"
38#include "backend/backend_spec.h"34#include "backend/backend_spec.h"
39#include "common/ascgraph_info_complete.h"35#include "common/ascgraph_info_complete.h"
40#include "codegen_tiling_cube_wrapper.h"36#include "codegen_tiling_cube_wrapper.h"
@@ -49,7 +45,6 @@ using namespace codegen;
49using namespace af::ops;45using namespace af::ops;
50using namespace ascgen_utils;46using namespace ascgen_utils;
51namespace {47namespace {
52- 
53bool CheckTilingHeadersValid(const std::map<std::string, std::string> &tiling_file_name_to_content) {48bool CheckTilingHeadersValid(const std::map<std::string, std::string> &tiling_file_name_to_content) {
54 for (const auto &pair : tiling_file_name_to_content) {49 for (const auto &pair : tiling_file_name_to_content) {
55 if (pair.second == INVALID_TILING) {50 if (pair.second == INVALID_TILING) {
@@ -141,7 +136,7 @@ std::string RenderEntryTranslationUnit(const std::string &body, const EntryTrans
141 autofuse::RequireGeneratedHeader(code.dependencies, autofuse::GeneratedHeaderId::kPgo);136 autofuse::RequireGeneratedHeader(code.dependencies, autofuse::GeneratedHeaderId::kPgo);
142 }137 }
143 if (options.enable_pgo_runtime) {138 if (options.enable_pgo_runtime) {
144- RequireSystemHeaders(code.dependencies, {"fstream", "securec.h", "unordered_set"});139+ RequireSystemHeaders(code.dependencies, {"fstream", "securec.h", "unordered_set", "utility"});
145 }140 }
146 if (options.include_solver) {141 if (options.include_solver) {
147 autofuse::RequireGeneratedHeader(code.dependencies, autofuse::GeneratedHeaderId::kSolver);142 autofuse::RequireGeneratedHeader(code.dependencies, autofuse::GeneratedHeaderId::kSolver);
@@ -460,6 +455,10 @@ std::map<std::string, std::string> TilingLib::GenerateForInductor(
460 const ascir::FusedScheduledResult &fused_schedule_result) const {455 const ascir::FusedScheduledResult &fused_schedule_result) const {
461 ascir::FusedScheduledResult elemwise_schedule_result = fused_schedule_result;456 ascir::FusedScheduledResult elemwise_schedule_result = fused_schedule_result;
462 const bool is_cube_fused_scheduled = ascgen_utils::IsCubeFusedScheduled(fused_schedule_result);457 const bool is_cube_fused_scheduled = ascgen_utils::IsCubeFusedScheduled(fused_schedule_result);
458+ if (enable_autofuse_pgo_ && !IsSupportedInductorPgoScene(fused_schedule_result)) {
459+ GELOGE(af::FAILED, "Inductor MSPTI PGO only supports static, non-CV kernels");
460+ return {{kTilingDefAndConstIdentify, ascgen_utils::INVALID_TILING}};
461+ }
463 if (is_cube_fused_scheduled) {462 if (is_cube_fused_scheduled) {
464 GE_ASSERT_SUCCESS(ascgen_utils::ProcessCubeFusionResultDynamic(elemwise_schedule_result));463 GE_ASSERT_SUCCESS(ascgen_utils::ProcessCubeFusionResultDynamic(elemwise_schedule_result));
465 }464 }
@@ -473,23 +472,7 @@ std::map<std::string, std::string> TilingLib::GenerateForInductor(
473 ss << "#pragma GCC diagnostic pop\n";472 ss << "#pragma GCC diagnostic pop\n";
474 ss << TilingFuncDefForInductor(fused_schedule_result, elemwise_schedule_result) << std::endl;473 ss << TilingFuncDefForInductor(fused_schedule_result, elemwise_schedule_result) << std::endl;
475 if (!is_cube_fused_scheduled) {474 if (!is_cube_fused_scheduled) {
476- ss << this->GenCandidateSolutionProtocolForInductor("AutofuseTilingData") << std::endl;475+ GenInductorTopnSources(elemwise_schedule_result, ss, tiling_file_name_to_content);
477- ss << this->GenTopnSelectorHelpersForInductor() << std::endl;
478- ss << this->GenBuiltinTfPgoConfigsForInductor() << std::endl;
479- ss << this->GenInductorConfigParserForInductor() << std::endl;
480- ss << GenGetTilingKeyCount(elemwise_schedule_result) << std::endl;
481- if (!ascgen_utils::IsSingleGroup(elemwise_schedule_result)) {
482- ss << GenUpdateCurPerfAndBlockByGroupHelper() << std::endl;
483- }
484- ss << this->GenEvaluateModeledPerfForInductor("AutofuseTilingData", elemwise_schedule_result) << std::endl;
485- ss << "extern \"C\" double GetModeledPerfForTesting(const AutofuseTilingData *tiling_data) {\n"
486- << " if (tiling_data == nullptr) { return 0.0; }\n"
487- << " double modeled_perf = EvaluateModeledPerf(*tiling_data);\n"
488- << " return std::isfinite(modeled_perf) ? modeled_perf : DBL_MAX;\n"
489- << "}\n"
490- << std::endl;
491- ss << this->GenGetTopnSolutionsFuncForInductor(elemwise_schedule_result, "AutofuseTilingData") << std::endl;
492- ss << this->GenGetTilingDataReprFuncForInductor(elemwise_schedule_result, "AutofuseTilingData") << std::endl;
493 }476 }
494 // 生成GenConstTilingData方法（对所有场景生成，包括CV fusion静态shape）477 // 生成GenConstTilingData方法（对所有场景生成，包括CV fusion静态shape）
495 ss << TilingData("Autofuse").GenerateConst(fused_schedule_result) << std::endl;478 ss << TilingData("Autofuse").GenerateConst(fused_schedule_result) << std::endl;
@@ -506,6 +489,10 @@ std::map<std::string, std::string> TilingLib::GenerateForInductor(
506 return tiling_file_name_to_content;489 return tiling_file_name_to_content;
507}490}
508 491 
492+bool TilingLib::IsSupportedInductorPgoScene(const ascir::FusedScheduledResult &fused_schedule_result) const {
493+ return !ascgen_utils::IsCubeFusedScheduled(fused_schedule_result) && IsStaticSchedResult(fused_schedule_result);
494+}
495+ 
509void TilingLib::GenPgoMixTilingTable(const ascir::FusedScheduledResult &fused_schedule_result,496void TilingLib::GenPgoMixTilingTable(const ascir::FusedScheduledResult &fused_schedule_result,
510 std::stringstream &ss) const {497 std::stringstream &ss) const {
511 for (size_t graph_id = 0U; graph_id < fused_schedule_result.node_idx_to_scheduled_results.size(); graph_id++) {498 for (size_t graph_id = 0U; graph_id < fused_schedule_result.node_idx_to_scheduled_results.size(); graph_id++) {
@@ -772,6 +759,9 @@ std::map<std::string, std::string> TilingLib::GetTilingHeaders(const ascir::Fuse
772 tiling_file_name_to_content[kTilingHeadIdentify] += ss.str();759 tiling_file_name_to_content[kTilingHeadIdentify] += ss.str();
773 options.emplace("tiling_data_type_name", tiling_name);760 options.emplace("tiling_data_type_name", tiling_name);
774 options.emplace("solver_type", "AxesReorder");761 options.emplace("solver_type", "AxesReorder");
762+ if (is_inductor_scene) {
763+ options.emplace(att::kInternalEnableAutofusePgo, enable_autofuse_pgo_ ? "true" : "false");
764+ }
775 GE_CHK_BOOL_EXEC(765 GE_CHK_BOOL_EXEC(
776 this->codegen_func_(fused_schedule_result.fused_graph_name.GetString(), fused_schedule_result, options,766 this->codegen_func_(fused_schedule_result.fused_graph_name.GetString(), fused_schedule_result, options,
777 tiling_file_name_to_content, is_inductor_scene),767 tiling_file_name_to_content, is_inductor_scene),
Mautofuse/codegen/codegen_tiling.h+124-44
@@ -42,6 +42,8 @@ const std::string kCubeTilingHeadInclude = "#include \"autofuse_cube_tiling_data
42const std::string kCubeKernelTilingWrapperHpp = "ACubeKernelTilingWrapperHpp";42const std::string kCubeKernelTilingWrapperHpp = "ACubeKernelTilingWrapperHpp";
43const std::string kCubeKernelTilingWrapperCpp = "BCubeKernelTilingWrapperCpp";43const std::string kCubeKernelTilingWrapperCpp = "BCubeKernelTilingWrapperCpp";
44const std::string kCubeKernelTilingWrapperInclude = "#include \"cube_kernel_tiling_wrapper.h\"";44const std::string kCubeKernelTilingWrapperInclude = "#include \"cube_kernel_tiling_wrapper.h\"";
45+const std::string kPgoRunnerIdentify = "PgoRunner";
46+const std::string kPgoDeviceSourceIdentify = "PgoDeviceSource";
45 47 
46struct MatMulCubeInfo {48struct MatMulCubeInfo {
47 bool transpose_x1 = false;49 bool transpose_x1 = false;
@@ -97,9 +99,14 @@ struct PgoShapeStringStream {
97 std::stringstream tiling_set_shape_dim;99 std::stringstream tiling_set_shape_dim;
98 std::stringstream shape_dim_use;100 std::stringstream shape_dim_use;
99};101};
102+ 
103+// TilingLib declarations stay centralized to preserve the class layout and access control. Implementations are split by
104+// responsibility: common entry/workspace in codegen_tiling.cpp, Cube/CV in codegen_tiling_cube.cpp, PGO search in
105+// codegen_tiling_pgo_search.cpp, PGO IO/memory in codegen_tiling_pgo_memory.cpp, shared PGO runtime code generation in
106+// codegen_tiling_pgo_common.cpp, Inductor TopN/runner/proxy in the corresponding codegen_tiling_inductor_*.cpp files.
100class TilingLib {107class TilingLib {
101 public:108 public:
102- // Core generation entry points are implemented in codegen_tiling.cpp.109+ // codegen_tiling.cpp: common TF and Inductor entry generation.
103 TilingLib(const std::string &lib_path, const std::string &codegen_symbol_name);110 TilingLib(const std::string &lib_path, const std::string &codegen_symbol_name);
104 std::map<std::string, std::string> Generate(const ::ascir::FusedScheduledResult &fused_schedule_result,111 std::map<std::string, std::string> Generate(const ::ascir::FusedScheduledResult &fused_schedule_result,
105 const std::map<std::string, std::string> &shape_info,112 const std::map<std::string, std::string> &shape_info,
@@ -107,19 +114,29 @@ class TilingLib {
107 std::map<std::string, std::string> GenerateForInductor(114 std::map<std::string, std::string> GenerateForInductor(
108 const ::ascir::FusedScheduledResult &fused_schedule_result) const;115 const ::ascir::FusedScheduledResult &fused_schedule_result) const;
109 116 
110- // The TF PGO generation entry point is implemented in codegen_tiling_pgo_runtime.cpp.117+ // codegen_tiling_pgo_runtime.cpp: TF and Inductor shared PGO runtime orchestration.
111 std::string GenerateForPgo(const ::ascir::FusedScheduledResult &fused_schedule_result,118 std::string GenerateForPgo(const ::ascir::FusedScheduledResult &fused_schedule_result,
112 const std::string &pgo_dir) const;119 const std::string &pgo_dir) const;
113 std::string GetTilingIncludeHead(bool is_cv = false) const;120 std::string GetTilingIncludeHead(bool is_cv = false) const;
121+ bool IsInductorPgoEnabled() const {
122+ return enable_autofuse_pgo_;
123+ }
124+ void DisableInductorPgo() {
125+ enable_autofuse_pgo_ = false;
126+ }
114 127 
115 protected:128 protected:
116- // Common tiling generation is implemented in codegen_tiling.cpp.129+ // codegen_tiling.cpp: ordinary tiling entry and generated translation-unit assembly.
117 std::string TilingFuncDef(const ::ascir::FusedScheduledResult &fused_schedule_result,130 std::string TilingFuncDef(const ::ascir::FusedScheduledResult &fused_schedule_result,
118 const ::ascir::FusedScheduledResult &elemwise_schedule_result,131 const ::ascir::FusedScheduledResult &elemwise_schedule_result,
119 const std::map<std::string, std::string> &shape_info, const std::string &pgo_dir,132 const std::map<std::string, std::string> &shape_info, const std::string &pgo_dir,
120 const std::string &core_num) const;133 const std::string &core_num) const;
121 std::string TilingFuncDefForInductor(const ::ascir::FusedScheduledResult &fused_schedule_result,134 std::string TilingFuncDefForInductor(const ::ascir::FusedScheduledResult &fused_schedule_result,
122 const ::ascir::FusedScheduledResult &elemwise_schedule_result) const;135 const ::ascir::FusedScheduledResult &elemwise_schedule_result) const;
136+ bool IsSupportedInductorPgoScene(const ::ascir::FusedScheduledResult &fused_schedule_result) const;
137+ // codegen_tiling_inductor_topn.cpp: Inductor modeled/measured TopN source generation.
138+ void GenInductorTopnSources(const ::ascir::FusedScheduledResult &fused_schedule_result, std::stringstream &ss,
139+ std::map<std::string, std::string> &tiling_file_name_to_content) const;
123 std::map<std::string, std::string> GetTilingHeaders(const ::ascir::FusedScheduledResult &fused_schedule_result,140 std::map<std::string, std::string> GetTilingHeaders(const ::ascir::FusedScheduledResult &fused_schedule_result,
124 bool is_inductor_scene, bool is_cv = false) const;141 bool is_inductor_scene, bool is_cv = false) const;
125 std::string InferShapeDef(const ::ascir::HintGraph &graph) const;142 std::string InferShapeDef(const ::ascir::HintGraph &graph) const;
@@ -128,7 +145,7 @@ class TilingLib {
128 std::string OpInputDef(const ::ascir::NodeView &node) const;145 std::string OpInputDef(const ::ascir::NodeView &node) const;
129 std::string OpOutputDef(const ::ascir::NodeView &node) const;146 std::string OpOutputDef(const ::ascir::NodeView &node) const;
130 147 
131- // PGO tensor arguments and memory management are implemented in codegen_tiling_pgo_memory.cpp.148+ // codegen_tiling_pgo_memory.cpp: PGO IO declarations, size calculation and allocation lifecycle.
132 std::string ExternFunctionDeclare(const ::ascir::FusedScheduledResult &fused_schedule_result,149 std::string ExternFunctionDeclare(const ::ascir::FusedScheduledResult &fused_schedule_result,
133 const std::string tiling) const;150 const std::string tiling) const;
134 std::string PGOTensorArgsDef() const;151 std::string PGOTensorArgsDef() const;
@@ -146,24 +163,27 @@ class TilingLib {
146 uint32_t PGOSearchFuncGetInputOutputCount(const ::ascir::FusedScheduledResult &fused_schedule_result) const;163 uint32_t PGOSearchFuncGetInputOutputCount(const ::ascir::FusedScheduledResult &fused_schedule_result) const;
147 uint32_t PGOSearchFuncGetOutputCount(const ::ascir::FusedScheduledResult &fused_schedule_result) const;164 uint32_t PGOSearchFuncGetOutputCount(const ::ascir::FusedScheduledResult &fused_schedule_result) const;
148 std::string CalculateTensorMemorySizeStr(const ::ascir::TensorAttr &tensor) const;165 std::string CalculateTensorMemorySizeStr(const ::ascir::TensorAttr &tensor) const;
166+ std::string CalculateTensorMemorySizeStr(const ::ascir::TensorAttr &tensor,
167+ const ::af::Expression &element_offset) const;
168+ std::vector<std::string> CalculatePgoIoMemorySizeStrs(const ::ascir::FusedScheduledResult &fused_schedule_result,
169+ int64_t io_index, bool is_input,
170+ const ::ascir::TensorAttr &fallback_tensor) const;
149 std::string PGOSearchTensorMallocDef(const ::ascir::FusedScheduledResult &fused_schedule_result) const;171 std::string PGOSearchTensorMallocDef(const ::ascir::FusedScheduledResult &fused_schedule_result) const;
150 std::string PGOSearchTensorFreeDef(const ::ascir::FusedScheduledResult &fused_schedule_result) const;172 std::string PGOSearchTensorFreeDef(const ::ascir::FusedScheduledResult &fused_schedule_result) const;
151- 173+ // codegen_tiling.cpp: fallback headers and ordinary tiling helpers.
152- // Fallback headers are implemented in codegen_tiling.cpp.
153 std::string StubHeadersWithoutCodegenFunc() const;174 std::string StubHeadersWithoutCodegenFunc() const;
154 std::string GetStubTilingHeaders(const ::ascir::FusedScheduledResult &fused_schedule_result) const;175 std::string GetStubTilingHeaders(const ::ascir::FusedScheduledResult &fused_schedule_result) const;
155 std::string GetStubTilingApi(const ::ascir::FusedScheduledResult &fused_schedule_result, bool include_pgo) const;176 std::string GetStubTilingApi(const ::ascir::FusedScheduledResult &fused_schedule_result, bool include_pgo) const;
156 void PopulateFallbackAtomicHeaders(std::map<std::string, std::string> &tiling_file_name_to_content,177 void PopulateFallbackAtomicHeaders(std::map<std::string, std::string> &tiling_file_name_to_content,
157 const ::ascir::FusedScheduledResult &fused_schedule_result, bool use_att_codegen,178 const ::ascir::FusedScheduledResult &fused_schedule_result, bool use_att_codegen,
158 bool include_pgo) const;179 bool include_pgo) const;
159- 180+ // codegen_tiling_pgo_search.cpp: shared TF/Inductor PGO search source generation.
160- // Shared PGO search helpers are implemented in codegen_tiling_pgo_search.cpp.
161 std::string GenGetAutoFuseTilingInput(bool is_inductor_scene) const;181 std::string GenGetAutoFuseTilingInput(bool is_inductor_scene) const;
162 std::string GenGetResLimitStru(void) const;182 std::string GenGetResLimitStru(void) const;
163 bool IsMixKernelTaskType(const ::ascir::FusedScheduledResult &fused_schedule_result) const;183 bool IsMixKernelTaskType(const ::ascir::FusedScheduledResult &fused_schedule_result) const;
164 184 
165 private:185 private:
166- // Common tiling generation is implemented in codegen_tiling.cpp.186+ // codegen_tiling.cpp: common tiling, workspace, shape and cache generation.
167 // 判断某个 origin_var 是否被特定 schedule_group 使用187 // 判断某个 origin_var 是否被特定 schedule_group 使用
168 bool IsVarUsedInScheduleGroup(const std::string &var_define, const ::ascir::ScheduleGroup &schedule_group) const;188 bool IsVarUsedInScheduleGroup(const std::string &var_define, const ::ascir::ScheduleGroup &schedule_group) const;
169 std::string GenGetTilingSizeFunc(const ::ascir::FusedScheduledResult &fused_schedule_result,189 std::string GenGetTilingSizeFunc(const ::ascir::FusedScheduledResult &fused_schedule_result,
@@ -175,31 +195,39 @@ class TilingLib {
175 std::string GenTilingFuncForInductor(const ::ascir::FusedScheduledResult &fused_schedule_result,195 std::string GenTilingFuncForInductor(const ::ascir::FusedScheduledResult &fused_schedule_result,
176 const ::ascir::FusedScheduledResult &elemwise_schedule_result,196 const ::ascir::FusedScheduledResult &elemwise_schedule_result,
177 const std::string func, const std::string tiling) const;197 const std::string func, const std::string tiling) const;
178- 198+ // codegen_tiling_inductor_topn.cpp: candidate protocol, selection and multi-group performance aggregation.
179- // Inductor TopN generation is implemented in codegen_tiling_inductor_topn.cpp.
180 std::string GenGetTopnSolutionsFuncForInductor(const ::ascir::FusedScheduledResult &fused_schedule_result,199 std::string GenGetTopnSolutionsFuncForInductor(const ::ascir::FusedScheduledResult &fused_schedule_result,
181- const std::string &tiling) const;200+ const std::string &tiling, bool use_measured_perf = false,
201+ const std::string &entry_declaration = "") const;
202+ std::string GenModeledFallbackTopnForInductor(const ::ascir::FusedScheduledResult &fused_schedule_result,
203+ const std::string &tiling) const;
182 void GenTopnInitSearchTiling(std::stringstream &ss, const ::ascir::FusedScheduledResult &fused_schedule_result,204 void GenTopnInitSearchTiling(std::stringstream &ss, const ::ascir::FusedScheduledResult &fused_schedule_result,
183- const std::string &tiling, int symbol_value_count) const;205+ const std::string &tiling, int symbol_value_count, bool use_measured_perf) const;
184 void GenTopnGetTilingFunc(std::stringstream &ss, const ::ascir::FusedScheduledResult &fused_schedule_result,206 void GenTopnGetTilingFunc(std::stringstream &ss, const ::ascir::FusedScheduledResult &fused_schedule_result,
185- const std::string &tiling, int symbol_value_count) const;207+ const std::string &tiling, int symbol_value_count, bool use_measured_perf) const;
186 void GenTopnSearchTilingSetup(std::stringstream &ss, const std::string &tiling,208 void GenTopnSearchTilingSetup(std::stringstream &ss, const std::string &tiling,
187 const ::ascir::FusedScheduledResult &fused_schedule_result) const;209 const ::ascir::FusedScheduledResult &fused_schedule_result) const;
188 void GenTopnCollectCandidates(std::stringstream &ss, const std::string &tiling) const;210 void GenTopnCollectCandidates(std::stringstream &ss, const std::string &tiling) const;
211+ void GenTopnMeasuredCoreSearch(std::stringstream &ss, const std::string &tiling) const;
212+ void GenTopnMeasuredBatchProfiling(std::stringstream &ss) const;
213+ void GenTopnAppendMeasuredDefault(std::stringstream &ss) const;
189 void GenTopnSearchTilingKeyCall(std::stringstream &ss, const ::ascir::FusedScheduledResult &fused_schedule_result,214 void GenTopnSearchTilingKeyCall(std::stringstream &ss, const ::ascir::FusedScheduledResult &fused_schedule_result,
190 const std::string &search_cfg) const;215 const std::string &search_cfg) const;
191 void GenTopnSetFailureMessage(std::stringstream &ss, const std::string &indent, const std::string &reason) const;216 void GenTopnSetFailureMessage(std::stringstream &ss, const std::string &indent, const std::string &reason) const;
192 void GenTopnDefaultTiling(std::stringstream &ss, const std::string &tiling) const;217 void GenTopnDefaultTiling(std::stringstream &ss, const std::string &tiling) const;
193 void GenTopnSearchAndFinalChecks(std::stringstream &ss, const std::string &tiling,218 void GenTopnSearchAndFinalChecks(std::stringstream &ss, const std::string &tiling,
194- const ::ascir::FusedScheduledResult &fused_schedule_result) const;219+ const ::ascir::FusedScheduledResult &fused_schedule_result,
220+ bool use_measured_perf) const;
195 void GenGenerateTopnSolutionsEntry(std::stringstream &ss, const ::ascir::FusedScheduledResult &fused_schedule_result,221 void GenGenerateTopnSolutionsEntry(std::stringstream &ss, const ::ascir::FusedScheduledResult &fused_schedule_result,
196- const std::string &tiling,222+ const std::string &tiling, const codegen::PgoShapeStringStream &pgo_shape_dim,
197- const codegen::PgoShapeStringStream &pgo_shape_dim) const;223+ const std::string &entry_declaration) const;
224+ void GenInductorPgoProxyEntry(std::stringstream &ss, const std::string &tiling) const;
198 std::string GenCandidateSolutionProtocolForInductor(const std::string &tiling) const;225 std::string GenCandidateSolutionProtocolForInductor(const std::string &tiling) const;
226+ void GenDeduplicateCandidateSolutionsPrefix(std::stringstream &ss) const;
199 void GenDeduplicateCandidateSolutions(std::stringstream &ss) const;227 void GenDeduplicateCandidateSolutions(std::stringstream &ss) const;
228+ void GenDeduplicateMeasuredCandidateSolutions(std::stringstream &ss) const;
200 std::string GenTopnSelectorHelpersForInductor() const;229 std::string GenTopnSelectorHelpersForInductor() const;
201- std::string GenSearchConfigProtocolForInductor() const;230+ std::string GenMeasuredTopnSelectorHelpersForInductor() const;
202- std::string GenBuiltinTfPgoConfigsForInductor() const;
203 std::string GenInductorConfigParserForInductor() const;231 std::string GenInductorConfigParserForInductor() const;
204 std::string GenGetTilingDataReprFuncForInductor(const ::ascir::FusedScheduledResult &fused_schedule_result,232 std::string GenGetTilingDataReprFuncForInductor(const ::ascir::FusedScheduledResult &fused_schedule_result,
205 const std::string &tiling) const;233 const std::string &tiling) const;
@@ -217,8 +245,7 @@ class TilingLib {
217 const std::string &field_prefix, bool top_level) const;245 const std::string &field_prefix, bool top_level) const;
218 void GenReprSingleGroup(std::stringstream &ss, const ::ascir::FusedScheduledResult &fused_schedule_result) const;246 void GenReprSingleGroup(std::stringstream &ss, const ::ascir::FusedScheduledResult &fused_schedule_result) const;
219 void GenReprMultiGroup(std::stringstream &ss, const ::ascir::FusedScheduledResult &fused_schedule_result) const;247 void GenReprMultiGroup(std::stringstream &ss, const ::ascir::FusedScheduledResult &fused_schedule_result) const;
220- 248+ // codegen_tiling_pgo_search.cpp: PGO candidate traversal, tiling key and configuration persistence.
221- // PGO candidate search is implemented in codegen_tiling_pgo_search.cpp.
222 std::string GenPgoTilingFunc(const ::ascir::FusedScheduledResult &fused_schedule_result, const std::string &tiling,249 std::string GenPgoTilingFunc(const ::ascir::FusedScheduledResult &fused_schedule_result, const std::string &tiling,
223 codegen::PgoShapeStringStream &pgo_shape_dim, bool is_inductor_scene,250 codegen::PgoShapeStringStream &pgo_shape_dim, bool is_inductor_scene,
224 const std::string &core_num = "0") const;251 const std::string &core_num = "0") const;
@@ -229,6 +256,7 @@ class TilingLib {
229 std::string GenPgoTilingSearchPGO(const ::ascir::FusedScheduledResult &fused_schedule_result,256 std::string GenPgoTilingSearchPGO(const ::ascir::FusedScheduledResult &fused_schedule_result,
230 codegen::PgoShapeStringStream &pgo_shape_dim, const std::string &tiling,257 codegen::PgoShapeStringStream &pgo_shape_dim, const std::string &tiling,
231 bool is_inductor_scene, const std::string &core_num) const;258 bool is_inductor_scene, const std::string &core_num) const;
259+ void GenPgoTilingKeySearch(const ::ascir::FusedScheduledResult &fused_schedule_result, std::stringstream &ss) const;
232 260 
233 std::string GenPgoTilingSearch(const ::ascir::FusedScheduledResult &fused_schedule_result,261 std::string GenPgoTilingSearch(const ::ascir::FusedScheduledResult &fused_schedule_result,
234 codegen::PgoShapeStringStream &pgo_shape_dim, const std::string &tiling) const;262 codegen::PgoShapeStringStream &pgo_shape_dim, const std::string &tiling) const;
@@ -242,46 +270,98 @@ class TilingLib {
242 std::string GenPGOGetTilingKey(const std::string tiling) const;270 std::string GenPGOGetTilingKey(const std::string tiling) const;
243 std::string GenSavePGOSearchTilingDataFunc(const std::string tiling) const;271 std::string GenSavePGOSearchTilingDataFunc(const std::string tiling) const;
244 std::string GenSavePGOConfigTilingDataFunc() const;272 std::string GenSavePGOConfigTilingDataFunc() const;
245- 273+ // codegen_tiling_pgo_common.cpp: shared MSPTI callbacks and launch/runtime source generation.
246- // PGO runtime source generation is implemented in codegen_tiling_pgo_common.cpp.
247 void GenPgoSaveTilingKey(std::stringstream &ss) const;274 void GenPgoSaveTilingKey(std::stringstream &ss) const;
248 void GenPgoAppendSearchTilingData(std::stringstream &ss) const;275 void GenPgoAppendSearchTilingData(std::stringstream &ss) const;
249- void GenPgoKernelLaunchOpArgs(const ::ascir::FusedScheduledResult &fused_schedule_result,276+ void GenPgoKernelLaunchOpArgs(const ::ascir::FusedScheduledResult &fused_schedule_result, std::stringstream &ss,
250- std::stringstream &ss) const;277+ bool direct_link = false) const;
251 void GenDynamicLibraryLoaderCode(std::stringstream &ss) const;278 void GenDynamicLibraryLoaderCode(std::stringstream &ss) const;
252- void GenPgoHeaders(std::stringstream &ss) const;279+ void GenPgoHeaders(std::stringstream &ss, bool direct_link = false) const;
253 void GenPgoMain(const ::ascir::FusedScheduledResult &fused_schedule_result, std::stringstream &ss) const;280 void GenPgoMain(const ::ascir::FusedScheduledResult &fused_schedule_result, std::stringstream &ss) const;
254 void GenPgoEnvInit(const ::ascir::FusedScheduledResult &fused_schedule_result, std::stringstream &ss) const;281 void GenPgoEnvInit(const ::ascir::FusedScheduledResult &fused_schedule_result, std::stringstream &ss) const;
255 void GenPgoCardLock(std::stringstream &ss) const;282 void GenPgoCardLock(std::stringstream &ss) const;
256 void GenPgoMixTilingTable(const ::ascir::FusedScheduledResult &fused_schedule_result, std::stringstream &ss) const;283 void GenPgoMixTilingTable(const ::ascir::FusedScheduledResult &fused_schedule_result, std::stringstream &ss) const;
257 void GenPgoCheckTilingIsMix(const ::ascir::FusedScheduledResult &fused_schedule_result, std::stringstream &ss) const;284 void GenPgoCheckTilingIsMix(const ::ascir::FusedScheduledResult &fused_schedule_result, std::stringstream &ss) const;
285+ void GenPgoToolDeclarations(const ::ascir::FusedScheduledResult &fused_schedule_result, const std::string &pgo_dir,
286+ std::stringstream &ss, bool direct_link) const;
258 void GenPgoToolFunction(const ::ascir::FusedScheduledResult &fused_schedule_result, const std::string &pgo_dir,287 void GenPgoToolFunction(const ::ascir::FusedScheduledResult &fused_schedule_result, const std::string &pgo_dir,
259- std::stringstream &ss) const;288+ std::stringstream &ss, bool direct_link = false) const;
260- void GenPgoLaunchKernelInit(std::stringstream &ss) const;289+ void GenPgoLaunchKernelInit(std::stringstream &ss, bool direct_link = false) const;
261- void GenPgoLaunchParamsInit(const ::ascir::FusedScheduledResult &fused_schedule_result, std::stringstream &ss) const;290+ void GenInductorPgoKernelFunctionInit(std::stringstream &ss) const;
291+ void GenPgoKernelFunctionsInit(const std::string &bin_handle, std::stringstream &ss) const;
292+ void GenPgoLaunchParamsInit(const ::ascir::FusedScheduledResult &fused_schedule_result, std::stringstream &ss,
293+ bool direct_link = false) const;
262 void GenPgoLaunchParamsDeInit(std::stringstream &ss) const;294 void GenPgoLaunchParamsDeInit(std::stringstream &ss) const;
263 void GenPgoUpdateLaunchParams(std::stringstream &ss) const;295 void GenPgoUpdateLaunchParams(std::stringstream &ss) const;
264- void GenPgoLaunchParams(const ::ascir::FusedScheduledResult &fused_schedule_result, std::stringstream &ss) const;296+ void GenInductorPgoUpdateLaunchParams(std::stringstream &ss) const;
297+ void GenPgoCopyLaunchArgs(std::stringstream &ss, const std::string &kernel_type, const std::string &assignment) const;
298+ void GenPgoLaunchParams(const ::ascir::FusedScheduledResult &fused_schedule_result, std::stringstream &ss,
299+ bool direct_link = false) const;
265 void GenPgoDeinit(const ::ascir::FusedScheduledResult &fused_schedule_result, std::stringstream &ss) const;300 void GenPgoDeinit(const ::ascir::FusedScheduledResult &fused_schedule_result, std::stringstream &ss) const;
266 void GenPgoWrapperParmCall(const ::ascir::FusedScheduledResult &fused_schedule_result, std::stringstream &ss) const;301 void GenPgoWrapperParmCall(const ::ascir::FusedScheduledResult &fused_schedule_result, std::stringstream &ss) const;
302+ void GenPgoWrapperInit(std::stringstream &ss, bool direct_link) const;
267 void GenPgoWrapperKernelLaunch(std::stringstream &ss) const;303 void GenPgoWrapperKernelLaunch(std::stringstream &ss) const;
268- void GenPgoWrapper(const ::ascir::FusedScheduledResult &fused_schedule_result, std::stringstream &ss) const;304+ void GenPgoWrapper(const ::ascir::FusedScheduledResult &fused_schedule_result, std::stringstream &ss,
269- void GenPgoProfilingConstants(std::stringstream &ss) const;305+ bool direct_link = false) const;
306+ void GenPgoProfilingConstants(std::stringstream &ss, bool direct_link = false) const;
270 void GenPgoMsptiStringTable(std::stringstream &ss) const;307 void GenPgoMsptiStringTable(std::stringstream &ss) const;
271- void GenPgoMsptiRequest(std::stringstream &ss) const;308+ void GenPgoMsptiRequest(std::stringstream &ss, bool direct_link = false) const;
272- void GenPgoMsptiComplete(std::stringstream &ss) const;309+ void GenPgoDirectMsptiKernelHandlers(std::stringstream &ss) const;
273- void GenPgoMsptiToolFunction(std::stringstream &ss) const;310+ void GenPgoDirectMsptiComplete(std::stringstream &ss) const;
274- void GenPgoMsptiProfiling(std::stringstream &ss) const;311+ void GenPgoLegacyMsptiComplete(std::stringstream &ss) const;
312+ void GenPgoMsptiComplete(std::stringstream &ss, bool direct_link = false) const;
313+ void GenPgoMsptiToolFunction(std::stringstream &ss, bool direct_link = false) const;
314+ void GenPgoMsptiProfiling(std::stringstream &ss, bool direct_link = false) const;
315+ void GenPgoDirectBatchCallback(std::stringstream &ss) const;
275 void GenPgoBatchCallback(std::stringstream &ss) const;316 void GenPgoBatchCallback(std::stringstream &ss) const;
276- void GenPgoBatchProcess(std::stringstream &ss) const;317+ void GenPgoDirectBatchProcess(std::stringstream &ss) const;
277- void GenPgoGetProfilingBatch(const ::ascir::FusedScheduledResult &fused_schedule_result, std::stringstream &ss) const;318+ void GenPgoBatchProcess(std::stringstream &ss, bool direct_link = false) const;
278- void GenPgoProfilingCallback(std::stringstream &ss) const;319+ void GenPgoProfilingBatchSetup(std::stringstream &ss, bool direct_link) const;
279- void GenPgoGetProfiling(const ::ascir::FusedScheduledResult &fused_schedule_result, std::stringstream &ss) const;320+ void GenPgoGetProfilingBatch(const ::ascir::FusedScheduledResult &fused_schedule_result, std::stringstream &ss,
321+ bool direct_link = false) const;
322+ void GenPgoDirectProfilingCallback(std::stringstream &ss) const;
323+ void GenPgoLegacyProfilingCallback(std::stringstream &ss) const;
324+ void GenPgoProfilingCallback(std::stringstream &ss, bool direct_link = false) const;
325+ void GenPgoProfilingSetup(std::stringstream &ss, bool direct_link) const;
326+ void GenPgoProfilingLaunch(std::stringstream &ss, bool direct_link) const;
327+ void GenPgoProfilingWorkspaceCleanup(std::stringstream &ss, bool direct_link) const;
328+ void GenPgoGetProfiling(const ::ascir::FusedScheduledResult &fused_schedule_result, std::stringstream &ss,
329+ bool direct_link = false) const;
280 void GenPgoFunc(const ::ascir::FusedScheduledResult &fused_schedule_result, std::stringstream &ss) const;330 void GenPgoFunc(const ::ascir::FusedScheduledResult &fused_schedule_result, std::stringstream &ss) const;
281 void GenPgoStaticFunc(const ::ascir::FusedScheduledResult &fused_schedule_result, std::stringstream &ss) const;331 void GenPgoStaticFunc(const ::ascir::FusedScheduledResult &fused_schedule_result, std::stringstream &ss) const;
282 void GenPgoProfiling(const ::ascir::FusedScheduledResult &fused_schedule_result, std::stringstream &ss) const;332 void GenPgoProfiling(const ::ascir::FusedScheduledResult &fused_schedule_result, std::stringstream &ss) const;
283- 333+ // codegen_tiling_pgo_runtime.cpp: shared runner composition.
284- // Common tiling helpers are implemented in codegen_tiling.cpp.334+ void GenSharedPgoRuntimeLaunch(const ::ascir::FusedScheduledResult &fused_schedule_result, const std::string &pgo_dir,
335+ std::stringstream &ss, bool direct_link) const;
336+ void GenSharedPgoRuntimeProfiling(const ::ascir::FusedScheduledResult &fused_schedule_result, std::stringstream &ss,
337+ bool direct_link) const;
338+ std::string GenInductorPgoRunner(const ::ascir::FusedScheduledResult &fused_schedule_result) const;
339+ // codegen_tiling_inductor_pgo_runner.cpp: standalone runner protocol, parsing and ACL runtime.
340+ void GenInductorPgoResultProtocol(std::stringstream &ss) const;
341+ void GenInductorPgoResultTypes(std::stringstream &ss) const;
342+ void GenInductorPgoRecordWriter(std::stringstream &ss) const;
343+ void GenInductorPgoResultWriter(std::stringstream &ss) const;
344+ void GenInductorPgoSearchWriter(std::stringstream &ss) const;
345+ void GenInductorPgoArgValidators(std::stringstream &ss) const;
346+ void GenInductorPgoArgParser(std::stringstream &ss) const;
347+ void GenInductorPgoContextGuard(std::stringstream &ss) const;
348+ void GenInductorPgoHostLoader(std::stringstream &ss) const;
349+ void GenInductorPgoRuntime(const ::ascir::FusedScheduledResult &fused_schedule_result, std::stringstream &ss) const;
350+ void GenInductorPgoAclRuntime(std::stringstream &ss) const;
351+ void GenInductorPgoMemoryRuntime(const ::ascir::FusedScheduledResult &fused_schedule_result,
352+ std::stringstream &ss) const;
353+ void GenInductorPgoDeinitRuntime(const ::ascir::FusedScheduledResult &fused_schedule_result,
354+ std::stringstream &ss) const;
355+ void GenInductorPgoMain(std::stringstream &ss) const;
356+ std::string GenTopnPgoContextAbi() const;
357+ // codegen_tiling_inductor_pgo_proxy.cpp: parent process proxy, manifest and child process launch.
358+ void GenInductorPgoProxyIncludes(std::stringstream &ss, const std::string &tiling) const;
359+ void GenInductorPgoProxyFileHelpers(std::stringstream &ss) const;
360+ void GenInductorPgoProxySha256(std::stringstream &ss) const;
361+ void GenInductorPgoProxyManifest(std::stringstream &ss) const;
362+ void GenInductorPgoProxyResultParser(std::stringstream &ss, const std::string &tiling) const;
363+ void GenInductorPgoProxySpawn(std::stringstream &ss) const;
364+ void GenInductorPgoProxyFunction(std::stringstream &ss, const std::string &tiling) const;
285 std::string GenExternTilingFunc(const ::ascir::FusedScheduledResult &fused_schedule_result,365 std::string GenExternTilingFunc(const ::ascir::FusedScheduledResult &fused_schedule_result,
286 const std::map<std::string, std::string> &shape_info, const std::string tiling,366 const std::map<std::string, std::string> &shape_info, const std::string tiling,
287 const std::string &pgo_dir, const std::string &core_num) const;367 const std::string &pgo_dir, const std::string &core_num) const;
@@ -306,8 +386,7 @@ class TilingLib {
306 std::string GenGetTilingKeyCount(const ::ascir::FusedScheduledResult &fused_schedule_result) const;386 std::string GenGetTilingKeyCount(const ::ascir::FusedScheduledResult &fused_schedule_result) const;
307 std::string GenGetTilingKeyForStatic() const;387 std::string GenGetTilingKeyForStatic() const;
308 std::string GenGetTilingKeyKernelTypeForStatic(const ::ascir::FusedScheduledResult &fused_schedule_result) const;388 std::string GenGetTilingKeyKernelTypeForStatic(const ::ascir::FusedScheduledResult &fused_schedule_result) const;
309- 389+ // codegen_tiling_cube.cpp: Cube/CV tiling and MatMul metadata extraction.
310- // Cube tiling generation is implemented in codegen_tiling_cube.cpp.
311 std::string GenCVTilingFunc() const;390 std::string GenCVTilingFunc() const;
312 std::string GenTilingDataBlockDimAndWss() const;391 std::string GenTilingDataBlockDimAndWss() const;
313 std::map<std::string, std::string> GenerateCVFusionStatic(392 std::map<std::string, std::string> GenerateCVFusionStatic(
@@ -320,6 +399,7 @@ class TilingLib {
320 const ::ascir::FusedScheduledResult &elemwise_schedule_result,399 const ::ascir::FusedScheduledResult &elemwise_schedule_result,
321 const std::map<std::string, std::string> &shape_info, const std::string &pgo_dir,400 const std::map<std::string, std::string> &shape_info, const std::string &pgo_dir,
322 const std::string &core_num) const;401 const std::string &core_num) const;
402+ // codegen_tiling.cpp: Cube/CV entry orchestration shared with the common translation-unit renderer.
323 std::map<std::string, std::string> GenerateCVFusion(const ::ascir::FusedScheduledResult &fused_schedule_result,403 std::map<std::string, std::string> GenerateCVFusion(const ::ascir::FusedScheduledResult &fused_schedule_result,
324 const std::map<std::string, std::string> &shape_info,404 const std::map<std::string, std::string> &shape_info,
325 const std::string &pgo_dir, const std::string &core_num) const;405 const std::string &pgo_dir, const std::string &core_num) const;
Mautofuse/codegen/codegen_tiling_cube.cpp+1-1
@@ -71,8 +71,8 @@ void AppendCvUbFusionStageSizeName(const ascir::FusedScheduledResult &fused_sche
71 }71 }
72 ss << "}" << std::endl;72 ss << "}" << std::endl;
73}73}
74-} // namespace
75 74 
75+} // namespace
76std::string DtypeToStr(ge::DataType dtype) {76std::string DtypeToStr(ge::DataType dtype) {
77 const std::map<ge::DataType, const ge::char_t *> kTypeName = {77 const std::map<ge::DataType, const ge::char_t *> kTypeName = {
78 {ge::DT_FLOAT, "float32"}, {ge::DT_FLOAT16, "float16"}, {ge::DT_BF16, "bfloat16"}, {ge::DT_INT8, "int8"},78 {ge::DT_FLOAT, "float32"}, {ge::DT_FLOAT16, "float16"}, {ge::DT_BF16, "bfloat16"}, {ge::DT_INT8, "int8"},
Aautofuse/codegen/codegen_tiling_inductor_pgo_proxy.cpp+458-0
@@ -0,0 +1,458 @@
1+/**
2+ * Copyright (c) 2026 Huawei Technologies Co., Ltd.
3+ * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4+ * CANN Open Software License Agreement Version 2.0 (the "License").
5+ * Please refer to the License for details. You may not use this file except in compliance with the License.
6+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7+ * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8+ * See LICENSE in the root of the software repository for the full text of the License.
9+ */
10+ 
11+#include "codegen_tiling.h"
12+ 
13+namespace codegen {
14+ 
15+namespace {
16+ 
17+void AppendInductorPgoProxySystemIncludes(std::stringstream &ss) {
18+ ss << R"(
19+#include <spawn.h>
20+#include <signal.h>
21+#include <sys/wait.h>
22+#include <unistd.h>
23+#include <dlfcn.h>
24+#include <climits>
25+#include <algorithm>
26+#include <chrono>
27+#include <cerrno>
28+#include <cstdlib>
29+#include <cstring>
30+#include <fstream>
31+#include <mutex>
32+#include <thread>
33+#include <type_traits>
34+#include <utility>
35+#include "acl/acl.h"
36+#ifndef AUTOFUSE_PGO_GENERATION
37+#define AUTOFUSE_PGO_GENERATION ""
38+#endif
39+#ifndef AUTOFUSE_PGO_RUNNER_TIMEOUT_SECONDS
40+#define AUTOFUSE_PGO_RUNNER_TIMEOUT_SECONDS 1800
41+#endif
42+extern char **environ;
43+)";
44+}
45+ 
46+void AppendInductorPgoProxyDeclarations(std::stringstream &ss, const std::string &tiling) {
47+ ss << "extern \"C\" int64_t GenerateTopnSolutions("
48+ << "const std::vector<std::map<std::string, std::string>> &, int64_t, std::vector<" << tiling
49+ << "> &, std::vector<int64_t> &, std::vector<int64_t> &, ResLimit *);" << std::endl;
50+ ss << "namespace inductor_pgo_fallback {" << std::endl;
51+ ss << "static int64_t GenerateModeledFallbackTopnSolutions("
52+ << "const std::vector<std::map<std::string, std::string>> &, int64_t, std::vector<" << tiling
53+ << "> &, std::vector<int64_t> &, std::vector<int64_t> &, ResLimit *);" << std::endl;
54+ ss << "} // namespace inductor_pgo_fallback" << std::endl;
55+ ss << R"(
56+namespace {
57+constexpr uint32_t kPgoBundleSchemaVersion = 1U;
58+constexpr int64_t kMaxPgoTopn = 1024;
59+constexpr size_t kMaxPgoManifestSize = 1024U * 1024U;
60+constexpr size_t kMaxPgoArtifactSize = 512U * 1024U * 1024U;
61+constexpr size_t kMaxPgoResultSize = 256U * 1024U * 1024U;
62+constexpr char kPgoKernelFormat[] = "aicore_binary_elf_v1";
63+constexpr char kPgoTopnMagic[] = "AUTOFUSE_PGO_TOPN_V1";
64+constexpr uint32_t kPgoTopnProtocolVersion = 1U;
65+constexpr uint32_t kPgoTopnRecordHeaderSize = 32U;
66+constexpr char kPgoGeneration[] = AUTOFUSE_PGO_GENERATION;
67+static_assert(std::is_trivially_copyable<AutofuseTilingData>::value);
68+static_assert(std::is_standard_layout<AutofuseTilingData>::value);
69+struct InductorPgoArtifacts {
70+ std::string tiling_so; std::string generation_dir; std::string runner;
71+ std::string kernel; std::string manifest; std::string ld_preload;
72+};
73+)";
74+}
75+ 
76+} // namespace
77+ 
78+void TilingLib::GenInductorPgoProxyIncludes(std::stringstream &ss, const std::string &tiling) const {
79+ AppendInductorPgoProxySystemIncludes(ss);
80+ AppendInductorPgoProxyDeclarations(ss, tiling);
81+}
82+ 
83+void TilingLib::GenInductorPgoProxyFileHelpers(std::stringstream &ss) const {
84+ ss << R"(
85+bool ReadPgoFile(const std::string &path, size_t limit, std::vector<uint8_t> &data) {
86+ std::ifstream file(path, std::ios::binary | std::ios::ate);
87+ if (!file.is_open()) { return false; }
88+ const auto size = file.tellg();
89+ if (size < 0 || static_cast<uint64_t>(size) > limit) { return false; }
90+ data.resize(static_cast<size_t>(size)); file.seekg(0, std::ios::beg);
91+ return data.empty() || static_cast<bool>(file.read(reinterpret_cast<char *>(data.data()), size));
92+}
93+ 
94+std::string PgoBaseName(const std::string &path) {
95+ const auto pos = path.find_last_of('/');
96+ return pos == std::string::npos ? path : path.substr(pos + 1U);
97+}
98+ 
99+bool GetJsonString(const std::string &json, const std::string &key, size_t begin, std::string &value) {
100+ const std::string token = "\"" + key + "\"";
101+ const auto key_pos = json.find(token, begin);
102+ const auto colon = key_pos == std::string::npos ? key_pos : json.find(':', key_pos + token.size());
103+ const auto quote = colon == std::string::npos ? colon : json.find('"', colon + 1U);
104+ const auto end = quote == std::string::npos ? quote : json.find('"', quote + 1U);
105+ if (end == std::string::npos) { return false; }
106+ value = json.substr(quote + 1U, end - quote - 1U); return true;
107+}
108+ 
109+bool GetJsonUint(const std::string &json, const std::string &key, uint32_t &value) {
110+ const std::string token = "\"" + key + "\""; const auto key_pos = json.find(token);
111+ const auto colon = key_pos == std::string::npos ? key_pos : json.find(':', key_pos + token.size());
112+ if (colon == std::string::npos) { return false; }
113+ char *end = nullptr; errno = 0; const auto parsed = std::strtoul(json.c_str() + colon + 1U, &end, 10);
114+ if (errno != 0 || end == json.c_str() + colon + 1U || parsed > UINT32_MAX) { return false; }
115+ value = static_cast<uint32_t>(parsed); return true;
116+}
117+)";
118+}
119+ 
120+void TilingLib::GenInductorPgoProxySha256(std::stringstream &ss) const {
121+ ss << R"(
122+bool ComputeFileSha256(const std::string &path, std::string &hex) {
123+ std::vector<uint8_t> data;
124+ if (!ReadPgoFile(path, kMaxPgoArtifactSize, data)) { return false; }
125+ void *crypto = dlopen("libcrypto.so.3", RTLD_NOW | RTLD_LOCAL);
126+ if (crypto == nullptr) { crypto = dlopen("libcrypto.so", RTLD_NOW | RTLD_LOCAL); }
127+ if (crypto == nullptr) { return false; }
128+ using Sha256Fn = unsigned char *(*)(const unsigned char *, size_t, unsigned char *);
129+ auto sha256 = reinterpret_cast<Sha256Fn>(dlsym(crypto, "SHA256"));
130+ unsigned char digest[32] = {};
131+ const bool valid = sha256 != nullptr && sha256(data.data(), data.size(), digest) != nullptr;
132+ dlclose(crypto);
133+ if (!valid) { return false; }
134+ constexpr char digits[] = "0123456789abcdef"; hex.resize(64U);
135+ for (size_t i = 0; i < 32U; ++i) {
136+ hex[2U * i] = digits[digest[i] >> 4U]; hex[2U * i + 1U] = digits[digest[i] & 0xFU];
137+ }
138+ return true;
139+}
140+ 
141+bool ValidateArtifactHash(const std::string &json, const std::string &name, const std::string &path) {
142+ const auto section = json.find("\"" + name + "\"");
143+ std::string file_name; std::string expected_hash; std::string actual_hash;
144+ return section != std::string::npos && GetJsonString(json, "file", section, file_name) &&
145+ GetJsonString(json, "sha256", section, expected_hash) && file_name == PgoBaseName(path) &&
146+ expected_hash.size() == 64U && ComputeFileSha256(path, actual_hash) && actual_hash == expected_hash;
147+}
148+)";
149+}
150+ 
151+namespace {
152+ 
153+void AppendInductorPgoManifestValidation(std::stringstream &ss) {
154+ ss << R"(
155+bool ValidateInductorPgoManifest(InductorPgoArtifacts &artifacts) {
156+ std::vector<uint8_t> bytes;
157+ if (!ReadPgoFile(artifacts.manifest, kMaxPgoManifestSize, bytes)) {
158+ OP_LOGE(OP_NAME, "Read Inductor PGO manifest failed: %s", artifacts.manifest.c_str());
159+ return false;
160+ }
161+ const std::string json(bytes.begin(), bytes.end());
162+ std::string generation; std::string ld_preload;
163+ uint32_t bundle_schema_version = 0; uint32_t result_protocol_version = 0;
164+ if (!GetJsonUint(json, "bundle_schema_version", bundle_schema_version) ||
165+ !GetJsonString(json, "generation", 0U, generation) ||
166+ !GetJsonUint(json, "result_protocol_version", result_protocol_version) ||
167+ !GetJsonString(json, "ld_preload", 0U, ld_preload)) {
168+ OP_LOGE(OP_NAME, "Invalid Inductor PGO manifest fields: %s", artifacts.manifest.c_str());
169+ return false;
170+ }
171+ const bool valid = bundle_schema_version == kPgoBundleSchemaVersion && generation == kPgoGeneration &&
172+ result_protocol_version == kPgoTopnProtocolVersion &&
173+ ValidateArtifactHash(json, "tiling_so", artifacts.tiling_so) &&
174+ ValidateArtifactHash(json, "runner", artifacts.runner) &&
175+ ValidateArtifactHash(json, "kernel", artifacts.kernel);
176+ if (!valid) {
177+ OP_LOGE(OP_NAME, "Inductor PGO manifest validation failed: %s", artifacts.manifest.c_str());
178+ return false;
179+ }
180+ artifacts.ld_preload = ld_preload;
181+ return true;
182+}
183+)";
184+}
185+ 
186+void AppendInductorPgoArtifactResolution(std::stringstream &ss) {
187+ ss << R"(
188+bool ResolveInductorPgoArtifactsUncached(InductorPgoArtifacts &artifacts) {
189+ Dl_info info = {};
190+ if (kPgoGeneration[0] == '\0' ||
191+ dladdr(reinterpret_cast<const void *>(&ResolveInductorPgoArtifactsUncached), &info) == 0 ||
192+ info.dli_fname == nullptr) {
193+ OP_LOGE(OP_NAME, "Resolve Inductor PGO artifacts failed: invalid generation or tiling so path");
194+ return false;
195+ }
196+ char real_path[PATH_MAX] = {};
197+ if (realpath(info.dli_fname, real_path) == nullptr) {
198+ OP_LOGE(OP_NAME, "Resolve Inductor PGO tiling so realpath failed: %s", info.dli_fname);
199+ return false;
200+ }
201+ const std::string tiling_entry = real_path;
202+ artifacts.generation_dir = tiling_entry + ".pgo." + kPgoGeneration;
203+ const std::string base = PgoBaseName(tiling_entry);
204+ artifacts.tiling_so = artifacts.generation_dir + "/" + base;
205+ artifacts.runner = artifacts.generation_dir + "/" + base + ".pgo_runner";
206+ artifacts.kernel = artifacts.generation_dir + "/" + base + ".pgo_kernel." + kPgoKernelFormat;
207+ artifacts.manifest = artifacts.generation_dir + "/manifest.json";
208+ if (access(artifacts.tiling_so.c_str(), R_OK) != 0 || access(artifacts.runner.c_str(), X_OK) != 0 ||
209+ access(artifacts.kernel.c_str(), R_OK) != 0) {
210+ OP_LOGE(OP_NAME, "Inductor PGO sidecar missing: tiling=%s runner=%s kernel=%s", artifacts.tiling_so.c_str(),
211+ artifacts.runner.c_str(), artifacts.kernel.c_str());
212+ return false;
213+ }
214+ return ValidateInductorPgoManifest(artifacts);
215+}
216+)";
217+}
218+ 
219+void AppendInductorPgoArtifactCache(std::stringstream &ss) {
220+ ss << R"(
221+bool ResolveInductorPgoArtifacts(InductorPgoArtifacts &artifacts) {
222+ static std::once_flag validation_once;
223+ static InductorPgoArtifacts cached_artifacts;
224+ static bool valid = false;
225+ std::call_once(validation_once, []() {
226+ InductorPgoArtifacts resolved_artifacts;
227+ valid = ResolveInductorPgoArtifactsUncached(resolved_artifacts);
228+ if (valid) { cached_artifacts = std::move(resolved_artifacts); }
229+ });
230+ if (!valid) { return false; }
231+ artifacts = cached_artifacts;
232+ return true;
233+}
234+)";
235+}
236+ 
237+} // namespace
238+ 
239+void TilingLib::GenInductorPgoProxyManifest(std::stringstream &ss) const {
240+ AppendInductorPgoManifestValidation(ss);
241+ AppendInductorPgoArtifactResolution(ss);
242+ AppendInductorPgoArtifactCache(ss);
243+}
244+ 
245+void TilingLib::GenInductorPgoProxyResultParser(std::stringstream &ss, const std::string &tiling) const {
246+ ss << R"(
247+template <typename T>
248+bool ReadPgoResultValue(const std::vector<uint8_t> &data, size_t &offset, T &value) {
249+ if (offset > data.size() || sizeof(T) > data.size() - offset) { return false; }
250+ std::memcpy(&value, data.data() + offset, sizeof(T)); offset += sizeof(T); return true;
251+}
252+ 
253+uint64_t HashProxyTiling(const void *data, size_t size) {
254+ const auto *bytes = static_cast<const uint8_t *>(data); uint64_t hash = 1469598103934665603ULL;
255+ for (size_t i = 0; i < size; ++i) { hash = (hash ^ bytes[i]) * 1099511628211ULL; }
256+ return hash;
257+}
258+)";
259+ ss << "bool ParseInductorPgoResult(const std::string &path, int64_t topn, std::vector<" << tiling
260+ << "> &tiling_datas, std::vector<int64_t> &workspaces, std::vector<int64_t> &block_dims) {" << std::endl;
261+ ss << R"( std::vector<uint8_t> data;
262+ if (!ReadPgoFile(path, kMaxPgoResultSize, data) || data.size() < 40U ||
263+ std::memcmp(data.data(), kPgoTopnMagic, 20U) != 0) { return false; }
264+ size_t offset = 20U; uint32_t version = 0; uint32_t flags = 0; uint32_t count = 0;
265+ uint32_t tiling_size = 0; uint32_t record_header_size = 0;
266+ if (!ReadPgoResultValue(data, offset, version) || !ReadPgoResultValue(data, offset, flags) ||
267+ !ReadPgoResultValue(data, offset, count) || !ReadPgoResultValue(data, offset, tiling_size) ||
268+ !ReadPgoResultValue(data, offset, record_header_size) || version != kPgoTopnProtocolVersion || flags != 0U ||
269+ count == 0U || count > static_cast<uint64_t>(topn) || tiling_size != sizeof(AutofuseTilingData) ||
270+ record_header_size != kPgoTopnRecordHeaderSize) { return false; }
271+)";
272+ ss << " std::vector<" << tiling << "> parsed_tilings; std::vector<int64_t> parsed_workspaces;" << std::endl;
273+ ss << R"( std::vector<int64_t> parsed_block_dims; parsed_tilings.reserve(count);
274+ for (uint32_t i = 0; i < count; ++i) {
275+ uint64_t repr_len = 0; int64_t workspace = 0; int64_t block_dim = 0; uint64_t tiling_hash = 0;
276+ AutofuseTilingData tiling_data = {};
277+ if (!ReadPgoResultValue(data, offset, repr_len) || !ReadPgoResultValue(data, offset, workspace) ||
278+ !ReadPgoResultValue(data, offset, block_dim) || !ReadPgoResultValue(data, offset, tiling_hash) ||
279+ !ReadPgoResultValue(data, offset, tiling_data) || repr_len == 0U || repr_len > 16U * 1024U * 1024U ||
280+ repr_len > data.size() - offset || workspace < 0 || block_dim <= 0 || block_dim > UINT32_MAX) { return false; }
281+ const std::string repr(reinterpret_cast<const char *>(data.data() + offset), repr_len); offset += repr_len;
282+ if (HashProxyTiling(&tiling_data, sizeof(tiling_data)) != tiling_hash ||
283+ GetTilingDataRepr(&tiling_data) != repr || static_cast<int64_t>(GetWorkspaceSize(tiling_data)) != workspace ||
284+ static_cast<int64_t>(tiling_data.get_block_dim()) != block_dim) { return false; }
285+ parsed_tilings.push_back(tiling_data); parsed_workspaces.push_back(workspace); parsed_block_dims.push_back(block_dim);
286+ }
287+ if (offset != data.size()) { return false; }
288+ tiling_datas.swap(parsed_tilings); workspaces.swap(parsed_workspaces); block_dims.swap(parsed_block_dims); return true;
289+}
290+)";
291+}
292+ 
293+namespace {
294+ 
295+void AppendInductorPgoResultPathAndWait(std::stringstream &ss) {
296+ ss << R"(
297+bool MakeInductorPgoResultPath(std::string &path) {
298+ char result_dir_template[] = "/tmp/autofuse_inductor_pgo_XXXXXX";
299+ if (mkdtemp(result_dir_template) == nullptr) { return false; }
300+ path = std::string(result_dir_template) + "/result.bin"; return true;
301+}
302+ 
303+std::string PgoParentPath(const std::string &path) {
304+ const auto pos = path.find_last_of('/');
305+ return pos == std::string::npos ? "." : path.substr(0U, pos);
306+}
307+ 
308+void RemoveInductorPgoResultPath(const std::string &path) {
309+ unlink(path.c_str()); unlink((path + ".tmp").c_str());
310+ rmdir(PgoParentPath(path).c_str());
311+}
312+ 
313+int WaitInductorPgoRunner(pid_t pid) {
314+ constexpr auto timeout = std::chrono::seconds(AUTOFUSE_PGO_RUNNER_TIMEOUT_SECONDS);
315+ const auto deadline = std::chrono::steady_clock::now() + timeout;
316+ int status = 0;
317+ while (std::chrono::steady_clock::now() < deadline) {
318+ const pid_t wait_ret = waitpid(pid, &status, WNOHANG);
319+ if (wait_ret == pid) { return WIFEXITED(status) && WEXITSTATUS(status) == 0 ? 0 : -1; }
320+ if (wait_ret < 0 && errno != EINTR) { return -1; }
321+ std::this_thread::sleep_for(std::chrono::milliseconds(10));
322+ }
323+ kill(pid, SIGKILL); while (waitpid(pid, &status, 0) < 0 && errno == EINTR) {}
324+ return -1;
325+}
326+)";
327+}
328+ 
329+void AppendInductorPgoRunnerEnvironment(std::stringstream &ss) {
330+ ss << R"(
331+std::vector<std::string> BuildInductorPgoRunnerEnv(const std::string &ld_preload) {
332+ std::vector<std::string> env_strings;
333+ bool found_ld_preload = false;
334+ for (char **env = environ; env != nullptr && *env != nullptr; ++env) {
335+ std::string item(*env);
336+ if (item.rfind("LD_PRELOAD=", 0U) != 0U) {
337+ env_strings.push_back(std::move(item));
338+ continue;
339+ }
340+ found_ld_preload = true;
341+ if (ld_preload.empty()) {
342+ env_strings.push_back(std::move(item));
343+ continue;
344+ }
345+ const std::string old_preload = item.substr(std::strlen("LD_PRELOAD="));
346+ env_strings.push_back("LD_PRELOAD=" + ld_preload + (old_preload.empty() ? "" : ":" + old_preload));
347+ }
348+ if (!found_ld_preload && !ld_preload.empty()) {
349+ env_strings.push_back("LD_PRELOAD=" + ld_preload);
350+ }
351+ return env_strings;
352+}
353+ 
354+std::vector<char *> BuildInductorPgoEnvp(std::vector<std::string> &env_strings) {
355+ std::vector<char *> envp;
356+ envp.reserve(env_strings.size() + 1U);
357+ for (auto &env : env_strings) {
358+ envp.push_back(const_cast<char *>(env.c_str()));
359+ }
360+ envp.push_back(nullptr);
361+ return envp;
362+}
363+)";
364+}
365+ 
366+void AppendInductorPgoRunnerSpawn(std::stringstream &ss) {
367+ ss << R"(
368+int SpawnInductorPgoRunner(const InductorPgoArtifacts &artifacts, int32_t device_id, const ResLimit &limit,
369+ int64_t topn, const std::string &result_path) {
370+ const std::string device = std::to_string(device_id); const std::string aiv = std::to_string(limit.aiv_num);
371+ const std::string ub = std::to_string(limit.ub_size); const std::string topn_arg = std::to_string(topn);
372+ std::vector<char *> argv = {const_cast<char *>(artifacts.runner.c_str()), const_cast<char *>(device.c_str()),
373+ const_cast<char *>(aiv.c_str()), const_cast<char *>(ub.c_str()), const_cast<char *>("autofuse"),
374+ const_cast<char *>(artifacts.tiling_so.c_str()), const_cast<char *>(artifacts.kernel.c_str()),
375+ const_cast<char *>(result_path.c_str()), const_cast<char *>(topn_arg.c_str()), nullptr};
376+ auto env_strings = BuildInductorPgoRunnerEnv(artifacts.ld_preload);
377+ auto envp = BuildInductorPgoEnvp(env_strings);
378+ pid_t pid = -1;
379+ const int spawn_ret = posix_spawn(&pid, artifacts.runner.c_str(), nullptr, nullptr, argv.data(), envp.data());
380+ return spawn_ret == 0 ? WaitInductorPgoRunner(pid) : -1;
381+}
382+)";
383+}
384+ 
385+} // namespace
386+ 
387+void TilingLib::GenInductorPgoProxySpawn(std::stringstream &ss) const {
388+ AppendInductorPgoResultPathAndWait(ss);
389+ AppendInductorPgoRunnerEnvironment(ss);
390+ AppendInductorPgoRunnerSpawn(ss);
391+}
392+ 
393+void TilingLib::GenInductorPgoProxyFunction(std::stringstream &ss, const std::string &tiling) const {
394+ ss << "int64_t FallbackToInductorModeledTopn("
395+ << "const std::vector<std::map<std::string, std::string>> &input_configs, int64_t topn, "
396+ << "const ResLimit &limit, std::vector<" << tiling
397+ << "> &tiling_datas, std::vector<int64_t> &workspaces, std::vector<int64_t> &block_dims) {" << std::endl;
398+ ss << R"( OP_LOGW(OP_NAME, "Inductor PGO failed, fallback to modeled TopN");
399+ ResLimit modeled_limit = limit;
400+ return inductor_pgo_fallback::GenerateModeledFallbackTopnSolutions(
401+ input_configs, topn, tiling_datas, workspaces, block_dims, &modeled_limit);
402+}
403+ 
404+)";
405+ ss << "static int64_t RunInductorPgoProxy("
406+ << "const std::vector<std::map<std::string, std::string>> &input_configs, int64_t topn, "
407+ << "std::vector<" << tiling << "> &tiling_datas, std::vector<int64_t> &workspaces, "
408+ << "std::vector<int64_t> &block_dims, ResLimit *res_limit) {" << std::endl;
409+ ss << R"( tiling_datas.clear(); workspaces.clear(); block_dims.clear();
410+ if (topn <= 0 || topn > kMaxPgoTopn) { return -1; }
411+ const ResLimit *limit = (res_limit == nullptr || res_limit->aiv_num == 0U) ? &g_no_limit_res : res_limit;
412+ if (limit->aiv_num == 0U || limit->ub_size <= 256U) { return -1; }
413+ InductorPgoArtifacts artifacts;
414+ if (!ResolveInductorPgoArtifacts(artifacts)) {
415+ return FallbackToInductorModeledTopn(input_configs, topn, *limit, tiling_datas, workspaces, block_dims);
416+ }
417+ int32_t device_id = -1;
418+ if (aclrtGetDevice(&device_id) != ACL_SUCCESS || device_id < 0) {
419+ OP_LOGW(OP_NAME, "Get current device failed for Inductor PGO");
420+ return FallbackToInductorModeledTopn(input_configs, topn, *limit, tiling_datas, workspaces, block_dims);
421+ }
422+ std::string result_path;
423+ if (!MakeInductorPgoResultPath(result_path)) {
424+ OP_LOGW(OP_NAME, "Create Inductor PGO result path failed");
425+ return FallbackToInductorModeledTopn(input_configs, topn, *limit, tiling_datas, workspaces, block_dims);
426+ }
427+ const int runner_ret = SpawnInductorPgoRunner(artifacts, device_id, *limit, topn, result_path);
428+ const bool parsed = runner_ret == 0 &&
429+ ParseInductorPgoResult(result_path, topn, tiling_datas, workspaces, block_dims);
430+ RemoveInductorPgoResultPath(result_path);
431+ if (!parsed) {
432+ OP_LOGW(OP_NAME, "Inductor PGO runner or result parsing failed, runner_ret=%d", runner_ret);
433+ return FallbackToInductorModeledTopn(input_configs, topn, *limit, tiling_datas, workspaces, block_dims);
434+ }
435+ return 0;
436+}
437+} // namespace
438+)";
439+}
440+ 
441+void TilingLib::GenInductorPgoProxyEntry(std::stringstream &ss, const std::string &tiling) const {
442+ GenInductorPgoProxyIncludes(ss, tiling);
443+ GenInductorPgoProxyFileHelpers(ss);
444+ GenInductorPgoProxySha256(ss);
445+ GenInductorPgoProxyManifest(ss);
446+ GenInductorPgoProxyResultParser(ss, tiling);
447+ GenInductorPgoProxySpawn(ss);
448+ GenInductorPgoProxyFunction(ss, tiling);
449+ ss << "extern \"C\" int64_t GenerateTopnSolutions("
450+ << "const std::vector<std::map<std::string, std::string>> &input_configs, int64_t topn, "
451+ << "std::vector<" << tiling << "> &tiling_datas, std::vector<int64_t> &workspaces, "
452+ << "std::vector<int64_t> &block_dims, ResLimit *res_limit) {" << std::endl;
453+ ss << " return RunInductorPgoProxy(input_configs, topn, tiling_datas, workspaces, block_dims," << std::endl;
454+ ss << " res_limit);" << std::endl;
455+ ss << "}" << std::endl;
456+}
457+ 
458+} // namespace codegen
Aautofuse/codegen/codegen_tiling_inductor_pgo_runner.cpp+412-0
@@ -0,0 +1,412 @@
1+/**
2+ * Copyright (c) 2026 Huawei Technologies Co., Ltd.
3+ * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4+ * CANN Open Software License Agreement Version 2.0 (the "License").
5+ * Please refer to the License for details. You may not use this file except in compliance with the License.
6+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7+ * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8+ * See LICENSE in the root of the software repository for the full text of the License.
9+ */
10+ 
11+#include "codegen_tiling.h"
12+ 
13+namespace codegen {
14+ 
15+void TilingLib::GenInductorPgoHostLoader(std::stringstream &ss) const {
16+ ss << R"(
17+void *g_pgo_tiling_handle = nullptr;
18+GenerateMeasuredTopnSolutionsType generate_measured_topn_solutions_fn = nullptr;
19+SetTopnPgoContextType set_topn_pgo_context_fn = nullptr;
20+ClearTopnPgoContextType clear_topn_pgo_context_fn = nullptr;
21+GetTilingDataReprType get_tiling_data_repr_fn = nullptr;
22+ 
23+template <typename T>
24+bool LoadInductorPgoSymbol(T &target, const char *name) {
25+ target = reinterpret_cast<T>(dlsym(g_pgo_tiling_handle, name));
26+ if (target == nullptr) { DLOGE("dlsym %s failed: %s", name, dlerror()); return false; }
27+ return true;
28+}
29+ 
30+int LoadInductorPgoHost(const InductorPgoRunnerArgs &args) {
31+ g_pgo_tiling_handle = dlopen(args.tiling_file.c_str(), RTLD_NOW | RTLD_LOCAL);
32+ if (g_pgo_tiling_handle == nullptr) { DLOGE("dlopen tiling failed: %s", dlerror()); return FAILED; }
33+ bool valid = LoadInductorPgoSymbol(generate_measured_topn_solutions_fn, "GenerateMeasuredTopnSolutions") &&
34+ LoadInductorPgoSymbol(set_topn_pgo_context_fn, "SetTopnPgoContext") &&
35+ LoadInductorPgoSymbol(clear_topn_pgo_context_fn, "ClearTopnPgoContext") &&
36+ LoadInductorPgoSymbol(get_tiling_data_repr_fn, "GetTilingDataRepr") &&
37+ LoadInductorPgoSymbol(get_tiling_key_count_fn, "GetTilingKeyCount") &&
38+ LoadInductorPgoSymbol(find_best_tiling_key_fn, "FindBestTilingKey");
39+ return valid ? SUCCESS : FAILED;
40+}
41+ 
42+void UnloadInductorPgoHost() {
43+ generate_measured_topn_solutions_fn = nullptr; set_topn_pgo_context_fn = nullptr;
44+ clear_topn_pgo_context_fn = nullptr; get_tiling_data_repr_fn = nullptr;
45+ get_tiling_key_count_fn = nullptr; find_best_tiling_key_fn = nullptr;
46+ if (g_pgo_tiling_handle != nullptr) { dlclose(g_pgo_tiling_handle); g_pgo_tiling_handle = nullptr; }
47+}
48+)" << std::endl;
49+}
50+ 
51+void TilingLib::GenInductorPgoResultProtocol(std::stringstream &ss) const {
52+ GenInductorPgoRecordWriter(ss);
53+ GenInductorPgoResultWriter(ss);
54+ GenPgoSaveTilingKey(ss);
55+ GenInductorPgoSearchWriter(ss);
56+ GenInductorPgoArgValidators(ss);
57+ GenInductorPgoArgParser(ss);
58+ GenInductorPgoContextGuard(ss);
59+}
60+ 
61+void TilingLib::GenInductorPgoResultTypes(std::stringstream &ss) const {
62+ ss << R"(
63+namespace {
64+constexpr char kPgoTopnMagic[] = "AUTOFUSE_PGO_TOPN_V1";
65+constexpr size_t kPgoTopnMagicSize = 20U;
66+constexpr uint32_t kPgoTopnProtocolVersion = 1U;
67+constexpr uint32_t kPgoTopnProtocolFlags = 0U;
68+constexpr uint32_t kPgoTopnRecordHeaderSize = 32U;
69+constexpr size_t kMaxPgoReprSize = 16U * 1024U * 1024U;
70+constexpr int64_t kMaxPgoTopn = 1024;
71+static_assert(sizeof(kPgoTopnMagic) - 1U == kPgoTopnMagicSize);
72+static_assert(std::is_trivially_copyable<AutofuseTilingData>::value);
73+static_assert(std::is_standard_layout<AutofuseTilingData>::value);
74+ 
75+using GenerateMeasuredTopnSolutionsType = int64_t (*)(
76+ const std::vector<std::map<std::string, std::string>> &, int64_t,
77+ std::vector<AutofuseTilingData> &, std::vector<int64_t> &, std::vector<int64_t> &, ResLimit *);
78+using InductorPgoProfilingCallback = long int (*)(
79+ PgoTensorArgs *, void *, uint32_t, AutofuseTilingData *, double *);
80+using InductorPgoProfilingBatchCallback = long int (*)(
81+ PgoTensorArgs *, void *, uint32_t, std::vector<AutofuseTilingDataPerf> *);
82+using SetTopnPgoContextType = int64_t (*)(
83+ PgoTensorArgs *, void *, InductorPgoProfilingCallback, InductorPgoProfilingBatchCallback,
84+ std::vector<AutofuseTilingDataPerf> *);
85+using ClearTopnPgoContextType = void (*)(void);
86+using GetTilingDataReprType = std::string (*)(const AutofuseTilingData *);
87+ 
88+struct InductorPgoRunnerArgs {
89+ int32_t device_id = -1;
90+ uint32_t aiv_num = 0;
91+ uint32_t ub_size = 0;
92+ std::string kernel_name;
93+ std::string tiling_file;
94+ std::string kernel_file;
95+ std::string result_file;
96+ int64_t topn = 0;
97+};
98+)" << std::endl;
99+}
100+ 
101+void TilingLib::GenInductorPgoRecordWriter(std::stringstream &ss) const {
102+ ss << R"(
103+template <typename T>
104+bool WritePgoValue(std::ofstream &out, const T &value) {
105+ out.write(reinterpret_cast<const char *>(&value), sizeof(value));
106+ return out.good();
107+}
108+ 
109+uint64_t HashPgoBytes(const void *data, size_t size) {
110+ const auto *bytes = static_cast<const uint8_t *>(data);
111+ uint64_t hash = 1469598103934665603ULL;
112+ for (size_t i = 0; i < size; ++i) { hash = (hash ^ bytes[i]) * 1099511628211ULL; }
113+ return hash;
114+}
115+ 
116+bool WritePgoRecord(std::ofstream &out, const AutofuseTilingData &tiling_data,
117+ int64_t workspace, int64_t block_dim) {
118+ if (get_tiling_data_repr_fn == nullptr) { return false; }
119+ const std::string repr = get_tiling_data_repr_fn(&tiling_data);
120+ if (repr.empty() || repr.size() > kMaxPgoReprSize || workspace < 0 || block_dim <= 0 ||
121+ block_dim > UINT32_MAX) {
122+ return false;
123+ }
124+ const uint64_t repr_len = repr.size();
125+ const uint64_t tiling_hash = HashPgoBytes(&tiling_data, sizeof(tiling_data));
126+ return WritePgoValue(out, repr_len) && WritePgoValue(out, workspace) &&
127+ WritePgoValue(out, block_dim) && WritePgoValue(out, tiling_hash) &&
128+ WritePgoValue(out, tiling_data) &&
129+ static_cast<bool>(out.write(repr.data(), static_cast<std::streamsize>(repr.size())));
130+}
131+)" << std::endl;
132+}
133+ 
134+void TilingLib::GenInductorPgoResultWriter(std::stringstream &ss) const {
135+ ss << R"(
136+int WritePgoTopnResult(const std::string &path, const std::vector<AutofuseTilingData> &tiling_datas,
137+ const std::vector<int64_t> &workspaces, const std::vector<int64_t> &block_dims) {
138+ if (tiling_datas.empty() || tiling_datas.size() != workspaces.size() || tiling_datas.size() != block_dims.size() ||
139+ tiling_datas.size() > std::numeric_limits<uint32_t>::max()) {
140+ return FAILED;
141+ }
142+ const std::string tmp_path = path + ".tmp";
143+ std::ofstream out(tmp_path, std::ios::binary | std::ios::trunc);
144+ const uint32_t count = static_cast<uint32_t>(tiling_datas.size());
145+ const uint32_t tiling_size = sizeof(AutofuseTilingData);
146+ out.write(kPgoTopnMagic, kPgoTopnMagicSize);
147+ if (!out.good() || !WritePgoValue(out, kPgoTopnProtocolVersion) || !WritePgoValue(out, kPgoTopnProtocolFlags) ||
148+ !WritePgoValue(out, count) || !WritePgoValue(out, tiling_size) ||
149+ !WritePgoValue(out, kPgoTopnRecordHeaderSize)) {
150+ out.close(); std::remove(tmp_path.c_str()); return FAILED;
151+ }
152+ for (size_t i = 0; i < tiling_datas.size(); ++i) {
153+ if (!WritePgoRecord(out, tiling_datas[i], workspaces[i], block_dims[i])) {
154+ out.close(); std::remove(tmp_path.c_str()); return FAILED;
155+ }
156+ }
157+ out.flush();
158+ if (!out.good()) { out.close(); std::remove(tmp_path.c_str()); return FAILED; }
159+ out.close();
160+ const int fd = ::open(tmp_path.c_str(), O_RDONLY);
161+ if (fd < 0 || ::fsync(fd) != 0) {
162+ if (fd >= 0) { ::close(fd); }
163+ std::remove(tmp_path.c_str()); return FAILED;
164+ }
165+ ::close(fd);
166+ if (std::rename(tmp_path.c_str(), path.c_str()) != 0) { std::remove(tmp_path.c_str()); return FAILED; }
167+ return SUCCESS;
168+}
169+)" << std::endl;
170+}
171+ 
172+void TilingLib::GenInductorPgoSearchWriter(std::stringstream &ss) const {
173+ ss << R"(
174+std::string PgoParentPath(const std::string &path) {
175+ const auto pos = path.find_last_of('/');
176+ return pos == std::string::npos ? "." : path.substr(0U, pos);
177+}
178+ 
179+int PublishPgoSearchFile(const std::string &tmp_path, const std::string &path) {
180+ const int fd = ::open(tmp_path.c_str(), O_RDONLY);
181+ if (fd < 0 || ::fsync(fd) != 0) {
182+ if (fd >= 0) { ::close(fd); }
183+ std::remove(tmp_path.c_str());
184+ return FAILED;
185+ }
186+ ::close(fd);
187+ if (std::rename(tmp_path.c_str(), path.c_str()) != 0) {
188+ std::remove(tmp_path.c_str());
189+ return FAILED;
190+ }
191+ return SUCCESS;
192+}
193+ 
194+int WritePgoSearchResult(const InductorPgoRunnerArgs &args,
195+ const std::vector<AutofuseTilingDataPerf> &measured_candidates) {
196+ if (measured_candidates.empty()) { return FAILED; }
197+ const std::string path = PgoParentPath(args.kernel_file) + "/" + std::string(PGO_GRAPH_NAME) + "_search.txt";
198+ const std::string tmp_path = path + ".tmp." + std::to_string(getpid());
199+ std::ofstream out(tmp_path, std::ios::out | std::ios::trunc);
200+ if (!out.is_open()) { return FAILED; }
201+ for (const auto &candidate : measured_candidates) {
202+ PgoSaveTilingKey(candidate.tiling_data, candidate.best_perf, out);
203+ if (!out.good()) { out.close(); std::remove(tmp_path.c_str()); return FAILED; }
204+ }
205+ out.flush();
206+ if (!out.good()) { out.close(); std::remove(tmp_path.c_str()); return FAILED; }
207+ out.close();
208+ return PublishPgoSearchFile(tmp_path, path);
209+}
210+)" << std::endl;
211+}
212+ 
213+void TilingLib::GenInductorPgoArgValidators(std::stringstream &ss) const {
214+ ss << R"(
215+bool ParseRunnerInteger(const char *text, int64_t min_value, int64_t max_value, int64_t &value) {
216+ if (text == nullptr || *text == '\0') { return false; }
217+ errno = 0;
218+ char *end = nullptr;
219+ const long long parsed = std::strtoll(text, &end, 10);
220+ if (errno != 0 || end == text || *end != '\0' || parsed < min_value || parsed > max_value) { return false; }
221+ value = static_cast<int64_t>(parsed);
222+ return true;
223+}
224+ 
225+bool IsValidKernelName(const std::string &name) {
226+ return !name.empty() && std::all_of(name.begin(), name.end(), [](unsigned char c) {
227+ return std::isalnum(c) != 0 || c == '_';
228+ });
229+}
230+)" << std::endl;
231+}
232+ 
233+void TilingLib::GenInductorPgoArgParser(std::stringstream &ss) const {
234+ ss << R"(
235+int ParseInductorPgoRunnerArgs(int argc, char *argv[], InductorPgoRunnerArgs &args) {
236+ if (argc != 9) {
237+ DLOGE("Usage: %s <device_id> <aiv_num> <ub_size> <kernel_name> <tiling_so> <dynamic_kernel> <result> <topn>",
238+ argv[0]);
239+ return FAILED;
240+ }
241+ int64_t device_id = 0;
242+ int64_t aiv_num = 0;
243+ int64_t ub_size = 0;
244+ if (!ParseRunnerInteger(argv[1], 0, INT32_MAX, device_id) ||
245+ !ParseRunnerInteger(argv[2], 1, UINT32_MAX, aiv_num) ||
246+ !ParseRunnerInteger(argv[3], 257, UINT32_MAX, ub_size) ||
247+ !ParseRunnerInteger(argv[8], 1, kMaxPgoTopn, args.topn)) {
248+ DLOGE("invalid numeric runner argument"); return FAILED;
249+ }
250+ args.device_id = static_cast<int32_t>(device_id);
251+ args.aiv_num = static_cast<uint32_t>(aiv_num);
252+ args.ub_size = static_cast<uint32_t>(ub_size);
253+ args.kernel_name = argv[4]; args.tiling_file = argv[5];
254+ args.kernel_file = argv[6]; args.result_file = argv[7];
255+ if (!IsValidKernelName(args.kernel_name) || args.tiling_file.empty() || args.kernel_file.empty() ||
256+ args.result_file.empty() || ::access(args.tiling_file.c_str(), R_OK) != 0 ||
257+ ::access(args.kernel_file.c_str(), R_OK) != 0) {
258+ DLOGE("invalid runner path or kernel name"); return FAILED;
259+ }
260+ return SUCCESS;
261+}
262+)" << std::endl;
263+}
264+ 
265+void TilingLib::GenInductorPgoContextGuard(std::stringstream &ss) const {
266+ ss << R"(
267+class PgoContextGuard {
268+ public:
269+ explicit PgoContextGuard(std::vector<AutofuseTilingDataPerf> *measured_candidates) {
270+ if (set_topn_pgo_context_fn != nullptr) {
271+ valid_ = set_topn_pgo_context_fn(&g_pgo_tensor_args, g_stream, PGOGetProfiling, PGOGetProfilingBatch,
272+ measured_candidates) == 0;
273+ }
274+ }
275+ ~PgoContextGuard() {
276+ if (valid_ && clear_topn_pgo_context_fn != nullptr) { clear_topn_pgo_context_fn(); }
277+ }
278+ bool IsValid() const { return valid_; }
279+ PgoContextGuard(const PgoContextGuard &) = delete;
280+ PgoContextGuard &operator=(const PgoContextGuard &) = delete;
281+ private:
282+ bool valid_ = false;
283+};
284+} // namespace
285+)" << std::endl;
286+}
287+ 
288+void TilingLib::GenInductorPgoRuntime(const ascir::FusedScheduledResult &fused_schedule_result,
289+ std::stringstream &ss) const {
290+ GenInductorPgoAclRuntime(ss);
291+ GenInductorPgoMemoryRuntime(fused_schedule_result, ss);
292+ GenInductorPgoDeinitRuntime(fused_schedule_result, ss);
293+}
294+ 
295+void TilingLib::GenInductorPgoAclRuntime(std::stringstream &ss) const {
296+ ss << R"(
297+int InitInductorPgoAcl(const InductorPgoRunnerArgs &args) {
298+ g_res_limit.aiv_num = args.aiv_num;
299+ g_res_limit.ub_size = args.ub_size;
300+ auto ret = aclInit(nullptr);
301+ if (ret != ACL_SUCCESS) { DLOGE("acl init failed, ERROR: %d", ret); return FAILED; }
302+ g_acl_initialized = true;
303+ ret = aclrtSetDevice(args.device_id);
304+ if (ret != ACL_SUCCESS) { DLOGE("acl set device failed, ERROR: %d", ret); return FAILED; }
305+ g_device_id = args.device_id;
306+ g_device_set = true;
307+ ret = aclrtCreateStream(&g_stream);
308+ if (ret != ACL_SUCCESS) { DLOGE("acl create stream failed, ERROR: %d", ret); return FAILED; }
309+ return SUCCESS;
310+}
311+)" << std::endl;
312+}
313+ 
314+void TilingLib::GenInductorPgoMemoryRuntime(const ascir::FusedScheduledResult &fused_schedule_result,
315+ std::stringstream &ss) const {
316+ ss << R"(
317+int InitInductorPgoMemory() {
318+ aclError ret = ACL_SUCCESS;
319+)";
320+ ss << PGOSearchTensorMallocDef(fused_schedule_result);
321+ ss << PGOSearchTensorArgsUpdateDef(fused_schedule_result);
322+ ss << R"( ret = LaunchParamsInit(&g_pgo_tensor_args);
323+ if (ret != ACL_SUCCESS) { return FAILED; }
324+ return SUCCESS;
325+}
326+)" << std::endl;
327+}
328+ 
329+void TilingLib::GenInductorPgoDeinitRuntime(const ascir::FusedScheduledResult &fused_schedule_result,
330+ std::stringstream &ss) const {
331+ ss << R"(
332+void DeInitInductorPgoMemory() {
333+ aclError ret = ACL_SUCCESS;
334+ if (g_workspace != nullptr) {
335+ ret = aclrtFree(g_workspace);
336+ if (ret != ACL_SUCCESS) { DLOGW("acl free workspace failed, ERROR: %d", ret); }
337+ g_workspace = nullptr;
338+ }
339+ PgoBinaryDeInit();
340+ LaunchParamsDeInit();
341+)";
342+ ss << PGOSearchTensorFreeDef(fused_schedule_result);
343+ ss << R"(}
344+ 
345+void DeInitInductorPgoAcl() {
346+ if (g_stream != nullptr) {
347+ auto ret = aclrtDestroyStream(g_stream);
348+ if (ret != ACL_SUCCESS) { DLOGW("acl destroy stream failed, ERROR: %d", ret); }
349+ g_stream = nullptr;
350+ }
351+ if (g_device_set) {
352+ auto ret = aclrtResetDevice(g_device_id);
353+ if (ret != ACL_SUCCESS) { DLOGW("acl reset device failed, ERROR: %d", ret); }
354+ g_device_set = false;
355+ }
356+ if (g_acl_initialized) {
357+ auto ret = aclFinalize();
358+ if (ret != ACL_SUCCESS) { DLOGW("acl finalize failed, ERROR: %d", ret); }
359+ g_acl_initialized = false;
360+ }
361+}
362+ 
363+void DeInitInductorPgoRuntime() {
364+ DeInitInductorPgoMemory();
365+ DeInitInductorPgoAcl();
366+}
367+)" << std::endl;
368+}
369+ 
370+void TilingLib::GenInductorPgoMain(std::stringstream &ss) const {
371+ ss << R"(
372+int RunInductorPgo(const InductorPgoRunnerArgs &args) {
373+ std::vector<AutofuseTilingDataPerf> measured_candidates;
374+ PgoContextGuard context_guard(&measured_candidates);
375+ if (!context_guard.IsValid() || generate_measured_topn_solutions_fn == nullptr) { return FAILED; }
376+ std::vector<AutofuseTilingData> tiling_datas;
377+ std::vector<int64_t> workspaces;
378+ std::vector<int64_t> block_dims;
379+ const auto ret = generate_measured_topn_solutions_fn(
380+ {}, args.topn, tiling_datas, workspaces, block_dims, &g_res_limit);
381+ if (ret != 0) { DLOGE("GenerateMeasuredTopnSolutions failed, ERROR: %" PRId64, ret); return FAILED; }
382+ if (tiling_datas.empty()) { return FAILED; }
383+ if (WritePgoTopnResult(args.result_file, tiling_datas, workspaces, block_dims) != SUCCESS) {
384+ DLOGE("Write PGO TopN result failed"); return FAILED;
385+ }
386+ if (WritePgoSearchResult(args, measured_candidates) != SUCCESS) {
387+ DLOGW("Write PGO search result failed");
388+ }
389+ return SUCCESS;
390+}
391+ 
392+int main(int argc, char *argv[]) {
393+ InductorPgoRunnerArgs args;
394+ if (ParseInductorPgoRunnerArgs(argc, argv, args) != SUCCESS) { return FAILED; }
395+ g_kernel_o_file = args.kernel_file;
396+ DLOGI("execute info: device_id: %d, graph_name: %s", args.device_id, args.kernel_name.c_str());
397+ const char *tmp_dir = std::getenv("TMPDIR");
398+ const std::string lock_dir = (tmp_dir != nullptr && *tmp_dir != '\0') ? tmp_dir : "/tmp";
399+ g_npu_lock_file = lock_dir + "/autofuse_pgo_npu_lock_" + std::to_string(args.device_id) + ".lock";
400+ CardLock lock(g_npu_lock_file.c_str());
401+ int ret = LoadInductorPgoHost(args);
402+ if (ret == SUCCESS) { ret = InitInductorPgoAcl(args); }
403+ if (ret == SUCCESS) { ret = InitInductorPgoMemory(); }
404+ if (ret == SUCCESS) { ret = RunInductorPgo(args); }
405+ DeInitInductorPgoRuntime();
406+ UnloadInductorPgoHost();
407+ return ret;
408+}
409+)" << std::endl;
410+}
411+ 
412+} // namespace codegen
Mautofuse/codegen/codegen_tiling_inductor_topn.cpp+274-58
@@ -28,6 +28,83 @@ void AppendTopnEntryInitialization(std::stringstream &ss) {
28}28}
29} // namespace29} // namespace
30 30 
31+void TilingLib::GenInductorTopnSources(const ascir::FusedScheduledResult &fused_schedule_result, std::stringstream &ss,
32+ std::map<std::string, std::string> &tiling_file_name_to_content) const {
33+ ss << GenCandidateSolutionProtocolForInductor("AutofuseTilingData") << std::endl;
34+ ss << (enable_autofuse_pgo_ ? GenMeasuredTopnSelectorHelpersForInductor() : GenTopnSelectorHelpersForInductor())
35+ << std::endl;
36+ if (!enable_autofuse_pgo_) {
37+ ss << GenInductorConfigParserForInductor() << std::endl;
38+ }
39+ ss << GenGetTilingKeyCount(fused_schedule_result) << std::endl;
40+ if (!ascgen_utils::IsSingleGroup(fused_schedule_result)) {
41+ ss << GenUpdateCurPerfAndBlockByGroupHelper() << std::endl;
42+ }
43+ if (enable_autofuse_pgo_) {
44+ ss << GenFindBestTilingKeyFunc(fused_schedule_result, "AutofuseTilingData") << std::endl;
45+ ss << GenTopnPgoContextAbi() << std::endl;
46+ } else {
47+ ss << GenEvaluateModeledPerfForInductor("AutofuseTilingData", fused_schedule_result) << std::endl;
48+ }
49+ ss << GenGetTopnSolutionsFuncForInductor(fused_schedule_result, "AutofuseTilingData", enable_autofuse_pgo_)
50+ << std::endl;
51+ ss << GenGetTilingDataReprFuncForInductor(fused_schedule_result, "AutofuseTilingData") << std::endl;
52+ if (enable_autofuse_pgo_) {
53+ ss << GenModeledFallbackTopnForInductor(fused_schedule_result, "AutofuseTilingData") << std::endl;
54+ tiling_file_name_to_content[kPgoRunnerIdentify] = GenInductorPgoRunner(fused_schedule_result);
55+ }
56+}
57+ 
58+std::string TilingLib::GenModeledFallbackTopnForInductor(const ascir::FusedScheduledResult &fused_schedule_result,
59+ const std::string &tiling) const {
60+ std::stringstream ss;
61+ ss << "namespace inductor_pgo_fallback {" << std::endl;
62+ ss << GenCandidateSolutionProtocolForInductor(tiling) << std::endl;
63+ ss << GenTopnSelectorHelpersForInductor() << std::endl;
64+ ss << GenInductorConfigParserForInductor() << std::endl;
65+ if (!ascgen_utils::IsSingleGroup(fused_schedule_result)) {
66+ ss << GenUpdateCurPerfAndBlockByGroupHelper() << std::endl;
67+ }
68+ ss << GenEvaluateModeledPerfForInductor(tiling, fused_schedule_result) << std::endl;
69+ ss << GenGetTopnSolutionsFuncForInductor(fused_schedule_result, tiling, false,
70+ "static int64_t GenerateModeledFallbackTopnSolutions")
71+ << std::endl;
72+ ss << "} // namespace inductor_pgo_fallback" << std::endl;
73+ return ss.str();
74+}
75+ 
76+std::string TilingLib::GenTopnPgoContextAbi() const {
77+ return R"(
78+extern "C" int64_t SetTopnPgoContext(PgoTensorArgs *tensor_args, void *stream,
79+ ProfilingCallback single_callback,
80+ ProfilingBatchCallback batch_callback,
81+ std::vector<AutofuseTilingDataPerf> *measured_candidates) {
82+ auto &config = PgoConfig::Instance();
83+ if (tensor_args == nullptr || stream == nullptr || single_callback == nullptr || batch_callback == nullptr ||
84+ measured_candidates == nullptr ||
85+ config.tensor_args != nullptr || config.stream != nullptr || config.single_callback != nullptr ||
86+ config.batch_callback != nullptr || config.measured_candidates != nullptr) {
87+ return -1;
88+ }
89+ config.tensor_args = tensor_args;
90+ config.stream = stream;
91+ PgoConfig::Instance().single_callback = single_callback;
92+ PgoConfig::Instance().batch_callback = batch_callback;
93+ PgoConfig::Instance().measured_candidates = measured_candidates;
94+ return 0;
95+}
96+ 
97+extern "C" void ClearTopnPgoContext() {
98+ auto &config = PgoConfig::Instance();
99+ config.batch_callback = nullptr;
100+ config.single_callback = nullptr;
101+ config.stream = nullptr;
102+ config.tensor_args = nullptr;
103+ config.measured_candidates = nullptr;
104+}
105+)";
106+}
107+ 
31void TilingLib::GenReprScheduleGroupFields(std::stringstream &ss, const ascir::ScheduleGroup &sg,108void TilingLib::GenReprScheduleGroupFields(std::stringstream &ss, const ascir::ScheduleGroup &sg,
32 const std::string &field_prefix, const std::string &emit_fn,109 const std::string &field_prefix, const std::string &emit_fn,
33 const std::string &indent, bool emit_first_arg) const {110 const std::string &indent, bool emit_first_arg) const {
@@ -271,7 +348,8 @@ void TilingLib::GenGroupPerfForScheduleResult(std::stringstream &ss, size_t asc_
271}348}
272 349 
273std::string TilingLib::GenGetTopnSolutionsFuncForInductor(const ascir::FusedScheduledResult &fused_schedule_result,350std::string TilingLib::GenGetTopnSolutionsFuncForInductor(const ascir::FusedScheduledResult &fused_schedule_result,
274- const std::string &tiling) const {351+ const std::string &tiling, bool use_measured_perf,
352+ const std::string &entry_declaration) const {
275 std::stringstream ss;353 std::stringstream ss;
276 codegen::PgoShapeStringStream pgo_shape_dim;354 codegen::PgoShapeStringStream pgo_shape_dim;
277 int symbol_value_count = 0;355 int symbol_value_count = 0;
@@ -285,13 +363,22 @@ std::string TilingLib::GenGetTopnSolutionsFuncForInductor(const ascir::FusedSche
285 }363 }
286 }364 }
287 365 
288- GenTopnGetTilingFunc(ss, fused_schedule_result, tiling, symbol_value_count);366+ GenTopnGetTilingFunc(ss, fused_schedule_result, tiling, symbol_value_count, use_measured_perf);
289- GenGenerateTopnSolutionsEntry(ss, fused_schedule_result, tiling, pgo_shape_dim);367+ const std::string resolved_entry_declaration =
368+ entry_declaration.empty()
369+ ? "extern \"C\" int64_t " +
370+ std::string(use_measured_perf ? "GenerateMeasuredTopnSolutions" : "GenerateTopnSolutions")
371+ : entry_declaration;
372+ GenGenerateTopnSolutionsEntry(ss, fused_schedule_result, tiling, pgo_shape_dim, resolved_entry_declaration);
373+ if (use_measured_perf) {
374+ GenInductorPgoProxyEntry(ss, tiling);
375+ }
290 return ss.str();376 return ss.str();
291}377}
292 378 
293void TilingLib::GenTopnInitSearchTiling(std::stringstream &ss, const ascir::FusedScheduledResult &fused_schedule_result,379void TilingLib::GenTopnInitSearchTiling(std::stringstream &ss, const ascir::FusedScheduledResult &fused_schedule_result,
294- const std::string &tiling, int symbol_value_count) const {380+ const std::string &tiling, int symbol_value_count,
381+ bool use_measured_perf) const {
295 ss << " const ResLimit *limit = (request.res_limit == nullptr || request.res_limit->aiv_num == 0) "382 ss << " const ResLimit *limit = (request.res_limit == nullptr || request.res_limit->aiv_num == 0) "
296 << "? &g_no_limit_res : request.res_limit;" << std::endl;383 << "? &g_no_limit_res : request.res_limit;" << std::endl;
297 ss << " if (request.symbol_values.size() != " << symbol_value_count << "ULL) {" << std::endl;384 ss << " if (request.symbol_values.size() != " << symbol_value_count << "ULL) {" << std::endl;
@@ -299,8 +386,12 @@ void TilingLib::GenTopnInitSearchTiling(std::stringstream &ss, const ascir::Fuse
299 ss << " return -1;" << std::endl;386 ss << " return -1;" << std::endl;
300 ss << " }" << std::endl;387 ss << " }" << std::endl;
301 ss << std::endl;388 ss << std::endl;
389+ if (use_measured_perf) {
390+ ss << " const uint32_t measured_aiv_num = std::min(limit->aiv_num, g_no_limit_res.aiv_num);" << std::endl;
391+ }
302 ss << " " << tiling << " search_tiling = {};" << std::endl;392 ss << " " << tiling << " search_tiling = {};" << std::endl;
303- ss << " search_tiling.set_block_dim(limit->aiv_num);" << std::endl;393+ ss << " search_tiling.set_block_dim(" << (use_measured_perf ? "measured_aiv_num" : "limit->aiv_num") << ");"
394+ << std::endl;
304 ss << " search_tiling.set_ub_size(limit->ub_size - 256);" << std::endl;395 ss << " search_tiling.set_ub_size(limit->ub_size - 256);" << std::endl;
305 {396 {
306 int idx = 0;397 int idx = 0;
@@ -318,7 +409,7 @@ void TilingLib::GenTopnInitSearchTiling(std::stringstream &ss, const ascir::Fuse
318}409}
319 410 
320void TilingLib::GenTopnGetTilingFunc(std::stringstream &ss, const ascir::FusedScheduledResult &fused_schedule_result,411void TilingLib::GenTopnGetTilingFunc(std::stringstream &ss, const ascir::FusedScheduledResult &fused_schedule_result,
321- const std::string &tiling, int symbol_value_count) const {412+ const std::string &tiling, int symbol_value_count, bool use_measured_perf) const {
322 ss << "static int64_t GetTopnCandidateSolutions(const GetTilingRequest &request, GetTilingResponse &response) {"413 ss << "static int64_t GetTopnCandidateSolutions(const GetTilingRequest &request, GetTilingResponse &response) {"
323 << std::endl;414 << std::endl;
324 ss << " response.candidate_solutions.clear();" << std::endl;415 ss << " response.candidate_solutions.clear();" << std::endl;
@@ -331,29 +422,34 @@ void TilingLib::GenTopnGetTilingFunc(std::stringstream &ss, const ascir::FusedSc
331 ss << " return -1;" << std::endl;422 ss << " return -1;" << std::endl;
332 ss << " }" << std::endl;423 ss << " }" << std::endl;
333 424 
334- GenTopnInitSearchTiling(ss, fused_schedule_result, tiling, symbol_value_count);425+ GenTopnInitSearchTiling(ss, fused_schedule_result, tiling, symbol_value_count, use_measured_perf);
335 GenTopnDefaultTiling(ss, tiling);426 GenTopnDefaultTiling(ss, tiling);
336 427 
337- ss << " const bool internal_no_config_path = (request.input_configs == nullptr);" << std::endl;428+ if (use_measured_perf) {
338- ss << " const bool explicit_no_config_path = request.input_configs != nullptr && request.input_configs->size() == 1 "429+ ss << " (void)request.input_configs;" << std::endl;
339- << "&& request.input_configs->front().empty();" << std::endl;430+ } else {
340- ss << " const bool original_config_path = internal_no_config_path || explicit_no_config_path;" << std::endl;431+ ss << " const bool internal_no_config_path = (request.input_configs == nullptr);" << std::endl;
341- ss << " std::vector<SearchConfig> configs;" << std::endl;432+ ss << " const bool explicit_no_config_path = request.input_configs != nullptr && "
342- ss << " std::vector<const SearchConfig *> config_ptrs;" << std::endl;433+ "request.input_configs->size() == 1 && request.input_configs->front().empty();"
343- ss << " if (original_config_path) {" << std::endl;434+ << std::endl;
344- ss << " config_ptrs.push_back(nullptr);" << std::endl;435+ ss << " const bool original_config_path = internal_no_config_path || explicit_no_config_path;" << std::endl;
345- ss << " } else {" << std::endl;436+ ss << " std::vector<SearchConfig> configs;" << std::endl;
346- ss << " configs = ParseSearchConfigs(*request.input_configs);" << std::endl;437+ ss << " std::vector<const SearchConfig *> config_ptrs;" << std::endl;
347- ss << " if (configs.empty()) {" << std::endl;438+ ss << " if (original_config_path) {" << std::endl;
348- GenTopnSetFailureMessage(ss, " ", "invalid input configs");439+ ss << " config_ptrs.push_back(nullptr);" << std::endl;
349- ss << " return -1;" << std::endl;440+ ss << " } else {" << std::endl;
350- ss << " }" << std::endl;441+ ss << " configs = ParseSearchConfigs(*request.input_configs);" << std::endl;
351- ss << " config_ptrs.reserve(configs.size());" << std::endl;442+ ss << " if (configs.empty()) {" << std::endl;
352- ss << " for (const auto &cfg : configs) { config_ptrs.push_back(&cfg); }" << std::endl;443+ GenTopnSetFailureMessage(ss, " ", "invalid input configs");
353- ss << " }" << std::endl;444+ ss << " return -1;" << std::endl;
445+ ss << " }" << std::endl;
446+ ss << " config_ptrs.reserve(configs.size());" << std::endl;
447+ ss << " for (const auto &cfg : configs) { config_ptrs.push_back(&cfg); }" << std::endl;
448+ ss << " }" << std::endl;
449+ }
354 ss << std::endl;450 ss << std::endl;
355 451 
356- GenTopnSearchAndFinalChecks(ss, tiling, fused_schedule_result);452+ GenTopnSearchAndFinalChecks(ss, tiling, fused_schedule_result, use_measured_perf);
357 ss << " return 0;" << std::endl;453 ss << " return 0;" << std::endl;
358 ss << "}" << std::endl;454 ss << "}" << std::endl;
359 ss << std::endl;455 ss << std::endl;
@@ -423,6 +519,88 @@ void TilingLib::GenTopnCollectCandidates(std::stringstream &ss, const std::strin
423 ss << std::endl;519 ss << std::endl;
424}520}
425 521 
522+namespace {
523+void GenTopnCollectMeasuredCandidatesForList(std::stringstream &ss, const std::string &indent,
524+ const std::string &candidate_list) {
525+ ss << indent << "for (const auto &raw_candidate : " << candidate_list << ") {" << std::endl;
526+ ss << indent
527+ << " if (!std::isfinite(raw_candidate.best_perf) || raw_candidate.best_perf <= 0.0 || "
528+ "raw_candidate.best_perf >= DBL_MAX) { continue; }"
529+ << std::endl;
530+ ss << indent << " CandidateSolution solution;" << std::endl;
531+ ss << indent << " solution.tiling_data = raw_candidate.tiling_data;" << std::endl;
532+ ss << indent << " solution.canonical_repr = GetTilingDataRepr(&raw_candidate.tiling_data);" << std::endl;
533+ ss << indent << " if (solution.canonical_repr.empty()) { continue; }" << std::endl;
534+ ss << indent << " solution.modeled_perf = raw_candidate.best_perf;" << std::endl;
535+ ss << indent << " solution.is_default = !default_repr.empty() && (solution.canonical_repr == default_repr);"
536+ << std::endl;
537+ ss << indent << " if (solution.is_default) { found_default_candidate = true; }" << std::endl;
538+ ss << indent << " response.candidate_solutions.push_back(solution);" << std::endl;
539+ ss << indent << "}" << std::endl;
540+}
541+} // namespace
542+ 
543+void TilingLib::GenTopnMeasuredCoreSearch(std::stringstream &ss, const std::string &tiling) const {
544+ ss << " PgoConfigRuntimeGuard pgo_config_guard;" << std::endl;
545+ ss << " std::vector<" << tiling << "> measured_tiling_datas;" << std::endl;
546+ ss << " PgoConfig::Instance().need_change_solver_run = true;" << std::endl;
547+ ss << " while (PgoConfig::Instance().pgo_threshold_index < PgoConfig::Instance().pgo_threshold_list_size) {"
548+ << std::endl;
549+ ss << " " << tiling << " cur_search_tiling = search_tiling;" << std::endl;
550+ ss << " if (!optiling::PGOByCoreNumSearchTilingKey(measured_tiling_datas, &cur_search_tiling, "
551+ "measured_aiv_num)) {"
552+ << std::endl;
553+ GenTopnSetFailureMessage(ss, " ", "PGOByCoreNumSearchTilingKey failed");
554+ ss << " return -1;" << std::endl;
555+ ss << " }" << std::endl;
556+ ss << " ++PgoConfig::Instance().pgo_threshold_index;" << std::endl;
557+ ss << " }" << std::endl;
558+}
559+ 
560+void TilingLib::GenTopnMeasuredBatchProfiling(std::stringstream &ss) const {
561+ ss << " std::vector<AutofuseTilingDataPerf> raw_candidates;" << std::endl;
562+ ss << " uint32_t workspace_size = 0U;" << std::endl;
563+ ss << " for (const auto &tiling_data : measured_tiling_datas) {" << std::endl;
564+ ss << " workspace_size = std::max(workspace_size, GetWorkspaceSize(tiling_data));" << std::endl;
565+ ss << " raw_candidates.push_back({tiling_data, DBL_MAX});" << std::endl;
566+ ss << " }" << std::endl;
567+ ss << " if (raw_candidates.empty()) {" << std::endl;
568+ GenTopnSetFailureMessage(ss, " ", "PGOByCoreNumSearchTilingKey returned no candidate");
569+ ss << " return -1;" << std::endl;
570+ ss << " }" << std::endl;
571+ ss << " auto measured_candidates = NormalizePgoMeasuredCandidates(std::move(raw_candidates));" << std::endl;
572+ ss << " if (PgoConfig::Instance().batch_callback(PgoConfig::Instance().tensor_args, "
573+ "PgoConfig::Instance().stream, workspace_size, &measured_candidates) != 0) {"
574+ << std::endl;
575+ GenTopnSetFailureMessage(ss, " ", "batch profiling callback failed");
576+ ss << " return -1;" << std::endl;
577+ ss << " }" << std::endl;
578+ GenTopnCollectMeasuredCandidatesForList(ss, " ", "measured_candidates");
579+}
580+ 
581+void TilingLib::GenTopnAppendMeasuredDefault(std::stringstream &ss) const {
582+ ss << " if (!default_repr.empty() && !found_default_candidate) {" << std::endl;
583+ ss << " if (PgoConfig::Instance().single_callback == nullptr) {" << std::endl;
584+ ss << " response.error_message = \"single profiling callback is not set\";" << std::endl;
585+ ss << " return -1;" << std::endl;
586+ ss << " }" << std::endl;
587+ ss << " double default_perf = DBL_MAX;" << std::endl;
588+ ss << " const uint32_t default_workspace = GetWorkspaceSize(default_tiling);" << std::endl;
589+ ss << " const auto callback_ret = PgoConfig::Instance().single_callback(PgoConfig::Instance().tensor_args, "
590+ "PgoConfig::Instance().stream, default_workspace, &default_tiling, &default_perf);"
591+ << std::endl;
592+ ss << " if (callback_ret != 0 || !std::isfinite(default_perf) || default_perf <= 0.0 || "
593+ "default_perf >= DBL_MAX) {"
594+ << std::endl;
595+ ss << " response.error_message = \"default topn candidate profiling failed\";" << std::endl;
596+ ss << " return -1;" << std::endl;
597+ ss << " }" << std::endl;
598+ ss << " response.candidate_solutions.push_back({default_tiling, default_perf, true, default_repr});" << std::endl;
599+ ss << " found_default_candidate = true;" << std::endl;
600+ ss << " }" << std::endl;
601+ ss << std::endl;
602+}
603+ 
426void TilingLib::GenTopnSearchTilingKeyCall(std::stringstream &ss,604void TilingLib::GenTopnSearchTilingKeyCall(std::stringstream &ss,
427 const ascir::FusedScheduledResult &fused_schedule_result,605 const ascir::FusedScheduledResult &fused_schedule_result,
428 const std::string &search_cfg) const {606 const std::string &search_cfg) const {
@@ -430,9 +608,11 @@ void TilingLib::GenTopnSearchTilingKeyCall(std::stringstream &ss,
430 ss << "nullptr, ";608 ss << "nullptr, ";
431 const bool is_single_group = ascgen_utils::IsSingleGroup(fused_schedule_result);609 const bool is_single_group = ascgen_utils::IsSingleGroup(fused_schedule_result);
432 if (is_single_group) {610 if (is_single_group) {
433- ss << "nullptr, 0, best_perf, workspace_map, {}, " << search_cfg << ");" << std::endl;611+ ss << "nullptr, ";
612+ ss << "0, best_perf, workspace_map, {}, " << search_cfg << ");" << std::endl;
434 } else {613 } else {
435- ss << "nullptr, 0, best_perf, " << search_cfg << ");" << std::endl;614+ ss << "nullptr, ";
615+ ss << "0, best_perf, " << search_cfg << ");" << std::endl;
436 }616 }
437}617}
438 618 
@@ -454,11 +634,24 @@ void TilingLib::GenTopnDefaultTiling(std::stringstream &ss, const std::string &t
454}634}
455 635 
456void TilingLib::GenTopnSearchAndFinalChecks(std::stringstream &ss, const std::string &tiling,636void TilingLib::GenTopnSearchAndFinalChecks(std::stringstream &ss, const std::string &tiling,
457- const ascir::FusedScheduledResult &fused_schedule_result) const {637+ const ascir::FusedScheduledResult &fused_schedule_result,
458- ss << " PgoConfig::Instance().ResetRuntimeOverrides();" << std::endl;638+ bool use_measured_perf) const {
459- ss << " size_t failed_config_count = 0U;" << std::endl;639+ if (use_measured_perf) {
460- GenTopnSearchTilingSetup(ss, tiling, fused_schedule_result);640+ ss << " if (PgoConfig::Instance().tensor_args == nullptr || PgoConfig::Instance().stream == nullptr || "
461- GenTopnCollectCandidates(ss, tiling);641+ "PgoConfig::Instance().single_callback == nullptr || PgoConfig::Instance().batch_callback == nullptr) {"
642+ << std::endl;
643+ ss << " response.error_message = \"PGO runtime context is not set\";" << std::endl;
644+ ss << " return -1;" << std::endl;
645+ ss << " }" << std::endl;
646+ GenTopnMeasuredCoreSearch(ss, tiling);
647+ GenTopnMeasuredBatchProfiling(ss);
648+ GenTopnAppendMeasuredDefault(ss);
649+ } else {
650+ ss << " PgoConfig::Instance().ResetRuntimeOverrides();" << std::endl;
651+ ss << " size_t failed_config_count = 0U;" << std::endl;
652+ GenTopnSearchTilingSetup(ss, tiling, fused_schedule_result);
653+ GenTopnCollectCandidates(ss, tiling);
654+ }
462 ss << " if (!found_default_candidate) {" << std::endl;655 ss << " if (!found_default_candidate) {" << std::endl;
463 ss << " if (response.error_message.empty()) {" << std::endl;656 ss << " if (response.error_message.empty()) {" << std::endl;
464 GenTopnSetFailureMessage(ss, " ", "default topn candidate not found");657 GenTopnSetFailureMessage(ss, " ", "default topn candidate not found");
@@ -479,8 +672,9 @@ void TilingLib::GenTopnSearchAndFinalChecks(std::stringstream &ss, const std::st
479void TilingLib::GenGenerateTopnSolutionsEntry(std::stringstream &ss,672void TilingLib::GenGenerateTopnSolutionsEntry(std::stringstream &ss,
480 const ascir::FusedScheduledResult &fused_schedule_result,673 const ascir::FusedScheduledResult &fused_schedule_result,
481 const std::string &tiling,674 const std::string &tiling,
482- const codegen::PgoShapeStringStream &pgo_shape_dim) const {675+ const codegen::PgoShapeStringStream &pgo_shape_dim,
483- ss << "extern \"C\" int64_t GenerateTopnSolutions(";676+ const std::string &entry_declaration) const {
677+ ss << entry_declaration << "(";
484 ss << pgo_shape_dim.shape_dim_def.str();678 ss << pgo_shape_dim.shape_dim_def.str();
485 ss << "const std::vector<std::map<std::string, std::string>> &input_configs, int64_t topn, ";679 ss << "const std::vector<std::map<std::string, std::string>> &input_configs, int64_t topn, ";
486 ss << "std::vector<" << tiling << "> &tiling_datas, std::vector<int64_t> &workspaces, ";680 ss << "std::vector<" << tiling << "> &tiling_datas, std::vector<int64_t> &workspaces, ";
@@ -551,7 +745,7 @@ std::string TilingLib::GenCandidateSolutionProtocolForInductor(const std::string
551 return ss.str();745 return ss.str();
552}746}
553 747 
554-void TilingLib::GenDeduplicateCandidateSolutions(std::stringstream &ss) const {748+void TilingLib::GenDeduplicateCandidateSolutionsPrefix(std::stringstream &ss) const {
555 ss << "inline void DeduplicateCandidateSolutions(std::vector<CandidateSolution> &solutions) {" << std::endl;749 ss << "inline void DeduplicateCandidateSolutions(std::vector<CandidateSolution> &solutions) {" << std::endl;
556 ss << " std::unordered_map<std::string, size_t> repr_to_index;" << std::endl;750 ss << " std::unordered_map<std::string, size_t> repr_to_index;" << std::endl;
557 ss << " std::vector<CandidateSolution> deduplicated;" << std::endl;751 ss << " std::vector<CandidateSolution> deduplicated;" << std::endl;
@@ -565,6 +759,10 @@ void TilingLib::GenDeduplicateCandidateSolutions(std::stringstream &ss) const {
565 ss << " continue;" << std::endl;759 ss << " continue;" << std::endl;
566 ss << " }" << std::endl;760 ss << " }" << std::endl;
567 ss << " auto &kept = deduplicated[iter->second];" << std::endl;761 ss << " auto &kept = deduplicated[iter->second];" << std::endl;
762+}
763+ 
764+void TilingLib::GenDeduplicateCandidateSolutions(std::stringstream &ss) const {
765+ GenDeduplicateCandidateSolutionsPrefix(ss);
568 ss << " if (!(std::fabs(kept.modeled_perf - solution.modeled_perf) < 1e-8)) {" << std::endl;766 ss << " if (!(std::fabs(kept.modeled_perf - solution.modeled_perf) < 1e-8)) {" << std::endl;
569 ss << " OP_LOGW(OP_NAME, \"same repr with different modeled_perf, keep first: kept=%.6f, current=%.6f, "767 ss << " OP_LOGW(OP_NAME, \"same repr with different modeled_perf, keep first: kept=%.6f, current=%.6f, "
570 "repr=%s\", "768 "repr=%s\", "
@@ -580,6 +778,15 @@ void TilingLib::GenDeduplicateCandidateSolutions(std::stringstream &ss) const {
580 ss << std::endl;778 ss << std::endl;
581}779}
582 780 
781+void TilingLib::GenDeduplicateMeasuredCandidateSolutions(std::stringstream &ss) const {
782+ GenDeduplicateCandidateSolutionsPrefix(ss);
783+ ss << " if (kept.modeled_perf > solution.modeled_perf) { kept = solution; }" << std::endl;
784+ ss << " }" << std::endl;
785+ ss << " solutions.swap(deduplicated);" << std::endl;
786+ ss << "}" << std::endl;
787+ ss << std::endl;
788+}
789+ 
583std::string TilingLib::GenTopnSelectorHelpersForInductor() const {790std::string TilingLib::GenTopnSelectorHelpersForInductor() const {
584 std::stringstream ss;791 std::stringstream ss;
585 ss << "// Topn selector helpers: default-first, modeled_perf ascending, canonical_repr tiebreak." << std::endl;792 ss << "// Topn selector helpers: default-first, modeled_perf ascending, canonical_repr tiebreak." << std::endl;
@@ -619,31 +826,40 @@ std::string TilingLib::GenTopnSelectorHelpersForInductor() const {
619 return ss.str();826 return ss.str();
620}827}
621 828 
622-std::string TilingLib::GenSearchConfigProtocolForInductor() const {829+std::string TilingLib::GenMeasuredTopnSelectorHelpersForInductor() const {
623 std::stringstream ss;830 std::stringstream ss;
624- ss << "// SearchConfig for dual-path PGO: TF builtin and Inductor request configs." << std::endl;831+ ss << "// Topn selector helpers: measured best first, default fallback second." << std::endl;
625- ss << "struct SearchConfig {" << std::endl;832+ ss << "inline bool CompareCandidateSolution(const CandidateSolution &lhs, const CandidateSolution &rhs) {"
626- ss << " bool ub_threshold_enabled = false;" << std::endl;833+ << std::endl;
627- ss << " double ub_threshold = 0.0;" << std::endl;834+ ss << " if (lhs.modeled_perf < rhs.modeled_perf || rhs.modeled_perf < lhs.modeled_perf) { "
628- ss << " bool corenum_threshold_enabled = false;" << std::endl;835+ "return lhs.modeled_perf < rhs.modeled_perf; }"
629- ss << " double corenum_threshold = 1.0;" << std::endl;836+ << std::endl;
630- ss << " bool enable_multicore_ub_tradeoff = false;" << std::endl;837+ ss << " return lhs.canonical_repr < rhs.canonical_repr;" << std::endl;
631- ss << "};" << std::endl;838+ ss << "}" << std::endl;
632 ss << std::endl;839 ss << std::endl;
633- return ss.str();840+ GenDeduplicateMeasuredCandidateSolutions(ss);
634-}841+ ss << "inline void SelectTopnCandidateSolutions(std::vector<CandidateSolution> &solutions, int64_t topn) {"
635- 842+ << std::endl;
636-std::string TilingLib::GenBuiltinTfPgoConfigsForInductor() const {843+ ss << " DeduplicateCandidateSolutions(solutions);" << std::endl;
637- std::stringstream ss;844+ ss << " std::sort(solutions.begin(), solutions.end(), CompareCandidateSolution);" << std::endl;
638- ss << "// Builtin TF PGO search configs: 5 fixed threshold configurations." << std::endl;845+ ss << " auto *measured_candidates = PgoConfig::Instance().measured_candidates;" << std::endl;
639- ss << "inline std::vector<SearchConfig> GetBuiltinTfPgoConfigs() {" << std::endl;846+ ss << " if (measured_candidates != nullptr) {" << std::endl;
640- ss << " return {" << std::endl;847+ ss << " measured_candidates->clear();" << std::endl;
641- ss << " {true, 0.2, true, 0.4, false}," << std::endl;848+ ss << " measured_candidates->reserve(solutions.size());" << std::endl;
642- ss << " {true, 0.1, true, 0.4, false}," << std::endl;849+ ss << " for (const auto &solution : solutions) {" << std::endl;
643- ss << " {true, 0.0, true, 1.0, false}," << std::endl;850+ ss << " measured_candidates->push_back({solution.tiling_data, solution.modeled_perf});" << std::endl;
644- ss << " {true, 0.05, true, 1.0, false}," << std::endl;851+ ss << " }" << std::endl;
645- ss << " {true, 0.1, true, 0.8, false}," << std::endl;852+ ss << " }" << std::endl;
646- ss << " };" << std::endl;853+ ss << " if (topn > 1 && solutions.size() > 1U) {" << std::endl;
854+ ss << " const auto default_solution = std::find_if(solutions.begin(), solutions.end()," << std::endl;
855+ ss << " [](const CandidateSolution &solution) { return solution.is_default; });" << std::endl;
856+ ss << " if (default_solution != solutions.end() && default_solution != solutions.begin()) {" << std::endl;
857+ ss << " std::rotate(solutions.begin() + 1, default_solution, default_solution + 1);" << std::endl;
858+ ss << " }" << std::endl;
859+ ss << " }" << std::endl;
860+ ss << " if (topn > 0 && static_cast<int64_t>(solutions.size()) > topn) {" << std::endl;
861+ ss << " solutions.resize(static_cast<size_t>(topn));" << std::endl;
862+ ss << " }" << std::endl;
647 ss << "}" << std::endl;863 ss << "}" << std::endl;
648 ss << std::endl;864 ss << std::endl;
649 return ss.str();865 return ss.str();
Mautofuse/codegen/codegen_tiling_pgo_common.cpp+462-92
@@ -16,7 +16,6 @@
16 16 
17namespace codegen {17namespace codegen {
18using namespace ascgen_utils;18using namespace ascgen_utils;
19- 
20namespace {19namespace {
21bool IsNeedFfts() {20bool IsNeedFfts() {
22 const auto backend_spec = optimize::BackendSpec::GetInstance();21 const auto backend_spec = optimize::BackendSpec::GetInstance();
@@ -42,7 +41,7 @@ void AppendPgoLogDefs(std::stringstream &ss) {
42}41}
43} // namespace42} // namespace
44 43 
45-void TilingLib::GenPgoHeaders(std::stringstream &ss) const {44+void TilingLib::GenPgoHeaders(std::stringstream &ss, bool direct_link) const {
46 ss << "#include <cinttypes>" << std::endl;45 ss << "#include <cinttypes>" << std::endl;
47 ss << "#include <unistd.h>" << std::endl;46 ss << "#include <unistd.h>" << std::endl;
48 ss << "#include <fcntl.h>" << std::endl;47 ss << "#include <fcntl.h>" << std::endl;
@@ -59,10 +58,21 @@ void TilingLib::GenPgoHeaders(std::stringstream &ss) const {
59 ss << "#include <cstring>" << std::endl;58 ss << "#include <cstring>" << std::endl;
60 ss << "#include <securec.h>" << std::endl;59 ss << "#include <securec.h>" << std::endl;
61 ss << "#include <fstream>" << std::endl;60 ss << "#include <fstream>" << std::endl;
61+ if (direct_link) {
62+ ss << "#include <atomic>" << std::endl;
63+ ss << "#include <cctype>" << std::endl;
64+ ss << "#include <climits>" << std::endl;
65+ ss << "#include <cstdio>" << std::endl;
66+ ss << "#include <cstdlib>" << std::endl;
67+ ss << "#include <limits>" << std::endl;
68+ ss << "#include <type_traits>" << std::endl;
69+ }
62 ss << "#include <map>" << std::endl;70 ss << "#include <map>" << std::endl;
63 ss << "#include <string>" << std::endl;71 ss << "#include <string>" << std::endl;
64 ss << "#include <thread>" << std::endl;72 ss << "#include <thread>" << std::endl;
73+ ss << "#include <utility>" << std::endl;
65 ss << "#include <unordered_map>" << std::endl;74 ss << "#include <unordered_map>" << std::endl;
75+ ss << "#include <unordered_set>" << std::endl;
66 ss << "#include <vector>" << std::endl << std::endl;76 ss << "#include <vector>" << std::endl << std::endl;
67 77 
68 ss << "#include \"acl/acl.h\"" << std::endl;78 ss << "#include \"acl/acl.h\"" << std::endl;
@@ -186,11 +196,11 @@ void TilingLib::GenPgoAppendSearchTilingData(std::stringstream &ss) const {
186}196}
187 197 
188void TilingLib::GenPgoKernelLaunchOpArgs(const ascir::FusedScheduledResult &fused_schedule_result,198void TilingLib::GenPgoKernelLaunchOpArgs(const ascir::FusedScheduledResult &fused_schedule_result,
189- std::stringstream &ss) const {199+ std::stringstream &ss, bool direct_link) const {
190 ss << "struct AivKernelLaunchOpArgs {" << std::endl;200 ss << "struct AivKernelLaunchOpArgs {" << std::endl;
191 ss << PGOSearchStructInputOutputDef(fused_schedule_result);201 ss << PGOSearchStructInputOutputDef(fused_schedule_result);
192 ss << " uint64_t workspace_addr;" << std::endl;202 ss << " uint64_t workspace_addr;" << std::endl;
193- ss << " uint64_t tiling_addr;" << std::endl;203+ ss << (direct_link ? " AutofuseTilingData tiling_data;" : " uint64_t tiling_addr;") << std::endl;
194 ss << "};" << std::endl;204 ss << "};" << std::endl;
195 205 
196 ss << "struct MixKernelLaunchOpArgs {" << std::endl;206 ss << "struct MixKernelLaunchOpArgs {" << std::endl;
@@ -199,7 +209,7 @@ void TilingLib::GenPgoKernelLaunchOpArgs(const ascir::FusedScheduledResult &fuse
199 }209 }
200 ss << PGOSearchStructInputOutputDef(fused_schedule_result);210 ss << PGOSearchStructInputOutputDef(fused_schedule_result);
201 ss << " uint64_t workspace_addr;" << std::endl;211 ss << " uint64_t workspace_addr;" << std::endl;
202- ss << " uint64_t tiling_addr;" << std::endl;212+ ss << (direct_link ? " AutofuseTilingData tiling_data;" : " uint64_t tiling_addr;") << std::endl;
203 ss << "};" << std::endl;213 ss << "};" << std::endl;
204 214 
205 ss << "void *g_workspace = nullptr;" << std::endl;215 ss << "void *g_workspace = nullptr;" << std::endl;
@@ -227,8 +237,8 @@ void TilingLib::GenPgoCheckTilingIsMix(const ascir::FusedScheduledResult &fused_
227 ss << "}" << std::endl;237 ss << "}" << std::endl;
228}238}
229 239 
230-void TilingLib::GenPgoLaunchParamsInit(const ascir::FusedScheduledResult &fused_schedule_result,240+void TilingLib::GenPgoLaunchParamsInit(const ascir::FusedScheduledResult &fused_schedule_result, std::stringstream &ss,
231- std::stringstream &ss) const {241+ bool direct_link) const {
232 ss << "aclError LaunchParamsInit(PgoTensorArgs *tensor_args) {" << std::endl;242 ss << "aclError LaunchParamsInit(PgoTensorArgs *tensor_args) {" << std::endl;
233 ss << " static void *ffts = nullptr;" << std::endl;243 ss << " static void *ffts = nullptr;" << std::endl;
234 ss << " aclError ret = ACL_SUCCESS;" << std::endl;244 ss << " aclError ret = ACL_SUCCESS;" << std::endl;
@@ -243,7 +253,9 @@ void TilingLib::GenPgoLaunchParamsInit(const ascir::FusedScheduledResult &fused_
243 ss << " return FAILED;" << std::endl;253 ss << " return FAILED;" << std::endl;
244 ss << " }" << std::endl;254 ss << " }" << std::endl;
245 ss << PGOSearchFuncInputOutputStructAssignDef(fused_schedule_result, " g_launch_params.aiv_args");255 ss << PGOSearchFuncInputOutputStructAssignDef(fused_schedule_result, " g_launch_params.aiv_args");
246- ss << " g_launch_params.aiv_args.tiling_addr = reinterpret_cast<uint64_t>(g_tiling_device_addr);" << std::endl;256+ if (!direct_link) {
257+ ss << " g_launch_params.aiv_args.tiling_addr = reinterpret_cast<uint64_t>(g_tiling_device_addr);" << std::endl;
258+ }
247 if (IsNeedFfts()) {259 if (IsNeedFfts()) {
248 ss << " ret = aclrtGetHardwareSyncAddr(&ffts);" << std::endl;260 ss << " ret = aclrtGetHardwareSyncAddr(&ffts);" << std::endl;
249 ss << " if (ret != ACL_SUCCESS) {" << std::endl;261 ss << " if (ret != ACL_SUCCESS) {" << std::endl;
@@ -253,7 +265,9 @@ void TilingLib::GenPgoLaunchParamsInit(const ascir::FusedScheduledResult &fused_
253 ss << " g_launch_params.mix_args.ffts = reinterpret_cast<uint64_t>(ffts);" << std::endl;265 ss << " g_launch_params.mix_args.ffts = reinterpret_cast<uint64_t>(ffts);" << std::endl;
254 }266 }
255 ss << PGOSearchFuncInputOutputStructAssignDef(fused_schedule_result, " g_launch_params.mix_args");267 ss << PGOSearchFuncInputOutputStructAssignDef(fused_schedule_result, " g_launch_params.mix_args");
256- ss << " g_launch_params.mix_args.tiling_addr = reinterpret_cast<uint64_t>(g_tiling_device_addr);" << std::endl;268+ if (!direct_link) {
269+ ss << " g_launch_params.mix_args.tiling_addr = reinterpret_cast<uint64_t>(g_tiling_device_addr);" << std::endl;
270+ }
257 ss << " ret = aclrtMalloc(&g_launch_params.aiv_args_device, sizeof(AivKernelLaunchOpArgs), "271 ss << " ret = aclrtMalloc(&g_launch_params.aiv_args_device, sizeof(AivKernelLaunchOpArgs), "
258 "ACL_MEM_MALLOC_HUGE_FIRST);"272 "ACL_MEM_MALLOC_HUGE_FIRST);"
259 << std::endl;273 << std::endl;
@@ -291,6 +305,17 @@ void TilingLib::GenPgoLaunchParamsDeInit(std::stringstream &ss) const {
291 ss << "}" << std::endl;305 ss << "}" << std::endl;
292}306}
293 307 
308+void TilingLib::GenPgoCopyLaunchArgs(std::stringstream &ss, const std::string &kernel_type,
309+ const std::string &assignment) const {
310+ ss << " " << assignment << "aclrtMemcpy(g_launch_params." << kernel_type << "_args_device, sizeof(g_launch_params."
311+ << kernel_type << "_args), (void *)&g_launch_params." << kernel_type << "_args, sizeof(g_launch_params."
312+ << kernel_type << "_args), ACL_MEMCPY_HOST_TO_DEVICE);" << std::endl;
313+ ss << " if (ret != ACL_SUCCESS) {" << std::endl;
314+ ss << " DLOGE(\"memcpy " << kernel_type << "_args to device failed, ERROR: %d\", ret);" << std::endl;
315+ ss << " return FAILED;" << std::endl;
316+ ss << " }" << std::endl;
317+}
318+ 
294void TilingLib::GenPgoUpdateLaunchParams(std::stringstream &ss) const {319void TilingLib::GenPgoUpdateLaunchParams(std::stringstream &ss) const {
295 ss << "aclError UpdateLaunchParam(const AutofuseTilingData &tiling_data) {" << std::endl;320 ss << "aclError UpdateLaunchParam(const AutofuseTilingData &tiling_data) {" << std::endl;
296 ss << " if (IsMixTiling(tiling_data)) {" << std::endl;321 ss << " if (IsMixTiling(tiling_data)) {" << std::endl;
@@ -302,13 +327,7 @@ void TilingLib::GenPgoUpdateLaunchParams(std::stringstream &ss) const {
302 ss << " return FAILED;" << std::endl;327 ss << " return FAILED;" << std::endl;
303 ss << " }" << std::endl;328 ss << " }" << std::endl;
304 ss << " g_launch_params.mix_args.workspace_addr = reinterpret_cast<uint64_t>(g_workspace);" << std::endl;329 ss << " g_launch_params.mix_args.workspace_addr = reinterpret_cast<uint64_t>(g_workspace);" << std::endl;
305- ss << " ret = aclrtMemcpy(g_launch_params.mix_args_device, sizeof(g_launch_params.mix_args), (void "330+ GenPgoCopyLaunchArgs(ss, "mix", "ret = ");
306- "*)&g_launch_params.mix_args, sizeof(g_launch_params.mix_args), ACL_MEMCPY_HOST_TO_DEVICE);"
307- << std::endl;
308- ss << " if (ret != ACL_SUCCESS) {" << std::endl;
309- ss << " DLOGE(\"memcpy mix_args to device failed, ERROR: %d\", ret);" << std::endl;
310- ss << " return FAILED;" << std::endl;
311- ss << " }" << std::endl;
312 ss << " } else {" << std::endl;331 ss << " } else {" << std::endl;
313 ss << " auto ret = aclrtMemcpy((void *)g_launch_params.aiv_args.tiling_addr, sizeof(AutofuseTilingData), (void "332 ss << " auto ret = aclrtMemcpy((void *)g_launch_params.aiv_args.tiling_addr, sizeof(AutofuseTilingData), (void "
314 "*)&tiling_data, "333 "*)&tiling_data, "
@@ -318,20 +337,29 @@ void TilingLib::GenPgoUpdateLaunchParams(std::stringstream &ss) const {
318 ss << " return FAILED;" << std::endl;337 ss << " return FAILED;" << std::endl;
319 ss << " }" << std::endl;338 ss << " }" << std::endl;
320 ss << " g_launch_params.aiv_args.workspace_addr = reinterpret_cast<uint64_t>(g_workspace);" << std::endl;339 ss << " g_launch_params.aiv_args.workspace_addr = reinterpret_cast<uint64_t>(g_workspace);" << std::endl;
321- ss << " ret = aclrtMemcpy(g_launch_params.aiv_args_device, sizeof(g_launch_params.aiv_args), (void "340+ GenPgoCopyLaunchArgs(ss, "aiv", "ret = ");
322- "*)&g_launch_params.aiv_args, sizeof(g_launch_params.aiv_args), ACL_MEMCPY_HOST_TO_DEVICE);"
323- << std::endl;
324- ss << " if (ret != ACL_SUCCESS) {" << std::endl;
325- ss << " DLOGE(\"memcpy aiv_args to device failed, ERROR: %d\", ret);" << std::endl;
326- ss << " return FAILED;" << std::endl;
327- ss << " }" << std::endl;
328 ss << " }" << std::endl;341 ss << " }" << std::endl;
329 ss << " return ACL_SUCCESS;" << std::endl;342 ss << " return ACL_SUCCESS;" << std::endl;
330 ss << "}" << std::endl;343 ss << "}" << std::endl;
331}344}
332 345 
333-void TilingLib::GenPgoLaunchParams(const ascir::FusedScheduledResult &fused_schedule_result,346+void TilingLib::GenInductorPgoUpdateLaunchParams(std::stringstream &ss) const {
334- std::stringstream &ss) const {347+ ss << "aclError UpdateLaunchParam(const AutofuseTilingData &tiling_data) {" << std::endl;
348+ ss << " if (IsMixTiling(tiling_data)) {" << std::endl;
349+ ss << " g_launch_params.mix_args.tiling_data = tiling_data;" << std::endl;
350+ ss << " g_launch_params.mix_args.workspace_addr = reinterpret_cast<uint64_t>(g_workspace);" << std::endl;
351+ GenPgoCopyLaunchArgs(ss, "mix", "auto ret = ");
352+ ss << " } else {" << std::endl;
353+ ss << " g_launch_params.aiv_args.tiling_data = tiling_data;" << std::endl;
354+ ss << " g_launch_params.aiv_args.workspace_addr = reinterpret_cast<uint64_t>(g_workspace);" << std::endl;
355+ GenPgoCopyLaunchArgs(ss, "aiv", "auto ret = ");
356+ ss << " }" << std::endl;
357+ ss << " return ACL_SUCCESS;" << std::endl;
358+ ss << "}" << std::endl;
359+}
360+ 
361+void TilingLib::GenPgoLaunchParams(const ascir::FusedScheduledResult &fused_schedule_result, std::stringstream &ss,
362+ bool direct_link) const {
335 ss << "struct LaunchParams {" << std::endl;363 ss << "struct LaunchParams {" << std::endl;
336 ss << " AivKernelLaunchOpArgs aiv_args;" << std::endl;364 ss << " AivKernelLaunchOpArgs aiv_args;" << std::endl;
337 ss << " void *aiv_args_device;" << std::endl;365 ss << " void *aiv_args_device;" << std::endl;
@@ -339,28 +367,38 @@ void TilingLib::GenPgoLaunchParams(const ascir::FusedScheduledResult &fused_sche
339 ss << " void *mix_args_device;" << std::endl;367 ss << " void *mix_args_device;" << std::endl;
340 ss << "} g_launch_params;" << std::endl;368 ss << "} g_launch_params;" << std::endl;
341 369 
342- GenPgoLaunchParamsInit(fused_schedule_result, ss);370+ GenPgoLaunchParamsInit(fused_schedule_result, ss, direct_link);
343 GenPgoLaunchParamsDeInit(ss);371 GenPgoLaunchParamsDeInit(ss);
344- GenPgoUpdateLaunchParams(ss);372+ if (direct_link) {
373+ GenInductorPgoUpdateLaunchParams(ss);
374+ } else {
375+ GenPgoUpdateLaunchParams(ss);
376+ }
345}377}
346 378 
347-void TilingLib::GenPgoToolFunction(const ascir::FusedScheduledResult &fused_schedule_result, const std::string &pgo_dir,379+void TilingLib::GenPgoToolDeclarations(const ascir::FusedScheduledResult &fused_schedule_result,
348- std::stringstream &ss) const {380+ const std::string &pgo_dir, std::stringstream &ss, bool direct_link) const {
349- std::string graph_name = CamelToLowerSneak(fused_schedule_result.fused_graph_name.GetString());381+ const std::string graph_name = CamelToLowerSneak(GenValidName(fused_schedule_result.fused_graph_name.GetString()));
350 ss << "namespace {" << std::endl;382 ss << "namespace {" << std::endl;
351 ss << "constexpr bool g_is_mix_operator = " << (IsMixKernelTaskType(fused_schedule_result) ? "true;" : "false;")383 ss << "constexpr bool g_is_mix_operator = " << (IsMixKernelTaskType(fused_schedule_result) ? "true;" : "false;")
352 << std::endl;384 << std::endl;
353 ss << "static bool g_is_static_kernel = false;" << std::endl;385 ss << "static bool g_is_static_kernel = false;" << std::endl;
354 GenPgoMixTilingTable(fused_schedule_result, ss);386 GenPgoMixTilingTable(fused_schedule_result, ss);
355 GenPgoCheckTilingIsMix(fused_schedule_result, ss);387 GenPgoCheckTilingIsMix(fused_schedule_result, ss);
356- ss << "static std::string g_kernel_name;" << std::endl;388+ if (direct_link) {
389+ ss << "constexpr char kInductorPgoKernelName[] = \"" << graph_name << "\";" << std::endl;
390+ } else {
391+ ss << "static std::string g_kernel_name;" << std::endl;
392+ }
357 ss << "static std::string g_kernel_o_file;" << std::endl;393 ss << "static std::string g_kernel_o_file;" << std::endl;
358 ss << "static std::string g_npu_lock_file;" << std::endl;394 ss << "static std::string g_npu_lock_file;" << std::endl;
359 ss << "#define PGO_GRAPH_NAME \"" << graph_name << "\"" << std::endl;395 ss << "#define PGO_GRAPH_NAME \"" << graph_name << "\"" << std::endl;
360- ss << "const char *pgo_dir = \"" << pgo_dir << "\";" << std::endl;396+ if (!direct_link) {
361- ss << "const char *config_file = \"" << pgo_dir << "/" << graph_name << "_config.txt" << "\";" << std::endl;397+ ss << "const char *pgo_dir = \"" << pgo_dir << "\";" << std::endl;
362- ss << "const char *search_file = \"" << pgo_dir << "/" << graph_name << "_search.txt" << "\";" << std::endl;398+ ss << "const char *config_file = \"" << pgo_dir << "/" << graph_name << "_config.txt" << "\";" << std::endl;
363- ss << "const char *kernel_file = \"" << pgo_dir << "/lib" << graph_name << ".so" << "\";" << std::endl;399+ ss << "const char *search_file = \"" << pgo_dir << "/" << graph_name << "_search.txt" << "\";" << std::endl;
400+ ss << "const char *kernel_file = \"" << pgo_dir << "/lib" << graph_name << ".so" << "\";" << std::endl;
401+ }
364 ss << "#define SUCCESS 0" << std::endl;402 ss << "#define SUCCESS 0" << std::endl;
365 ss << "#define FAILED 1" << std::endl;403 ss << "#define FAILED 1" << std::endl;
366 404 
@@ -370,16 +408,31 @@ void TilingLib::GenPgoToolFunction(const ascir::FusedScheduledResult &fused_sche
370 AppendPgoLogDefs(ss);408 AppendPgoLogDefs(ss);
371 409 
372 GenPgoCardLock(ss);410 GenPgoCardLock(ss);
373- GenPgoAppendSearchTilingData(ss);411+ if (!direct_link) {
374- GenPgoKernelLaunchOpArgs(fused_schedule_result, ss);412+ GenPgoAppendSearchTilingData(ss);
413+ }
414+ GenPgoKernelLaunchOpArgs(fused_schedule_result, ss, direct_link);
375 415 
376- GenDynamicLibraryLoaderCode(ss);416+ if (!direct_link) {
417+ GenDynamicLibraryLoaderCode(ss);
418+ }
419+}
377 420 
378- ss << "aclrtStream g_stream;" << std::endl;421+void TilingLib::GenPgoToolFunction(const ascir::FusedScheduledResult &fused_schedule_result, const std::string &pgo_dir,
422+ std::stringstream &ss, bool direct_link) const {
423+ GenPgoToolDeclarations(fused_schedule_result, pgo_dir, ss, direct_link);
424+ 
425+ ss << (direct_link ? "aclrtStream g_stream = nullptr;" : "aclrtStream g_stream;") << std::endl;
379 ss << PGOSearchTensorInputOutputDef(fused_schedule_result) << std::endl;426 ss << PGOSearchTensorInputOutputDef(fused_schedule_result) << std::endl;
380- ss << "void *g_tiling_device_addr = nullptr;" << std::endl;427+ if (direct_link) {
428+ ss << "bool g_acl_initialized = false;" << std::endl;
429+ ss << "bool g_device_set = false;" << std::endl;
430+ ss << "int32_t g_device_id = -1;" << std::endl;
431+ } else {
432+ ss << "void *g_tiling_device_addr = nullptr;" << std::endl;
433+ }
381 434 
382- GenPgoLaunchParams(fused_schedule_result, ss);435+ GenPgoLaunchParams(fused_schedule_result, ss, direct_link);
383 436 
384 ss << "struct ResLimit {" << std::endl;437 ss << "struct ResLimit {" << std::endl;
385 ss << " uint32_t valid_num = 0;" << std::endl;438 ss << " uint32_t valid_num = 0;" << std::endl;
@@ -388,7 +441,7 @@ void TilingLib::GenPgoToolFunction(const ascir::FusedScheduledResult &fused_sche
388 ss << " uint32_t ub_size = 0;" << std::endl;441 ss << " uint32_t ub_size = 0;" << std::endl;
389 ss << " uint32_t resv[10];" << std::endl;442 ss << " uint32_t resv[10];" << std::endl;
390 ss << "};" << std::endl;443 ss << "};" << std::endl;
391- ss << "ResLimit g_res_limit = {1, {}};" << std::endl;444+ ss << (direct_link ? "ResLimit g_res_limit = {1, 0, 0, 0, {}};" : "ResLimit g_res_limit = {1, {}};") << std::endl;
392 ss << "inline bool IsEqual(double a, double b) {" << std::endl;445 ss << "inline bool IsEqual(double a, double b) {" << std::endl;
393 ss << " const double epsilon = 1e-8;" << std::endl;446 ss << " const double epsilon = 1e-8;" << std::endl;
394 ss << " double abs = (a > b) ? (a - b) : (b - a);" << std::endl;447 ss << " double abs = (a > b) ? (a - b) : (b - a);" << std::endl;
@@ -409,7 +462,7 @@ void TilingLib::GenPgoWrapperParmCall(const ascir::FusedScheduledResult &fused_s
409 if (CanUseTilingKey(fused_schedule_result)) {462 if (CanUseTilingKey(fused_schedule_result)) {
410 ss << " if (find_best_tiling_key_fn != nullptr) {" << std::endl;463 ss << " if (find_best_tiling_key_fn != nullptr) {" << std::endl;
411 ss << " tiling_key = find_best_tiling_key_fn(*tiling_data);" << std::endl;464 ss << " tiling_key = find_best_tiling_key_fn(*tiling_data);" << std::endl;
412- ss << " if (tiling_key == -1) {" << std::endl;465+ ss << " if (tiling_key < 0 || static_cast<uint64_t>(tiling_key) >= tiling_key_count) {" << std::endl;
413 ss << " DLOGE(\"find best tiling key failed\");" << std::endl;466 ss << " DLOGE(\"find best tiling key failed\");" << std::endl;
414 ss << " return FAILED;" << std::endl;467 ss << " return FAILED;" << std::endl;
415 ss << " }" << std::endl;468 ss << " }" << std::endl;
@@ -439,20 +492,26 @@ void TilingLib::GenPgoWrapperKernelLaunch(std::stringstream &ss) const {
439 ss << " auto ret_async = aclrtSynchronizeStream(g_stream);" << std::endl;492 ss << " auto ret_async = aclrtSynchronizeStream(g_stream);" << std::endl;
440}493}
441 494 
442-void TilingLib::GenPgoWrapper(const ascir::FusedScheduledResult &fused_schedule_result, std::stringstream &ss) const {495+void TilingLib::GenPgoWrapperInit(std::stringstream &ss, bool direct_link) const {
443 ss << "typedef uint64_t (*GetTilingKeyCountType)(void);" << std::endl;496 ss << "typedef uint64_t (*GetTilingKeyCountType)(void);" << std::endl;
444 ss << "GetTilingKeyCountType get_tiling_key_count_fn = "497 ss << "GetTilingKeyCountType get_tiling_key_count_fn = "
445- "reinterpret_cast<GetTilingKeyCountType>(GetFunc(\"GetTilingKeyCount\"));"498+ << (direct_link ? "nullptr;" : "reinterpret_cast<GetTilingKeyCountType>(GetFunc(\"GetTilingKeyCount\"));")
446 << std::endl;499 << std::endl;
447- if (CanUseTilingKey(fused_schedule_result)) {500+ ss << "typedef int64_t (*FindBestTilingKeyType)(AutofuseTilingData &t);" << std::endl;
448- ss << "typedef int64_t (*FindBestTilingKeyType)(AutofuseTilingData &t);" << std::endl;501+ ss << "FindBestTilingKeyType find_best_tiling_key_fn = "
449- ss << "FindBestTilingKeyType find_best_tiling_key_fn = "502+ << (direct_link ? "nullptr;" : "reinterpret_cast<FindBestTilingKeyType>(GetFunc(\"FindBestTilingKey\"));")
450- "reinterpret_cast<FindBestTilingKeyType>(GetFunc(\"FindBestTilingKey\"));"503+ << std::endl;
451- << std::endl;504+ if (direct_link) {
505+ ss << "static aclrtBinHandle g_pgo_bin_handle = nullptr;" << std::endl;
452 }506 }
453 ss << "int WrapperOnlyLaunch(uint32_t workspace_size, AutofuseTilingData *tiling_data) {" << std::endl;507 ss << "int WrapperOnlyLaunch(uint32_t workspace_size, AutofuseTilingData *tiling_data) {" << std::endl;
508+ if (direct_link) {
509+ ss << " (void)workspace_size;" << std::endl;
510+ }
454 ss << " static bool inited = false;" << std::endl;511 ss << " static bool inited = false;" << std::endl;
455- ss << " static aclrtBinHandle bin_handle = nullptr;" << std::endl;512+ if (!direct_link) {
513+ ss << " static aclrtBinHandle bin_handle = nullptr;" << std::endl;
514+ }
456 const auto backend_spce = optimize::BackendSpec::GetInstance();515 const auto backend_spce = optimize::BackendSpec::GetInstance();
457 if (backend_spce != nullptr && backend_spce->set_local_memory_size > 0) {516 if (backend_spce != nullptr && backend_spce->set_local_memory_size > 0) {
458 ss << " static aclrtLaunchKernelCfg kernel_cfg{};" << std::endl;517 ss << " static aclrtLaunchKernelCfg kernel_cfg{};" << std::endl;
@@ -464,9 +523,14 @@ void TilingLib::GenPgoWrapper(const ascir::FusedScheduledResult &fused_schedule_
464 ss << " }" << std::endl;523 ss << " }" << std::endl;
465 ss << " static uint64_t tiling_key_count = get_tiling_key_count_fn();" << std::endl;524 ss << " static uint64_t tiling_key_count = get_tiling_key_count_fn();" << std::endl;
466 ss << " static std::vector<aclrtFuncHandle> func_handles(tiling_key_count);" << std::endl;525 ss << " static std::vector<aclrtFuncHandle> func_handles(tiling_key_count);" << std::endl;
526+}
527+ 
528+void TilingLib::GenPgoWrapper(const ascir::FusedScheduledResult &fused_schedule_result, std::stringstream &ss,
529+ bool direct_link) const {
530+ GenPgoWrapperInit(ss, direct_link);
467 531 
468 GenPgoWrapperParmCall(fused_schedule_result, ss);532 GenPgoWrapperParmCall(fused_schedule_result, ss);
469- GenPgoLaunchKernelInit(ss);533+ GenPgoLaunchKernelInit(ss, direct_link);
470 GenPgoWrapperKernelLaunch(ss);534 GenPgoWrapperKernelLaunch(ss);
471 ss << " if (ret != ACL_SUCCESS) {" << std::endl;535 ss << " if (ret != ACL_SUCCESS) {" << std::endl;
472 ss << " DLOGE(\"aclrtLaunchKernelV2 failed, ERROR: %d\", ret);" << std::endl;536 ss << " DLOGE(\"aclrtLaunchKernelV2 failed, ERROR: %d\", ret);" << std::endl;
@@ -478,9 +542,17 @@ void TilingLib::GenPgoWrapper(const ascir::FusedScheduledResult &fused_schedule_
478 ss << " }" << std::endl;542 ss << " }" << std::endl;
479 ss << " return ret;" << std::endl;543 ss << " return ret;" << std::endl;
480 ss << "}" << std::endl << std::endl;544 ss << "}" << std::endl << std::endl;
545+ if (direct_link) {
546+ ss << "void PgoBinaryDeInit() {" << std::endl;
547+ ss << " if (g_pgo_bin_handle == nullptr) { return; }" << std::endl;
548+ ss << " auto ret = aclrtBinaryUnLoad(g_pgo_bin_handle);" << std::endl;
549+ ss << " if (ret != ACL_SUCCESS) { DLOGW(\"acl unload binary failed, ERROR: %d\", ret); }" << std::endl;
550+ ss << " g_pgo_bin_handle = nullptr;" << std::endl;
551+ ss << "}" << std::endl << std::endl;
552+ }
481}553}
482 554 
483-void TilingLib::GenPgoProfilingConstants(std::stringstream &ss) const {555+void TilingLib::GenPgoProfilingConstants(std::stringstream &ss, bool direct_link) const {
484 ss << "#define ALIGN_SIZE (8)" << std::endl;556 ss << "#define ALIGN_SIZE (8)" << std::endl;
485 ss << "#define ALIGN_BUFFER(buffer, align) \\" << std::endl;557 ss << "#define ALIGN_BUFFER(buffer, align) \\" << std::endl;
486 ss << " (((uintptr_t) (buffer) & ((align)-1)) ? ((buffer) + (align) - ((uintptr_t) (buffer) & ((align)-1))) : "558 ss << " (((uintptr_t) (buffer) & ((align)-1)) ? ((buffer) + (align) - ((uintptr_t) (buffer) & ((align)-1))) : "
@@ -491,6 +563,22 @@ void TilingLib::GenPgoProfilingConstants(std::stringstream &ss) const {
491 ss << "constexpr int max_flush_times = 5;" << std::endl;563 ss << "constexpr int max_flush_times = 5;" << std::endl;
492 ss << "constexpr size_t mspti_buffer_size = 16ULL * 1024 * 1024;" << std::endl;564 ss << "constexpr size_t mspti_buffer_size = 16ULL * 1024 * 1024;" << std::endl;
493 ss << "static double best_perf = DBL_MAX;" << std::endl;565 ss << "static double best_perf = DBL_MAX;" << std::endl;
566+ if (direct_link) {
567+ ss << R"(
568+static std::atomic<bool> g_mspti_activity_error{false};
569+static std::atomic<uint64_t> g_profiling_record_count{0U};
570+ 
571+void ClearProfilingRecords() {
572+ for (auto &item : g_profiling_map) { free(item.second); }
573+ g_profiling_map.clear();
574+ g_profiling_record_count.store(0U, std::memory_order_release);
575+}
576+ 
577+void ResetProfilingRound() {
578+ ClearProfilingRecords();
579+ g_mspti_activity_error = false;
580+})" << std::endl;
581+ }
494}582}
495 583 
496void TilingLib::GenPgoMsptiStringTable(std::stringstream &ss) const {584void TilingLib::GenPgoMsptiStringTable(std::stringstream &ss) const {
@@ -525,7 +613,26 @@ static const char* GetResultCodeString(msptiResult result) {
525})" << std::endl;613})" << std::endl;
526}614}
527 615 
528-void TilingLib::GenPgoMsptiRequest(std::stringstream &ss) const {616+void TilingLib::GenPgoMsptiRequest(std::stringstream &ss, bool direct_link) const {
617+ if (direct_link) {
618+ ss << R"(
619+void UserBufferRequest(uint8_t **buffer, size_t *size, size_t *records_num) {
620+ DLOGD("[mspti] UserBufferRequest...");
621+ uint8_t *mspti_buffer = reinterpret_cast<uint8_t *>(malloc(mspti_buffer_size + ALIGN_SIZE));
622+ if (mspti_buffer == nullptr) {
623+ DLOGE("[mspti] malloc mspti_buffer failed");
624+ g_mspti_activity_error = true;
625+ *buffer = nullptr;
626+ *size = 0;
627+ *records_num = 0;
628+ return;
629+ }
630+ *buffer = ALIGN_BUFFER(mspti_buffer, ALIGN_SIZE);
631+ *size = mspti_buffer_size;
632+ *records_num = 0;
633+})" << std::endl;
634+ return;
635+ }
529 ss << R"(636 ss << R"(
530void UserBufferRequest(uint8_t **buffer, size_t *size, size_t *records_num) {637void UserBufferRequest(uint8_t **buffer, size_t *size, size_t *records_num) {
531 DLOGD("[mspti] UserBufferRequest...");638 DLOGD("[mspti] UserBufferRequest...");
@@ -543,7 +650,44 @@ void UserBufferRequest(uint8_t **buffer, size_t *size, size_t *records_num) {
543})" << std::endl;650})" << std::endl;
544}651}
545 652 
546-void TilingLib::GenPgoMsptiComplete(std::stringstream &ss) const {653+void TilingLib::GenPgoDirectMsptiKernelHandlers(std::stringstream &ss) const {
654+ ss << R"(
655+void SavePgoKernel(const msptiActivityKernel *kernel) {
656+ if (kernel == nullptr) { g_mspti_activity_error = true; return; }
657+ auto *record_copy = static_cast<msptiActivity *>(malloc(sizeof(msptiActivityKernel)));
658+ if (record_copy == nullptr) { g_mspti_activity_error = true; return; }
659+ std::memcpy(record_copy, kernel, sizeof(msptiActivityKernel));
660+ if (!g_profiling_map.emplace(kernel->start, record_copy).second) {
661+ free(record_copy);
662+ g_mspti_activity_error = true;
663+ } else {
664+ g_profiling_record_count.fetch_add(1U, std::memory_order_release);
665+ }
666+}
667+)";
668+}
669+ 
670+void TilingLib::GenPgoDirectMsptiComplete(std::stringstream &ss) const {
671+ ss << R"(
672+void UserBufferComplete(uint8_t *buffer, size_t size, size_t valid_size) {
673+ DLOGD("[mspti] UserBufferComplete, buf addr: %" PRIuPTR ", size: %zu, valid size: %zu", (uintptr_t)buffer, size, valid_size);
674+ if (buffer == nullptr && valid_size > 0U) { g_mspti_activity_error = true; return; }
675+ msptiActivity *mspti_record = nullptr;
676+ msptiResult status = MSPTI_SUCCESS;
677+ while (valid_size > 0U) {
678+ status = msptiActivityGetNextRecord(buffer, valid_size, &mspti_record);
679+ if (status == MSPTI_ERROR_MAX_LIMIT_REACHED) { break; }
680+ if (status != MSPTI_SUCCESS) { g_mspti_activity_error = true; break; }
681+ if (mspti_record->kind == MSPTI_ACTIVITY_KIND_KERNEL) {
682+ auto *kernel = reinterpret_cast<msptiActivityKernel *>(mspti_record);
683+ SavePgoKernel(kernel);
684+ }
685+ }
686+ free(buffer);
687+})" << std::endl;
688+}
689+ 
690+void TilingLib::GenPgoLegacyMsptiComplete(std::stringstream &ss) const {
547 ss << R"(691 ss << R"(
548void UserBufferComplete(uint8_t *buffer, size_t size, size_t valid_size) {692void UserBufferComplete(uint8_t *buffer, size_t size, size_t valid_size) {
549 DLOGD("[mspti] UserBufferComplete, buf addr: %" PRIuPTR ", size: %zu, valid size: %zu", (uintptr_t)buffer, size, valid_size);693 DLOGD("[mspti] UserBufferComplete, buf addr: %" PRIuPTR ", size: %zu, valid size: %zu", (uintptr_t)buffer, size, valid_size);
@@ -575,7 +719,52 @@ void UserBufferComplete(uint8_t *buffer, size_t size, size_t valid_size) {
575})" << std::endl;719})" << std::endl;
576}720}
577 721 
578-void TilingLib::GenPgoMsptiToolFunction(std::stringstream &ss) const {722+void TilingLib::GenPgoMsptiComplete(std::stringstream &ss, bool direct_link) const {
723+ if (direct_link) {
724+ GenPgoDirectMsptiKernelHandlers(ss);
725+ GenPgoDirectMsptiComplete(ss);
726+ return;
727+ }
728+ GenPgoLegacyMsptiComplete(ss);
729+}
730+ 
731+void TilingLib::GenPgoMsptiToolFunction(std::stringstream &ss, bool direct_link) const {
732+ if (direct_link) {
733+ ss << R"(
734+msptiResult SetUpMspti(msptiSubscriberHandle *subscriber) {
735+ DLOGD("[mspti] setup mspti");
736+ *subscriber = nullptr;
737+ msptiResult result = msptiSubscribe(subscriber, nullptr, nullptr);
738+ if (result != MSPTI_SUCCESS) { return result; }
739+ result = msptiActivityRegisterCallbacks(UserBufferRequest, UserBufferComplete);
740+ if (result != MSPTI_SUCCESS) { msptiUnsubscribe(*subscriber); return result; }
741+ result = msptiActivityEnable(MSPTI_ACTIVITY_KIND_KERNEL);
742+ if (result != MSPTI_SUCCESS) { msptiUnsubscribe(*subscriber); }
743+ return result;
744+}
745+ 
746+msptiResult FlushPgoActivities(uint64_t expected_records) {
747+ if (g_profiling_record_count.load(std::memory_order_acquire) >= expected_records) { return MSPTI_SUCCESS; }
748+ msptiResult result = MSPTI_SUCCESS;
749+ for (int flush_count = 0; flush_count < max_flush_times; ++flush_count) {
750+ result = msptiActivityFlushAll(1);
751+ if (result != MSPTI_SUCCESS ||
752+ g_profiling_record_count.load(std::memory_order_acquire) >= expected_records) { break; }
753+ std::this_thread::sleep_for(std::chrono::milliseconds(10 * (flush_count + 1)));
754+ }
755+ return result;
756+}
757+ 
758+msptiResult TearDownMspti(msptiSubscriberHandle *subscriber) {
759+ DLOGD("[mspti] tear down mspti");
760+ msptiResult result = *subscriber == nullptr ? MSPTI_SUCCESS : msptiUnsubscribe(*subscriber);
761+ *subscriber = nullptr;
762+ const msptiResult flush_result = msptiActivityFlushAll(1);
763+ if (result == MSPTI_SUCCESS) { result = flush_result; }
764+ return result;
765+})" << std::endl;
766+ return;
767+ }
579 ss << R"(768 ss << R"(
580void SetUpMspti(msptiSubscriberHandle* subscriber) {769void SetUpMspti(msptiSubscriberHandle* subscriber) {
581 DLOGD("[mspti] setup mspti");770 DLOGD("[mspti] setup mspti");
@@ -591,12 +780,44 @@ void TearDownMspti(msptiSubscriberHandle *subscriber) {
591})" << std::endl;780})" << std::endl;
592}781}
593 782 
594-void TilingLib::GenPgoMsptiProfiling(std::stringstream &ss) const {783+void TilingLib::GenPgoMsptiProfiling(std::stringstream &ss, bool direct_link) const {
595- GenPgoProfilingConstants(ss);784+ GenPgoProfilingConstants(ss, direct_link);
596 GenPgoMsptiStringTable(ss);785 GenPgoMsptiStringTable(ss);
597- GenPgoMsptiRequest(ss);786+ GenPgoMsptiRequest(ss, direct_link);
598- GenPgoMsptiComplete(ss);787+ GenPgoMsptiComplete(ss, direct_link);
599- GenPgoMsptiToolFunction(ss);788+ GenPgoMsptiToolFunction(ss, direct_link);
789+}
790+ 
791+void TilingLib::GenPgoDirectBatchCallback(std::stringstream &ss) const {
792+ ss << R"( result = aclrtSynchronizeStream(g_stream);
793+ const uint64_t expected_records = batch_size * loop;
794+ const msptiResult teardown_result = TearDownMspti(&subscriber);
795+ const msptiResult flush_result = FlushPgoActivities(expected_records);
796+ if (result != ACL_SUCCESS || g_mspti_activity_error || teardown_result != MSPTI_SUCCESS ||
797+ flush_result != MSPTI_SUCCESS ||
798+ g_profiling_map.size() != expected_records) {
799+ DLOGE("invalid batch activity: sync=%" PRId64 ", flush=%d, teardown=%d, error=%d, actual=%zu, expected=%" PRIu64,
800+ result, flush_result, teardown_result, g_mspti_activity_error.load(), g_profiling_map.size(), expected_records);
801+ ClearProfilingRecords();
802+ return -1;
803+ }
804+ auto record = g_profiling_map.begin();
805+ for (uint64_t i = 0; i < batch_size; ++i) {
806+ uint64_t total_duration = 0;
807+ std::vector<uint64_t> durations;
808+ for (uint64_t j = 0; j < loop; ++j) {
809+ auto *kernel = reinterpret_cast<msptiActivityKernel *>(record->second);
810+ durations.push_back(kernel->end - kernel->start);
811+ ++record;
812+ }
813+ std::sort(durations.begin(), durations.end(), std::greater<uint64_t>());
814+ for (size_t k = 1; k < 6; ++k) { total_duration += durations[k]; }
815+ const double average_duration = static_cast<double>(total_duration) / 5;
816+ (begin + i)->best_perf = average_duration;
817+ if (best_perf > average_duration) { best_perf = average_duration; }
818+ }
819+ ClearProfilingRecords();
820+)";
600}821}
601 822 
602void TilingLib::GenPgoBatchCallback(std::stringstream &ss) const {823void TilingLib::GenPgoBatchCallback(std::stringstream &ss) const {
@@ -626,7 +847,7 @@ void TilingLib::GenPgoBatchCallback(std::stringstream &ss) const {
626 ss << " durations.push_back(kernel->end - kernel->start);" << std::endl;847 ss << " durations.push_back(kernel->end - kernel->start);" << std::endl;
627 ss << " std::advance(it, 1);" << std::endl;848 ss << " std::advance(it, 1);" << std::endl;
628 ss << " }" << std::endl;849 ss << " }" << std::endl;
629- ss << " std::sort(durations.begin(), durations.end(), std::greater<int>());" << std::endl;850+ ss << " std::sort(durations.begin(), durations.end(), std::greater<uint64_t>());" << std::endl;
630 ss << " for (size_t k = 1; k < 6; ++k) {" << std::endl;851 ss << " for (size_t k = 1; k < 6; ++k) {" << std::endl;
631 ss << " total_duration += durations[k];" << std::endl;852 ss << " total_duration += durations[k];" << std::endl;
632 ss << " }" << std::endl;853 ss << " }" << std::endl;
@@ -644,7 +865,47 @@ void TilingLib::GenPgoBatchCallback(std::stringstream &ss) const {
644 ss << " }" << std::endl;865 ss << " }" << std::endl;
645}866}
646 867 
647-void TilingLib::GenPgoBatchProcess(std::stringstream &ss) const {868+void TilingLib::GenPgoDirectBatchProcess(std::stringstream &ss) const {
869+ ss << R"(int ProfilingBatchProcess(uint32_t workspace_size, std::vector<AutofuseTilingDataPerf>::iterator begin,
870+ std::vector<AutofuseTilingDataPerf>::iterator end) {
871+ const uint64_t batch_size = end - begin;
872+ ResetProfilingRound();
873+ msptiSubscriberHandle subscriber = nullptr;
874+ if (SetUpMspti(&subscriber) != MSPTI_SUCCESS) { return -1; }
875+ static int64_t count = 0;
876+ ++count;
877+ int64_t result = 0;
878+ for (auto it = begin; it != end; ++it) {
879+ it->best_perf = DBL_MAX;
880+ AutofuseTilingData &tiling_data = it->tiling_data;
881+ if (UpdateLaunchParam(tiling_data) != ACL_SUCCESS) {
882+ TearDownMspti(&subscriber);
883+ ClearProfilingRecords();
884+ return -1;
885+ }
886+ for (uint64_t i = 0; i < loop; ++i) {
887+ result = WrapperOnlyLaunch(workspace_size, &tiling_data);
888+ if (result != 0) {
889+ DLOGE("ProfilingBatchProcess launch failed loop:%" PRIu64, i);
890+ TearDownMspti(&subscriber);
891+ ClearProfilingRecords();
892+ return -1;
893+ }
894+ }
895+ }
896+)";
897+ GenPgoDirectBatchCallback(ss);
898+ ss << R"( return 0;
899+}
900+ 
901+)";
902+}
903+ 
904+void TilingLib::GenPgoBatchProcess(std::stringstream &ss, bool direct_link) const {
905+ if (direct_link) {
906+ GenPgoDirectBatchProcess(ss);
907+ return;
908+ }
648 ss << "int ProfilingBatchProcess(uint32_t workspace_size, std::vector<AutofuseTilingDataPerf>::iterator begin, "909 ss << "int ProfilingBatchProcess(uint32_t workspace_size, std::vector<AutofuseTilingDataPerf>::iterator begin, "
649 "std::vector<AutofuseTilingDataPerf>::iterator end) {"910 "std::vector<AutofuseTilingDataPerf>::iterator end) {"
650 << std::endl;911 << std::endl;
@@ -673,10 +934,11 @@ void TilingLib::GenPgoBatchProcess(std::stringstream &ss) const {
673 ss << "}" << std::endl << std::endl;934 ss << "}" << std::endl << std::endl;
674}935}
675 936 
676-void TilingLib::GenPgoGetProfilingBatch(const ascir::FusedScheduledResult &fused_schedule_result,937+void TilingLib::GenPgoProfilingBatchSetup(std::stringstream &ss, bool direct_link) const {
677- std::stringstream &ss) const {938+ if (direct_link) {
678- ss << "extern \"C\" long int PGOGetProfilingBatch(" << PGOSearchFuncInputOutputCallBackDef(fused_schedule_result)939+ ss << " (void)tensor_args;" << std::endl;
679- << "void* stream, uint32_t workspace_size, std::vector<AutofuseTilingDataPerf> *profiles) {" << std::endl;940+ ss << " (void)stream;" << std::endl;
941+ }
680 ss << " int case_num = profiles->size();" << std::endl;942 ss << " int case_num = profiles->size();" << std::endl;
681 ss << " DLOGI(\"PGOGetProfilingBatch case_num:%d\", case_num);" << std::endl;943 ss << " DLOGI(\"PGOGetProfilingBatch case_num:%d\", case_num);" << std::endl;
682 ss << " if (workspace_size > 0) {" << std::endl;944 ss << " if (workspace_size > 0) {" << std::endl;
@@ -686,6 +948,13 @@ void TilingLib::GenPgoGetProfilingBatch(const ascir::FusedScheduledResult &fused
686 ss << " return FAILED;" << std::endl;948 ss << " return FAILED;" << std::endl;
687 ss << " }" << std::endl;949 ss << " }" << std::endl;
688 ss << " }" << std::endl;950 ss << " }" << std::endl;
951+}
952+ 
953+void TilingLib::GenPgoGetProfilingBatch(const ascir::FusedScheduledResult &fused_schedule_result, std::stringstream &ss,
954+ bool direct_link) const {
955+ ss << "extern \"C\" long int PGOGetProfilingBatch(" << PGOSearchFuncInputOutputCallBackDef(fused_schedule_result)
956+ << "void* stream, uint32_t workspace_size, std::vector<AutofuseTilingDataPerf> *profiles) {" << std::endl;
957+ GenPgoProfilingBatchSetup(ss, direct_link);
689 ss << " int64_t result = 0;" << std::endl;958 ss << " int64_t result = 0;" << std::endl;
690 ss << " auto it = profiles->begin();" << std::endl;959 ss << " auto it = profiles->begin();" << std::endl;
691 ss << " while (it != profiles->end()) {" << std::endl;960 ss << " while (it != profiles->end()) {" << std::endl;
@@ -700,6 +969,15 @@ void TilingLib::GenPgoGetProfilingBatch(const ascir::FusedScheduledResult &fused
700 ss << " break;" << std::endl;969 ss << " break;" << std::endl;
701 ss << " }" << std::endl;970 ss << " }" << std::endl;
702 ss << " }" << std::endl;971 ss << " }" << std::endl;
972+ if (direct_link) {
973+ ss << " if (result != 0) {" << std::endl;
974+ ss << " if (g_workspace != nullptr) {" << std::endl;
975+ ss << " aclrtFree(g_workspace);" << std::endl;
976+ ss << " g_workspace = nullptr;" << std::endl;
977+ ss << " }" << std::endl;
978+ ss << " return FAILED;" << std::endl;
979+ ss << " }" << std::endl;
980+ }
703 ss << " it = end_it;" << std::endl;981 ss << " it = end_it;" << std::endl;
704 ss << " }" << std::endl;982 ss << " }" << std::endl;
705 ss << " if (g_workspace != nullptr) {" << std::endl;983 ss << " if (g_workspace != nullptr) {" << std::endl;
@@ -708,12 +986,41 @@ void TilingLib::GenPgoGetProfilingBatch(const ascir::FusedScheduledResult &fused
708 ss << " DLOGE(\"free workspace failed, ERROR: %d\", ret);" << std::endl;986 ss << " DLOGE(\"free workspace failed, ERROR: %d\", ret);" << std::endl;
709 ss << " return FAILED;" << std::endl;987 ss << " return FAILED;" << std::endl;
710 ss << " }" << std::endl;988 ss << " }" << std::endl;
989+ if (direct_link) {
990+ ss << " g_workspace = nullptr;" << std::endl;
991+ }
711 ss << " }" << std::endl;992 ss << " }" << std::endl;
712 ss << " return 0;" << std::endl;993 ss << " return 0;" << std::endl;
713 ss << "}" << std::endl << std::endl;994 ss << "}" << std::endl << std::endl;
714}995}
715 996 
716-void TilingLib::GenPgoProfilingCallback(std::stringstream &ss) const {997+void TilingLib::GenPgoDirectProfilingCallback(std::stringstream &ss) const {
998+ ss << R"( result = aclrtSynchronizeStream(g_stream);
999+ const msptiResult teardown_result = TearDownMspti(&subscriber);
1000+ const msptiResult flush_result = FlushPgoActivities(loop);
1001+ if (result != ACL_SUCCESS || g_mspti_activity_error || teardown_result != MSPTI_SUCCESS ||
1002+ flush_result != MSPTI_SUCCESS ||
1003+ g_profiling_map.size() != loop) {
1004+ DLOGE("invalid activity: sync=%" PRId64 ", flush=%d, teardown=%d, error=%d, actual=%zu, expected=%" PRIu64,
1005+ result, flush_result, teardown_result, g_mspti_activity_error.load(), g_profiling_map.size(), loop);
1006+ ClearProfilingRecords();
1007+ return -1;
1008+ }
1009+ uint64_t total_duration = 0;
1010+ std::vector<uint64_t> durations;
1011+ for (const auto &pair : g_profiling_map) {
1012+ auto *kernel = reinterpret_cast<msptiActivityKernel *>(pair.second);
1013+ durations.push_back(kernel->end - kernel->start);
1014+ }
1015+ std::sort(durations.begin(), durations.end(), std::greater<uint64_t>());
1016+ for (size_t i = 1; i < 6; ++i) { total_duration += durations[i]; }
1017+ *outCostTime = static_cast<double>(total_duration) / 5;
1018+ if (best_perf > *outCostTime) { best_perf = *outCostTime; }
1019+ ClearProfilingRecords();
1020+)";
1021+}
1022+ 
1023+void TilingLib::GenPgoLegacyProfilingCallback(std::stringstream &ss) const {
717 ss << " result = aclrtSynchronizeStream(g_stream);" << std::endl;1024 ss << " result = aclrtSynchronizeStream(g_stream);" << std::endl;
718 ss << " if (result != 0) {" << std::endl;1025 ss << " if (result != 0) {" << std::endl;
719 ss << " DLOGE(\"sync stream failed\");" << std::endl;1026 ss << " DLOGE(\"sync stream failed\");" << std::endl;
@@ -744,7 +1051,7 @@ void TilingLib::GenPgoProfilingCallback(std::stringstream &ss) const {
744 ss << " durations.push_back(kernel->end - kernel->start);" << std::endl;1051 ss << " durations.push_back(kernel->end - kernel->start);" << std::endl;
745 ss << " DLOGD(\"kernel duration:%\" PRIu64 \"\", kernel->end - kernel->start);" << std::endl;1052 ss << " DLOGD(\"kernel duration:%\" PRIu64 \"\", kernel->end - kernel->start);" << std::endl;
746 ss << " }" << std::endl;1053 ss << " }" << std::endl;
747- ss << " std::sort(durations.begin(), durations.end(), std::greater<int>());" << std::endl;1054+ ss << " std::sort(durations.begin(), durations.end(), std::greater<uint64_t>());" << std::endl;
748 ss << " for (size_t i = 1; i < 6; ++i) {" << std::endl;1055 ss << " for (size_t i = 1; i < 6; ++i) {" << std::endl;
749 ss << " total_duration += durations[i];" << std::endl;1056 ss << " total_duration += durations[i];" << std::endl;
750 ss << " }" << std::endl;1057 ss << " }" << std::endl;
@@ -762,10 +1069,19 @@ void TilingLib::GenPgoProfilingCallback(std::stringstream &ss) const {
762 ss << " }" << std::endl;1069 ss << " }" << std::endl;
763}1070}
764 1071 
765-void TilingLib::GenPgoGetProfiling(const ascir::FusedScheduledResult &fused_schedule_result,1072+void TilingLib::GenPgoProfilingCallback(std::stringstream &ss, bool direct_link) const {
766- std::stringstream &ss) const {1073+ if (direct_link) {
767- ss << "extern \"C\" long int PGOGetProfiling(" << PGOSearchFuncInputOutputCallBackDef(fused_schedule_result)1074+ GenPgoDirectProfilingCallback(ss);
768- << "void *stream, uint32_t workspace_size, AutofuseTilingData *tiling_data, double *outCostTime) {" << std::endl;1075+ return;
1076+ }
1077+ GenPgoLegacyProfilingCallback(ss);
1078+}
1079+ 
1080+void TilingLib::GenPgoProfilingSetup(std::stringstream &ss, bool direct_link) const {
1081+ if (direct_link) {
1082+ ss << " (void)tensor_args;" << std::endl;
1083+ ss << " (void)stream;" << std::endl;
1084+ }
769 ss << " if (workspace_size > 0) {" << std::endl;1085 ss << " if (workspace_size > 0) {" << std::endl;
770 ss << " auto ret = aclrtMalloc(&g_workspace, workspace_size, ACL_MEM_MALLOC_HUGE_FIRST);" << std::endl;1086 ss << " auto ret = aclrtMalloc(&g_workspace, workspace_size, ACL_MEM_MALLOC_HUGE_FIRST);" << std::endl;
771 ss << " if (ret != ACL_SUCCESS) {" << std::endl;1087 ss << " if (ret != ACL_SUCCESS) {" << std::endl;
@@ -773,33 +1089,74 @@ void TilingLib::GenPgoGetProfiling(const ascir::FusedScheduledResult &fused_sche
773 ss << " return FAILED;" << std::endl;1089 ss << " return FAILED;" << std::endl;
774 ss << " }" << std::endl;1090 ss << " }" << std::endl;
775 ss << " }" << std::endl;1091 ss << " }" << std::endl;
776- ss << " g_profiling_map.clear();" << std::endl;1092+ if (direct_link) {
777- ss << " msptiSubscriberHandle subscriber;" << std::endl;1093+ ss << " ResetProfilingRound();" << std::endl;
778- ss << " SetUpMspti(&subscriber);" << std::endl << std::endl;1094+ ss << " msptiSubscriberHandle subscriber = nullptr;" << std::endl;
1095+ ss << " if (SetUpMspti(&subscriber) != MSPTI_SUCCESS) {" << std::endl;
1096+ ss << " if (g_workspace != nullptr) { aclrtFree(g_workspace); g_workspace = nullptr; }" << std::endl;
1097+ ss << " return -1;" << std::endl;
1098+ ss << " }" << std::endl << std::endl;
1099+ } else {
1100+ ss << " g_profiling_map.clear();" << std::endl;
1101+ ss << " msptiSubscriberHandle subscriber;" << std::endl;
1102+ ss << " SetUpMspti(&subscriber);" << std::endl << std::endl;
1103+ }
779 ss << " int64_t result = -1;" << std::endl;1104 ss << " int64_t result = -1;" << std::endl;
780 ss << " *outCostTime = DBL_MAX;" << std::endl;1105 ss << " *outCostTime = DBL_MAX;" << std::endl;
781 ss << " static int64_t count = 0;" << std::endl;1106 ss << " static int64_t count = 0;" << std::endl;
782 ss << " count++;" << std::endl << std::endl;1107 ss << " count++;" << std::endl << std::endl;
1108+}
783 1109 
784- ss << " UpdateLaunchParam(*tiling_data);" << std::endl;1110+void TilingLib::GenPgoProfilingLaunch(std::stringstream &ss, bool direct_link) const {
1111+ if (direct_link) {
1112+ ss << " if (UpdateLaunchParam(*tiling_data) != ACL_SUCCESS) {" << std::endl;
1113+ ss << " TearDownMspti(&subscriber);" << std::endl;
1114+ ss << " ClearProfilingRecords();" << std::endl;
1115+ ss << " if (g_workspace != nullptr) { aclrtFree(g_workspace); g_workspace = nullptr; }" << std::endl;
1116+ ss << " return -1;" << std::endl;
1117+ ss << " }" << std::endl;
1118+ } else {
1119+ ss << " UpdateLaunchParam(*tiling_data);" << std::endl;
1120+ }
785 ss << " for (uint64_t j = 0; j < loop; ++j) {" << std::endl;1121 ss << " for (uint64_t j = 0; j < loop; ++j) {" << std::endl;
786 ss << " result = WrapperOnlyLaunch(workspace_size, tiling_data);" << std::endl;1122 ss << " result = WrapperOnlyLaunch(workspace_size, tiling_data);" << std::endl;
787 ss << " if (result != 0) {" << std::endl;1123 ss << " if (result != 0) {" << std::endl;
788 ss << " DLOGE(\"launch failed loop:%\" PRIu64 \"\", j);" << std::endl;1124 ss << " DLOGE(\"launch failed loop:%\" PRIu64 \"\", j);" << std::endl;
789 ss << " TearDownMspti(&subscriber);" << std::endl;1125 ss << " TearDownMspti(&subscriber);" << std::endl;
1126+ if (direct_link) {
1127+ ss << " ClearProfilingRecords();" << std::endl;
1128+ ss << " if (g_workspace != nullptr) { aclrtFree(g_workspace); g_workspace = nullptr; }" << std::endl;
1129+ }
790 ss << " return -1;" << std::endl;1130 ss << " return -1;" << std::endl;
791 ss << " }" << std::endl;1131 ss << " }" << std::endl;
792 ss << " }" << std::endl << std::endl;1132 ss << " }" << std::endl << std::endl;
1133+}
793 1134 
1135+void TilingLib::GenPgoProfilingWorkspaceCleanup(std::stringstream &ss, bool direct_link) const {
794 ss << " if (g_workspace != nullptr) {" << std::endl;1136 ss << " if (g_workspace != nullptr) {" << std::endl;
795 ss << " auto ret = aclrtFree(g_workspace);" << std::endl;1137 ss << " auto ret = aclrtFree(g_workspace);" << std::endl;
796 ss << " if (ret != ACL_SUCCESS) {" << std::endl;1138 ss << " if (ret != ACL_SUCCESS) {" << std::endl;
797 ss << " DLOGE(\"free workspace failed, ERROR: %d\", ret);" << std::endl;1139 ss << " DLOGE(\"free workspace failed, ERROR: %d\", ret);" << std::endl;
798 ss << " TearDownMspti(&subscriber);" << std::endl;1140 ss << " TearDownMspti(&subscriber);" << std::endl;
1141+ if (direct_link) {
1142+ ss << " ClearProfilingRecords();" << std::endl;
1143+ }
799 ss << " return FAILED;" << std::endl;1144 ss << " return FAILED;" << std::endl;
800 ss << " }" << std::endl;1145 ss << " }" << std::endl;
1146+ if (direct_link) {
1147+ ss << " g_workspace = nullptr;" << std::endl;
1148+ }
801 ss << " }" << std::endl;1149 ss << " }" << std::endl;
802- GenPgoProfilingCallback(ss);1150+}
1151+ 
1152+void TilingLib::GenPgoGetProfiling(const ascir::FusedScheduledResult &fused_schedule_result, std::stringstream &ss,
1153+ bool direct_link) const {
1154+ ss << "extern \"C\" long int PGOGetProfiling(" << PGOSearchFuncInputOutputCallBackDef(fused_schedule_result)
1155+ << "void *stream, uint32_t workspace_size, AutofuseTilingData *tiling_data, double *outCostTime) {" << std::endl;
1156+ GenPgoProfilingSetup(ss, direct_link);
1157+ GenPgoProfilingLaunch(ss, direct_link);
1158+ GenPgoProfilingWorkspaceCleanup(ss, direct_link);
1159+ GenPgoProfilingCallback(ss, direct_link);
803 ss << " return 0;" << std::endl;1160 ss << " return 0;" << std::endl;
804 ss << "}" << std::endl << std::endl;1161 ss << "}" << std::endl << std::endl;
805}1162}
@@ -861,10 +1218,6 @@ void TilingLib::GenPgoStaticFunc(const ascir::FusedScheduledResult &fused_schedu
861}1218}
862 1219 
863void TilingLib::GenPgoProfiling(const ascir::FusedScheduledResult &fused_schedule_result, std::stringstream &ss) const {1220void TilingLib::GenPgoProfiling(const ascir::FusedScheduledResult &fused_schedule_result, std::stringstream &ss) const {
864- GenPgoMsptiProfiling(ss);
865- GenPgoBatchProcess(ss);
866- GenPgoGetProfilingBatch(fused_schedule_result, ss);
867- GenPgoGetProfiling(fused_schedule_result, ss);
868 ss << "typedef int64_t (*PGOSearchType)(char *search_file, char *config_file, AutofuseTilingData *tiling_data, "1221 ss << "typedef int64_t (*PGOSearchType)(char *search_file, char *config_file, AutofuseTilingData *tiling_data, "
869 "uint32_t *workspace_size, uint32_t *blockDim, void *resource_limit, "1222 "uint32_t *workspace_size, uint32_t *blockDim, void *resource_limit, "
870 << PGOSearchFuncInputOutputCallBackDef(fused_schedule_result)1223 << PGOSearchFuncInputOutputCallBackDef(fused_schedule_result)
@@ -948,18 +1301,21 @@ void TilingLib::GenPgoEnvInit(const ascir::FusedScheduledResult &fused_schedule_
948 ss << " }" << std::endl;1301 ss << " }" << std::endl;
949}1302}
950 1303 
951-void TilingLib::GenPgoLaunchKernelInit(std::stringstream &ss) const {1304+void TilingLib::GenInductorPgoKernelFunctionInit(std::stringstream &ss) const {
952- ss << " if (!inited) {" << std::endl;1305+ ss << " aclrtFuncHandle func_handle = nullptr;" << std::endl;
953- ss << " auto ret = aclrtBinaryLoadFromFile(g_kernel_o_file.c_str(), nullptr, &bin_handle);" << std::endl;1306+ ss << " ret = aclrtBinaryGetFunction(g_pgo_bin_handle, kInductorPgoKernelName, &func_handle);" << std::endl;
954 ss << " if (ret != ACL_SUCCESS) {" << std::endl;1307 ss << " if (ret != ACL_SUCCESS) {" << std::endl;
955- ss << " DLOGE(\"acl load binary from file failed, ERROR: %d\", ret);" << std::endl;1308+ ss << " DLOGE(\"acl get function failed, ERROR: %d\", ret);" << std::endl;
956 ss << " return FAILED;" << std::endl;1309 ss << " return FAILED;" << std::endl;
957 ss << " }" << std::endl;1310 ss << " }" << std::endl;
1311+ ss << " std::fill(func_handles.begin(), func_handles.end(), func_handle);" << std::endl;
1312+}
1313+ 
1314+void TilingLib::GenPgoKernelFunctionsInit(const std::string &bin_handle, std::stringstream &ss) const {
958 ss << " if (g_is_static_kernel) {" << std::endl;1315 ss << " if (g_is_static_kernel) {" << std::endl;
959 ss << " aclrtFuncHandle func_handle = nullptr;" << std::endl;1316 ss << " aclrtFuncHandle func_handle = nullptr;" << std::endl;
960- ss << " ret = aclrtBinaryGetFunction(bin_handle, (g_kernel_name + \"_\" + std::to_string(tiling_key)).c_str(), "1317+ ss << " ret = aclrtBinaryGetFunction(" << bin_handle
961- "&func_handle);"1318+ << ", (g_kernel_name + \"_\" + std::to_string(tiling_key)).c_str(), &func_handle);" << std::endl;
962- << std::endl;
963 ss << " if (ret != ACL_SUCCESS) {" << std::endl;1319 ss << " if (ret != ACL_SUCCESS) {" << std::endl;
964 ss << " DLOGE(\"acl get function failed, ERROR: %d\", ret);" << std::endl;1320 ss << " DLOGE(\"acl get function failed, ERROR: %d\", ret);" << std::endl;
965 ss << " return FAILED;" << std::endl;1321 ss << " return FAILED;" << std::endl;
@@ -968,9 +1324,8 @@ void TilingLib::GenPgoLaunchKernelInit(std::stringstream &ss) const {
968 ss << " } else {" << std::endl;1324 ss << " } else {" << std::endl;
969 ss << " for (uint64_t i = 0; i < tiling_key_count; ++i) {" << std::endl;1325 ss << " for (uint64_t i = 0; i < tiling_key_count; ++i) {" << std::endl;
970 ss << " aclrtFuncHandle func_handle = nullptr;" << std::endl;1326 ss << " aclrtFuncHandle func_handle = nullptr;" << std::endl;
971- ss << " ret = aclrtBinaryGetFunction(bin_handle, (g_kernel_name + \"_\" + std::to_string(i)).c_str(), "1327+ ss << " ret = aclrtBinaryGetFunction(" << bin_handle
972- "&func_handle);"1328+ << ", (g_kernel_name + \"_\" + std::to_string(i)).c_str(), &func_handle);" << std::endl;
973- << std::endl;
974 ss << " if (ret != ACL_SUCCESS) {" << std::endl;1329 ss << " if (ret != ACL_SUCCESS) {" << std::endl;
975 ss << " DLOGE(\"acl get function failed, ERROR: %d\", ret);" << std::endl;1330 ss << " DLOGE(\"acl get function failed, ERROR: %d\", ret);" << std::endl;
976 ss << " return FAILED;" << std::endl;1331 ss << " return FAILED;" << std::endl;
@@ -978,6 +1333,21 @@ void TilingLib::GenPgoLaunchKernelInit(std::stringstream &ss) const {
978 ss << " func_handles[i] = func_handle;" << std::endl;1333 ss << " func_handles[i] = func_handle;" << std::endl;
979 ss << " }" << std::endl;1334 ss << " }" << std::endl;
980 ss << " }" << std::endl;1335 ss << " }" << std::endl;
1336+}
1337+ 
1338+void TilingLib::GenPgoLaunchKernelInit(std::stringstream &ss, bool direct_link) const {
1339+ const std::string bin_handle = direct_link ? "g_pgo_bin_handle" : "bin_handle";
1340+ ss << " if (!inited) {" << std::endl;
1341+ ss << " auto ret = aclrtBinaryLoadFromFile(g_kernel_o_file.c_str(), nullptr, &" << bin_handle << ");" << std::endl;
1342+ ss << " if (ret != ACL_SUCCESS) {" << std::endl;
1343+ ss << " DLOGE(\"acl load binary from file failed, ERROR: %d\", ret);" << std::endl;
1344+ ss << " return FAILED;" << std::endl;
1345+ ss << " }" << std::endl;
1346+ if (direct_link) {
1347+ GenInductorPgoKernelFunctionInit(ss);
1348+ } else {
1349+ GenPgoKernelFunctionsInit(bin_handle, ss);
1350+ }
981 const auto backend_spce = optimize::BackendSpec::GetInstance();1351 const auto backend_spce = optimize::BackendSpec::GetInstance();
982 if (backend_spce != nullptr && backend_spce->set_local_memory_size > 0) {1352 if (backend_spce != nullptr && backend_spce->set_local_memory_size > 0) {
983 ss << " local_memory_size_attr.id = ACL_RT_LAUNCH_KERNEL_ATTR_DYN_UBUF_SIZE;" << std::endl;1353 ss << " local_memory_size_attr.id = ACL_RT_LAUNCH_KERNEL_ATTR_DYN_UBUF_SIZE;" << std::endl;
Mautofuse/codegen/codegen_tiling_pgo_memory.cpp+323-114
@@ -11,6 +11,8 @@
11#include "codegen_tiling.h"11#include "codegen_tiling.h"
12 12 
13#include <algorithm>13#include <algorithm>
14+#include <limits>
15+#include <set>
14#include <unordered_map>16#include <unordered_map>
15 17 
16#include "ascir_ops.h"18#include "ascir_ops.h"
@@ -24,6 +26,323 @@ using namespace af::ops;
24using namespace ascgen_utils;26using namespace ascgen_utils;
25using namespace ascir;27using namespace ascir;
26 28 
29+std::string TilingLib::CalculateTensorMemorySizeStr(const ascir::TensorAttr &tensor) const {
30+ return CalculateTensorMemorySizeStr(tensor, af::ops::Zero);
31+}
32+ 
33+namespace {
34+bool GetTensorTypeSize(ge::DataType dtype, uint64_t &type_size) {
35+ static const std::unordered_map<ge::DataType, uint64_t> type_size_map = {
36+ {ge::DT_FLOAT, 4U}, {ge::DT_FLOAT16, 2U}, {ge::DT_INT8, 1U}, {ge::DT_INT16, 2U}, {ge::DT_INT32, 4U},
37+ {ge::DT_INT64, 8U}, {ge::DT_UINT8, 1U}, {ge::DT_UINT16, 2U}, {ge::DT_UINT32, 4U}, {ge::DT_UINT64, 8U},
38+ {ge::DT_DOUBLE, 8U}, {ge::DT_BF16, 2U}, {ge::DT_BOOL, 1U}};
39+ const auto iter = type_size_map.find(dtype);
40+ if (iter == type_size_map.end()) {
41+ return false;
42+ }
43+ type_size = iter->second;
44+ return true;
45+}
46+ 
47+bool CheckedPgoSizeAdd(uint64_t lhs, uint64_t rhs, uint64_t &result) {
48+ constexpr uint64_t kMaxPgoIoSize = static_cast<uint64_t>(std::numeric_limits<int64_t>::max());
49+ if (lhs > kMaxPgoIoSize - rhs) {
50+ return false;
51+ }
52+ result = lhs + rhs;
53+ return true;
54+}
55+ 
56+bool CheckedPgoSizeMul(uint64_t lhs, uint64_t rhs, uint64_t &result) {
57+ constexpr uint64_t kMaxPgoIoSize = static_cast<uint64_t>(std::numeric_limits<int64_t>::max());
58+ if (lhs != 0U && rhs > kMaxPgoIoSize / lhs) {
59+ return false;
60+ }
61+ result = lhs * rhs;
62+ return true;
63+}
64+ 
65+enum class ConstMemorySizeStatus { kSuccess, kSymbolic, kInvalid };
66+ 
67+ConstMemorySizeStatus CalculateConstMemorySize(const ascir::TensorAttr &tensor, const af::Expression &element_offset,
68+ uint64_t type_size, uint64_t &memory_size) {
69+ bool all_const = element_offset.IsConstExpr();
70+ int64_t offset = 0;
71+ if ((all_const && !element_offset.GetConstValue(offset)) || offset < 0) {
72+ return ConstMemorySizeStatus::kInvalid;
73+ }
74+ std::vector<std::pair<int64_t, int64_t>> dims;
75+ bool has_zero_repeat = false;
76+ for (size_t i = 0UL; i < tensor.attr.repeats.size(); ++i) {
77+ int64_t repeat = 0;
78+ int64_t stride = 0;
79+ const bool dim_const = tensor.attr.repeats[i].IsConstExpr() && tensor.attr.strides[i].IsConstExpr();
80+ if (dim_const && (!tensor.attr.repeats[i].GetConstValue(repeat) || !tensor.attr.strides[i].GetConstValue(stride))) {
81+ return ConstMemorySizeStatus::kInvalid;
82+ }
83+ if (dim_const && (repeat < 0 || stride < 0)) {
84+ return ConstMemorySizeStatus::kInvalid;
85+ }
86+ has_zero_repeat = has_zero_repeat || (dim_const && repeat == 0);
87+ all_const = all_const && dim_const;
88+ dims.emplace_back(repeat, stride);
89+ }
90+ if (has_zero_repeat) {
91+ memory_size = 0U;
92+ return ConstMemorySizeStatus::kSuccess;
93+ }
94+ if (!all_const) {
95+ return ConstMemorySizeStatus::kSymbolic;
96+ }
97+ uint64_t span = static_cast<uint64_t>(offset) + 1U;
98+ for (const auto &[repeat, stride] : dims) {
99+ uint64_t contribution = 0U;
100+ if (!CheckedPgoSizeMul(static_cast<uint64_t>(repeat - 1), static_cast<uint64_t>(stride), contribution) ||
101+ !CheckedPgoSizeAdd(span, contribution, span)) {
102+ return ConstMemorySizeStatus::kInvalid;
103+ }
104+ }
105+ return CheckedPgoSizeMul(span, type_size, memory_size) ? ConstMemorySizeStatus::kSuccess
106+ : ConstMemorySizeStatus::kInvalid;
107+}
108+} // namespace
109+ 
110+std::string TilingLib::CalculateTensorMemorySizeStr(const ascir::TensorAttr &tensor,
111+ const af::Expression &element_offset) const {
112+ const auto dtype = tensor.attr.dtype.operator ge::DataType();
113+ uint64_t type_size = 0U;
114+ if (!GetTensorTypeSize(dtype, type_size)) {
115+ GELOGE(ge::GRAPH_FAILED, "Unsupported data type: %d", static_cast<int32_t>(dtype));
116+ return "0";
117+ }
118+ if (tensor.attr.repeats.empty() || tensor.attr.repeats.size() != tensor.attr.strides.size()) {
119+ GELOGE(ge::GRAPH_FAILED, "Invalid repeats or strides when calculating tensor memory size");
120+ return "0";
121+ }
122+ uint64_t const_memory_size = 0U;
123+ const auto const_status = CalculateConstMemorySize(tensor, element_offset, type_size, const_memory_size);
124+ if (const_status == ConstMemorySizeStatus::kSuccess) {
125+ return std::to_string(const_memory_size);
126+ }
127+ if (const_status == ConstMemorySizeStatus::kInvalid) {
128+ GELOGE(ge::GRAPH_FAILED, "Tensor memory size has invalid layout or overflow");
129+ return "0";
130+ }
131+ af::Expression element_span = af::sym::Add(element_offset, af::ops::One);
132+ for (size_t i = 0UL; i < tensor.attr.repeats.size(); ++i) {
133+ const auto repeat_span = af::sym::Sub(tensor.attr.repeats[i], af::ops::One);
134+ element_span = af::sym::Add(element_span, af::sym::Mul(repeat_span, tensor.attr.strides[i]));
135+ }
136+ const auto type_size_expr = af::Expression::Parse(std::to_string(type_size).c_str());
137+ af::Expression need_malloc_size = af::sym::Mul(element_span.Simplify(), type_size_expr).Simplify();
138+ GELOGD("Tensor element span: %s, need malloc size: %s", element_span.Str().get(), need_malloc_size.Str().get());
139+ return std::string(need_malloc_size.Str().get());
140+}
141+ 
142+namespace {
143+enum class PgoIoKind { kInput, kOutput };
144+using PgoVarReplacements = std::vector<std::pair<af::Expression, af::Expression>>;
145+ 
146+PgoVarReplacements BuildPgoVarReplacements(const ascir::ScheduledResult &result, size_t group_id) {
147+ PgoVarReplacements replacements;
148+ const auto dst_group_iter = result.var_relations.find(group_id);
149+ if (dst_group_iter == result.var_relations.end()) {
150+ return replacements;
151+ }
152+ for (const auto &relations_by_src : dst_group_iter->second) {
153+ for (const auto &[dst_var_name, src_expr] : relations_by_src.second) {
154+ replacements.emplace_back(af::Expression::Parse(dst_var_name.c_str()), src_expr);
155+ }
156+ }
157+ return replacements;
158+}
159+ 
160+af::Expression ApplyPgoVarReplacements(const af::Expression &expr, const PgoVarReplacements &replacements) {
161+ return replacements.empty() ? expr : expr.Replace(replacements).Simplify();
162+}
163+ 
164+ascir::TensorAttr ApplyPgoVarReplacements(const ascir::TensorAttr &tensor, const PgoVarReplacements &replacements) {
165+ ascir::TensorAttr replaced_tensor = tensor;
166+ for (auto &repeat : replaced_tensor.attr.repeats) {
167+ repeat = ApplyPgoVarReplacements(repeat, replacements);
168+ }
169+ for (auto &stride : replaced_tensor.attr.strides) {
170+ stride = ApplyPgoVarReplacements(stride, replacements);
171+ }
172+ return replaced_tensor;
173+}
174+ 
175+int64_t GetPgoIoIndex(const af::AscNodePtr &node, int64_t fallback_index) {
176+ int64_t index = fallback_index;
177+ if (node == nullptr || node->attr.ir_attr == nullptr) {
178+ return index;
179+ }
180+ (void)node->attr.ir_attr->GetAttrValue("index", index);
181+ return index;
182+}
183+ 
184+bool IsPgoIoNode(const af::AscNodePtr &node, int64_t io_index, PgoIoKind kind) {
185+ const bool type_matches =
186+ kind == PgoIoKind::kInput ? IsOps<Data>(node) || IsOps<ScalarData>(node) : IsOps<Output>(node);
187+ return type_matches && GetPgoIoIndex(node, -1) == io_index;
188+}
189+ 
190+template <typename TensorVisitor>
191+void VisitPgoNodeTensors(const af::AscNodePtr &node, PgoIoKind kind, TensorVisitor &visitor) {
192+ if (kind == PgoIoKind::kOutput) {
193+ const auto &input_nodes = node->GetInDataNodes();
194+ const auto *access_node = input_nodes.empty() ? nullptr : static_cast<const af::AscNode *>(input_nodes.at(0).get());
195+ visitor(node->inputs[0], access_node);
196+ return;
197+ }
198+ for (auto *out_node : node->GetOutNodesPtr()) {
199+ auto *asc_out_node = static_cast<af::AscNode *>(out_node);
200+ if (asc_out_node != nullptr) {
201+ visitor(asc_out_node->outputs[0], asc_out_node);
202+ }
203+ }
204+}
205+ 
206+template <typename TensorVisitor>
207+void VisitPgoGraphTensors(const af::AscGraph &graph, int64_t io_index, PgoIoKind kind,
208+ const PgoVarReplacements &replacements, TensorVisitor &visitor) {
209+ for (const auto &node : graph.GetAllNodes()) {
210+ if (IsPgoIoNode(node, io_index, kind)) {
211+ auto replace_and_visit = [&replacements, &visitor](const ascir::TensorAttr &tensor,
212+ const af::AscNode *access_node) {
213+ visitor(ApplyPgoVarReplacements(tensor, replacements), access_node, replacements);
214+ };
215+ VisitPgoNodeTensors(node, kind, replace_and_visit);
216+ }
217+ }
218+}
219+ 
220+template <typename TensorVisitor>
221+void VisitPgoCandidateTensors(const ascir::FusedScheduledResult &fused_schedule_result, int64_t io_index,
222+ PgoIoKind kind, TensorVisitor &visitor) {
223+ for (const auto &scheduled_results : fused_schedule_result.node_idx_to_scheduled_results) {
224+ for (const auto &result : scheduled_results) {
225+ for (size_t group_id = 0UL; group_id < result.schedule_groups.size(); ++group_id) {
226+ const auto replacements = BuildPgoVarReplacements(result, group_id);
227+ const auto &schedule_group = result.schedule_groups[group_id];
228+ for (const auto &impl_graph : schedule_group.impl_graphs) {
229+ VisitPgoGraphTensors(impl_graph, io_index, kind, replacements, visitor);
230+ }
231+ }
232+ }
233+ }
234+}
235+ 
236+void AppendPgoTensorMalloc(std::stringstream &ss, const std::string &tensor_name,
237+ const std::vector<std::string> &size_expressions) {
238+ if (size_expressions.empty()) {
239+ return;
240+ }
241+ if (std::find(size_expressions.begin(), size_expressions.end(), "") != size_expressions.end()) {
242+ ss << " DLOGE(\"Invalid or symbolic PGO " << tensor_name << " memory size\");" << std::endl;
243+ ss << " return FAILED;" << std::endl;
244+ return;
245+ }
246+ const std::string size_name = tensor_name + "_size";
247+ ss << " size_t " << size_name << " = " << size_expressions[0] << ";" << std::endl;
248+ for (size_t i = 1UL; i < size_expressions.size(); ++i) {
249+ ss << " " << size_name << " = std::max(" << size_name << ", static_cast<size_t>(" << size_expressions[i] << "));"
250+ << std::endl;
251+ }
252+ ss << " ret = aclrtMalloc(&" << tensor_name << ", " << size_name << ", ACL_MEM_MALLOC_HUGE_FIRST);" << std::endl;
253+ ss << " if (ret != ACL_SUCCESS) {" << std::endl;
254+ ss << " DLOGE(\"aclrtMalloc " << tensor_name << " failed. ERROR: %d\", ret);" << std::endl;
255+ ss << " return FAILED;" << std::endl;
256+ ss << " }" << std::endl;
257+}
258+} // namespace
259+ 
260+std::vector<std::string> TilingLib::CalculatePgoIoMemorySizeStrs(
261+ const ascir::FusedScheduledResult &fused_schedule_result, int64_t io_index, bool is_input,
262+ const ascir::TensorAttr &fallback_tensor) const {
263+ std::vector<std::string> size_expressions;
264+ std::set<std::string> seen_expressions;
265+ auto append_size = [this, &size_expressions, &seen_expressions](const ascir::TensorAttr &tensor,
266+ const af::AscNode *access_node,
267+ const PgoVarReplacements &replacements) {
268+ af::Expression element_offset = af::ops::Zero;
269+ if (access_node != nullptr && access_node->attr.ir_attr != nullptr) {
270+ (void)access_node->attr.ir_attr->GetAttrValue("offset", element_offset);
271+ }
272+ element_offset = ApplyPgoVarReplacements(element_offset, replacements);
273+ const std::string size_expression = CalculateTensorMemorySizeStr(tensor, element_offset);
274+ if (size_expression == "0" || !af::Expression::Parse(size_expression.c_str()).IsConstExpr()) {
275+ GELOGD("Reject invalid or symbolic PGO candidate memory size: %s", size_expression.c_str());
276+ if (seen_expressions.emplace("").second) {
277+ size_expressions.emplace_back("");
278+ }
279+ return;
280+ }
281+ if (seen_expressions.emplace(size_expression).second) {
282+ size_expressions.emplace_back(size_expression);
283+ }
284+ };
285+ append_size(fallback_tensor, nullptr, {});
286+ const PgoIoKind kind = is_input ? PgoIoKind::kInput : PgoIoKind::kOutput;
287+ VisitPgoCandidateTensors(fused_schedule_result, io_index, kind, append_size);
288+ return size_expressions;
289+}
290+ 
291+std::string TilingLib::PGOSearchTensorMallocDef(const ascir::FusedScheduledResult &fused_schedule_result) const {
292+ std::stringstream ss;
293+ int index = 0;
294+ for (auto &input : fused_schedule_result.input_nodes) {
295+ if (input->GetOutNodesPtr().empty()) {
296+ continue;
297+ }
298+ af::Node *out_node = input->GetOutNodesPtr()[0];
299+ af::AscNode *asc_out_node = static_cast<af::AscNode *>(out_node);
300+ const auto size_expressions = CalculatePgoIoMemorySizeStrs(fused_schedule_result, GetPgoIoIndex(input, index), true,
301+ asc_out_node->outputs[0]);
302+ AppendPgoTensorMalloc(ss, "input" + std::to_string(index), size_expressions);
303+ index++;
304+ }
305+ index = 0;
306+ for (auto &output : fused_schedule_result.output_nodes) {
307+ if (af::ops::IsOps<af::ascir_op::Output>(output)) {
308+ const auto size_expressions =
309+ CalculatePgoIoMemorySizeStrs(fused_schedule_result, GetPgoIoIndex(output, index), false, output->inputs[0]);
310+ AppendPgoTensorMalloc(ss, "output" + std::to_string(index), size_expressions);
311+ index++;
312+ }
313+ }
314+ return ss.str();
315+}
316+ 
317+std::string TilingLib::PGOSearchTensorFreeDef(const ascir::FusedScheduledResult &fused_schedule_result) const {
318+ std::stringstream ss;
319+ int index = 0;
320+ for ([[maybe_unused]] auto &input : fused_schedule_result.input_nodes) {
321+ ss << " if (input" << index << " != nullptr) {" << std::endl;
322+ ss << " ret = aclrtFree(input" << index << ");" << std::endl;
323+ ss << " if (ret != ACL_SUCCESS) {" << std::endl;
324+ ss << " DLOGW(\"aclrtFree input" << index << " failed. ERROR: %d\", ret);" << std::endl;
325+ ss << " }" << std::endl;
326+ ss << " input" << index << " = nullptr;" << std::endl;
327+ ss << " }" << std::endl;
328+ index++;
329+ }
330+ index = 0;
331+ for (auto &output : fused_schedule_result.output_nodes) {
332+ if (af::ops::IsOps<af::ascir_op::Output>(output)) {
333+ ss << " if (output" << index << " != nullptr) {" << std::endl;
334+ ss << " ret = aclrtFree(output" << index << ");" << std::endl;
335+ ss << " if (ret != ACL_SUCCESS) {" << std::endl;
336+ ss << " DLOGW(\"aclrtFree output" << index << " failed. ERROR: %d\", ret);" << std::endl;
337+ ss << " }" << std::endl;
338+ ss << " output" << index << " = nullptr;" << std::endl;
339+ ss << " }" << std::endl;
340+ index++;
341+ }
342+ }
343+ return ss.str();
344+}
345+ 
27std::string TilingLib::ExternFunctionDeclare(const ascir::FusedScheduledResult &fused_schedule_result,346std::string TilingLib::ExternFunctionDeclare(const ascir::FusedScheduledResult &fused_schedule_result,
28 const std::string tiling) const {347 const std::string tiling) const {
29 (void)tiling;348 (void)tiling;
@@ -62,9 +381,11 @@ void TilingLib::AppendPgoConfigDef(std::stringstream &ss) const {
62 ss << " pgo_ub_threshold_list = {0.2, 0.1, 0, 0.05, 0.1};" << std::endl;381 ss << " pgo_ub_threshold_list = {0.2, 0.1, 0, 0.05, 0.1};" << std::endl;
63 ss << " pgo_corenum_threshold_list = {0.4, 0.4, 1, 1, 0.8};" << std::endl;382 ss << " pgo_corenum_threshold_list = {0.4, 0.4, 1, 1, 0.8};" << std::endl;
64 ss << " }" << std::endl;383 ss << " }" << std::endl;
65- ss << " ProfilingCallback single_callback;" << std::endl;384+ ss << " ProfilingCallback single_callback = nullptr;" << std::endl;
66- ss << " ProfilingBatchCallback batch_callback;" << std::endl;385+ ss << " ProfilingBatchCallback batch_callback = nullptr;" << std::endl;
67 ss << " PgoTensorArgs *tensor_args = nullptr;" << std::endl;386 ss << " PgoTensorArgs *tensor_args = nullptr;" << std::endl;
387+ ss << " std::vector<AutofuseTilingDataPerf> *measured_candidates = nullptr;" << std::endl;
388+ ss << " void *stream = nullptr;" << std::endl;
68 ss << " int32_t pgo_algorithm = 1; // 0 for pruning, 1 for core num" << std::endl;389 ss << " int32_t pgo_algorithm = 1; // 0 for pruning, 1 for core num" << std::endl;
69 ss << " bool need_change_solver_run = false;" << std::endl;390 ss << " bool need_change_solver_run = false;" << std::endl;
70 ss << " size_t pgo_threshold_index = 0;" << std::endl;391 ss << " size_t pgo_threshold_index = 0;" << std::endl;
@@ -220,116 +541,4 @@ uint32_t TilingLib::PGOSearchFuncGetOutputCount(const ascir::FusedScheduledResul
220 return count;541 return count;
221}542}
222 543 
223-std::string TilingLib::CalculateTensorMemorySizeStr(const ascir::TensorAttr &tensor) const {
224- static const std::unordered_map<ge::DataType, af::Expression> type_size_map = {
225- {ge::DT_FLOAT, af::Expression::Parse("4")}, // sizeof(float)
226- {ge::DT_FLOAT16, af::Expression::Parse("2")}, // fp16 is 2 bytes
227- {ge::DT_INT8, af::Expression::Parse("1")}, // sizeof(int8_t)
228- {ge::DT_INT16, af::Expression::Parse("2")}, // sizeof(int16_t)
229- {ge::DT_INT32, af::Expression::Parse("4")}, // sizeof(int32_t)
230- {ge::DT_INT64, af::Expression::Parse("8")}, // sizeof(int64_t)
231- {ge::DT_UINT8, af::Expression::Parse("1")}, // sizeof(uint8_t)
232- {ge::DT_UINT16, af::Expression::Parse("2")}, // sizeof(uint16_t)
233- {ge::DT_UINT32, af::Expression::Parse("4")}, // sizeof(uint32_t)
234- {ge::DT_UINT64, af::Expression::Parse("8")}, // sizeof(uint64_t)
235- {ge::DT_DOUBLE, af::Expression::Parse("8")}, // sizeof(double)
236- {ge::DT_BF16, af::Expression::Parse("2")}, // bf16 is 2 bytes
237- {ge::DT_BOOL, af::Expression::Parse("1")} // sizeof(bool)
238- };
239- const auto dtype = tensor.attr.dtype.operator ge::DataType();
240- auto it = type_size_map.find(dtype);
241- if (it == type_size_map.end()) {
242- GELOGE(ge::GRAPH_FAILED, "Unsupported data type: %d", static_cast<int32_t>(dtype));
243- return "0";
244- }
245- af::Expression type_size = it->second;
246- if (tensor.attr.repeats.empty() || tensor.attr.strides.empty()) {
247- GELOGE(ge::GRAPH_FAILED, "Empty repeats or strides for tensor when calculating memory size");
248- return "0";
249- }
250- 
251- // 跳过brc场景下的0 strides
252- size_t stride_index = 0UL;
253- for (; stride_index < tensor.attr.strides.size(); ++stride_index) {
254- if (tensor.attr.strides[stride_index] != af::ops::Zero) {
255- break;
256- }
257- GELOGD("Tensor stride %zu is zero, try to skip to next non-zero stride.", stride_index);
258- }
259- 
260- // 全为brc轴时，元素个数为1，其他情况下为repeats[index] * strides[index]
261- af::Expression element_size = af::ops::One;
262- if (stride_index < tensor.attr.repeats.size() && stride_index < tensor.attr.strides.size()) {
263- element_size = af::sym::Mul(tensor.attr.repeats[stride_index], tensor.attr.strides[stride_index]).Simplify();
264- }
265- af::Expression need_malloc_size = af::sym::Mul(element_size, type_size).Simplify();
266- GELOGD("Tensor element size: %s, need malloc size: %s", element_size.Str().get(), need_malloc_size.Str().get());
267- return std::string(need_malloc_size.Str().get());
268-}
269- 
270-std::string TilingLib::PGOSearchTensorMallocDef(const ascir::FusedScheduledResult &fused_schedule_result) const {
271- std::stringstream ss;
272- int index = 0;
273- for (auto &input : fused_schedule_result.input_nodes) {
274- if (input->GetOutNodesPtr().empty()) {
275- continue;
276- }
277- af::Node *out_node = input->GetOutNodesPtr()[0];
278- af::AscNode *asc_out_node = static_cast<af::AscNode *>(out_node);
279- ss << " size_t input" << index << "_size = " << CalculateTensorMemorySizeStr(asc_out_node->outputs[0]) << ";"
280- << std::endl;
281- ss << " ret = aclrtMalloc(&input" << index << ", input" << index << "_size, ACL_MEM_MALLOC_HUGE_FIRST);"
282- << std::endl;
283- ss << " if (ret != ACL_SUCCESS) {" << std::endl;
284- ss << " DLOGE(\"aclrtMalloc input" << index << " failed. ERROR: %d\", ret);" << std::endl;
285- ss << " return FAILED;" << std::endl;
286- ss << " }" << std::endl;
287- index++;
288- }
289- index = 0;
290- for (auto &output : fused_schedule_result.output_nodes) {
291- if (af::ops::IsOps<af::ascir_op::Output>(output)) {
292- ss << " size_t output" << index << "_size = " << CalculateTensorMemorySizeStr(output->inputs[0]) << ";"
293- << std::endl;
294- ss << " ret = aclrtMalloc(&output" << index << ", output" << index << "_size, ACL_MEM_MALLOC_HUGE_FIRST);"
295- << std::endl;
296- ss << " if (ret != ACL_SUCCESS) {" << std::endl;
297- ss << " DLOGE(\"aclrtMalloc output" << index << " failed. ERROR: %d\", ret);" << std::endl;
298- ss << " return FAILED;" << std::endl;
299- ss << " }" << std::endl;
300- index++;
301- }
302- }
303- return ss.str();
304-}
305- 
306-std::string TilingLib::PGOSearchTensorFreeDef(const ascir::FusedScheduledResult &fused_schedule_result) const {
307- std::stringstream ss;
308- int index = 0;
309- for ([[maybe_unused]] auto &input : fused_schedule_result.input_nodes) {
310- ss << " if (input" << index << " != nullptr) {" << std::endl;
311- ss << " ret = aclrtFree(input" << index << ");" << std::endl;
312- ss << " if (ret != ACL_SUCCESS) {" << std::endl;
313- ss << " DLOGW(\"aclrtFree input" << index << " failed. ERROR: %d\", ret);" << std::endl;
314- ss << " }" << std::endl;
315- ss << " input" << index << " = nullptr;" << std::endl;
316- ss << " }" << std::endl;
317- index++;
318- }
319- index = 0;
320- for (auto &output : fused_schedule_result.output_nodes) {
321- if (af::ops::IsOps<af::ascir_op::Output>(output)) {
322- ss << " if (output" << index << " != nullptr) {" << std::endl;
323- ss << " ret = aclrtFree(output" << index << ");" << std::endl;
324- ss << " if (ret != ACL_SUCCESS) {" << std::endl;
325- ss << " DLOGW(\"aclrtFree output" << index << " failed. ERROR: %d\", ret);" << std::endl;
326- ss << " }" << std::endl;
327- ss << " output" << index << " = nullptr;" << std::endl;
328- ss << " }" << std::endl;
329- index++;
330- }
331- }
332- return ss.str();
333-}
334- 
335} // namespace codegen544} // namespace codegen
Mautofuse/codegen/codegen_tiling_pgo_runtime.cpp+29-8
@@ -11,21 +11,42 @@
11#include "codegen_tiling.h"11#include "codegen_tiling.h"
12 12 
13namespace codegen {13namespace codegen {
14+void TilingLib::GenSharedPgoRuntimeLaunch(const ascir::FusedScheduledResult &fused_schedule_result,
15+ const std::string &pgo_dir, std::stringstream &ss, bool direct_link) const {
16+ GenPgoToolFunction(fused_schedule_result, pgo_dir, ss, direct_link);
17+ GenPgoWrapper(fused_schedule_result, ss, direct_link);
18+}
19+ 
20+void TilingLib::GenSharedPgoRuntimeProfiling(const ascir::FusedScheduledResult &fused_schedule_result,
21+ std::stringstream &ss, bool direct_link) const {
22+ GenPgoMsptiProfiling(ss, direct_link);
23+ GenPgoBatchProcess(ss, direct_link);
24+ GenPgoGetProfilingBatch(fused_schedule_result, ss, direct_link);
25+ GenPgoGetProfiling(fused_schedule_result, ss, direct_link);
26+}
14 27 
15std::string TilingLib::GenerateForPgo(const ascir::FusedScheduledResult &fused_schedule_result,28std::string TilingLib::GenerateForPgo(const ascir::FusedScheduledResult &fused_schedule_result,
16 const std::string &pgo_dir) const {29 const std::string &pgo_dir) const {
17- // 生成PGO的头文件和函数定义
18 std::stringstream ss;30 std::stringstream ss;
19- GenPgoHeaders(ss);31+ GenPgoHeaders(ss, false);
20- // 生成PGO需要的工具函数32+ GenSharedPgoRuntimeLaunch(fused_schedule_result, pgo_dir, ss, false);
21- GenPgoToolFunction(fused_schedule_result, pgo_dir, ss);33+ GenSharedPgoRuntimeProfiling(fused_schedule_result, ss, false);
22- // 生成PGO需要的wrapper函数
23- GenPgoWrapper(fused_schedule_result, ss);
24- // 生成PGO需要的求解代码
25 GenPgoProfiling(fused_schedule_result, ss);34 GenPgoProfiling(fused_schedule_result, ss);
26- // 生成PGO的main函数
27 GenPgoMain(fused_schedule_result, ss);35 GenPgoMain(fused_schedule_result, ss);
28 return ss.str();36 return ss.str();
29}37}
30 38 
39+std::string TilingLib::GenInductorPgoRunner(const ascir::FusedScheduledResult &fused_schedule_result) const {
40+ std::stringstream ss;
41+ GenPgoHeaders(ss, true);
42+ GenSharedPgoRuntimeLaunch(fused_schedule_result, "", ss, true);
43+ GenInductorPgoResultTypes(ss);
44+ GenInductorPgoHostLoader(ss);
45+ GenSharedPgoRuntimeProfiling(fused_schedule_result, ss, true);
46+ GenInductorPgoResultProtocol(ss);
47+ GenInductorPgoRuntime(fused_schedule_result, ss);
48+ GenInductorPgoMain(ss);
49+ return ss.str();
50+}
51+ 
31} // namespace codegen52} // namespace codegen
Mautofuse/codegen/codegen_tiling_pgo_search.cpp+54-21
@@ -19,10 +19,37 @@ namespace codegen {
19using namespace ascgen_utils;19using namespace ascgen_utils;
20using namespace ascir;20using namespace ascir;
21 21 
22+namespace {
23+std::string GenPgoMeasuredSearchModel() {
24+ return R"(
25+inline std::string PgoMeasuredCandidateKey(const AutofuseTilingDataPerf &candidate) {
26+ const char *ptr = reinterpret_cast<const char *>(&candidate.tiling_data);
27+ return std::string(ptr, ptr + sizeof(AutofuseTilingData));
28+}
29+ 
30+inline std::vector<AutofuseTilingDataPerf> NormalizePgoMeasuredCandidates(
31+ std::vector<AutofuseTilingDataPerf> raw_candidates) {
32+ std::vector<AutofuseTilingDataPerf> candidates;
33+ candidates.reserve(raw_candidates.size());
34+ std::unordered_set<std::string> seen;
35+ for (auto &candidate : raw_candidates) {
36+ const std::string key = PgoMeasuredCandidateKey(candidate);
37+ if (!seen.insert(key).second) {
38+ continue;
39+ }
40+ candidates.push_back(std::move(candidate));
41+ }
42+ return candidates;
43+}
44+)";
45+}
46+} // namespace
47+ 
22std::string TilingLib::GenPgoTilingFunc(const ascir::FusedScheduledResult &fused_schedule_result,48std::string TilingLib::GenPgoTilingFunc(const ascir::FusedScheduledResult &fused_schedule_result,
23 const std::string &tiling, codegen::PgoShapeStringStream &pgo_shape_dim,49 const std::string &tiling, codegen::PgoShapeStringStream &pgo_shape_dim,
24 bool is_inductor_scene, const std::string &core_num) const {50 bool is_inductor_scene, const std::string &core_num) const {
25 std::stringstream ss;51 std::stringstream ss;
52+ ss << GenPgoMeasuredSearchModel();
26 // 生成 AutofuseTilingWithConfig 函数53 // 生成 AutofuseTilingWithConfig 函数
27 ss << GenPgoAutofuseTiling(fused_schedule_result, pgo_shape_dim, tiling, is_inductor_scene);54 ss << GenPgoAutofuseTiling(fused_schedule_result, pgo_shape_dim, tiling, is_inductor_scene);
28 // 生成 PgoSaveTilingKey 函数55 // 生成 PgoSaveTilingKey 函数
@@ -97,22 +124,21 @@ std::string TilingLib::GenProfilingAllTilingData(std::string tiling_data_list_na
97 std::stringstream ss;124 std::stringstream ss;
98 ss << " double out_cost = DBL_MAX;" << std::endl;125 ss << " double out_cost = DBL_MAX;" << std::endl;
99 ss << " *workspaceSize = 0;" << std::endl;126 ss << " *workspaceSize = 0;" << std::endl;
100- ss << " std::unordered_set<std::string> solver_filter;" << std::endl;127+ ss << " std::vector<AutofuseTilingDataPerf> raw_search_candidates;" << std::endl;
101 ss << " for (const auto &tiling_data_item : " << tiling_data_list_name << ") {" << std::endl;128 ss << " for (const auto &tiling_data_item : " << tiling_data_list_name << ") {" << std::endl;
102- ss << " const char *ptr = reinterpret_cast<const char*>(&tiling_data_item);" << std::endl;
103- ss << " std::string key(ptr, ptr + sizeof(AutofuseTilingData));" << std::endl;
104- ss << " if (!solver_filter.insert(key).second) {" << std::endl;
105- ss << " continue;" << std::endl;
106- ss << " }" << std::endl;
107 ss << " *workspaceSize = std::max(GetWorkspaceSize(tiling_data_item), *workspaceSize);" << std::endl;129 ss << " *workspaceSize = std::max(GetWorkspaceSize(tiling_data_item), *workspaceSize);" << std::endl;
108 ss << " AutofuseTilingDataPerf tiling_data_perf;" << std::endl;130 ss << " AutofuseTilingDataPerf tiling_data_perf;" << std::endl;
109 ss << " tiling_data_perf.tiling_data = tiling_data_item;" << std::endl;131 ss << " tiling_data_perf.tiling_data = tiling_data_item;" << std::endl;
110 ss << " tiling_data_perf.best_perf = DBL_MAX;" << std::endl;132 ss << " tiling_data_perf.best_perf = DBL_MAX;" << std::endl;
111- ss << " " << tiling_data_perf_list_name << ".push_back(tiling_data_perf);" << std::endl;133+ ss << " raw_search_candidates.push_back(tiling_data_perf);" << std::endl;
112 ss << " }" << std::endl;134 ss << " }" << std::endl;
113 if (!is_inductor_scene) {135 if (!is_inductor_scene) {
114 ss << " *workspaceSize += 16 * 1024 * 1024;" << std::endl;136 ss << " *workspaceSize += 16 * 1024 * 1024;" << std::endl;
115 }137 }
138+ ss << " auto normalized_search_candidates = NormalizePgoMeasuredCandidates(std::move(raw_search_candidates));"
139+ << std::endl;
140+ ss << " " << tiling_data_perf_list_name << ".insert(" << tiling_data_perf_list_name
141+ << ".end(), normalized_search_candidates.begin(), normalized_search_candidates.end());" << std::endl;
116 ss << " PgoConfig::Instance().batch_callback(" << PGOSearchFuncInputOutputCall(fused_schedule_result)142 ss << " PgoConfig::Instance().batch_callback(" << PGOSearchFuncInputOutputCall(fused_schedule_result)
117 << "stream, *workspaceSize, &" << tiling_data_perf_list_name << ");" << std::endl;143 << "stream, *workspaceSize, &" << tiling_data_perf_list_name << ");" << std::endl;
118 return ss.str();144 return ss.str();
@@ -131,6 +157,8 @@ std::string TilingLib::GenPgoTilingSearchByCoreNum(const ascir::FusedScheduledRe
131 ss << "void *stream=nullptr, ProfilingCallback prof_callback=nullptr, ProfilingBatchCallback "157 ss << "void *stream=nullptr, ProfilingCallback prof_callback=nullptr, ProfilingBatchCallback "
132 "prof_batch_callback=nullptr) {"158 "prof_batch_callback=nullptr) {"
133 << std::endl;159 << std::endl;
160+ ss << " (void)prof_callback;" << std::endl;
161+ ss << " (void)prof_batch_callback;" << std::endl;
134 ss << " const ResLimit *limit = (res_limit == nullptr) ? &g_no_limit_res : res_limit;" << std::endl;162 ss << " const ResLimit *limit = (res_limit == nullptr) ? &g_no_limit_res : res_limit;" << std::endl;
135 ss << pgo_shape_dim.tiling_set_shape_dim.str();163 ss << pgo_shape_dim.tiling_set_shape_dim.str();
136 ss << " double best_perf = DBL_MAX;" << std::endl;164 ss << " double best_perf = DBL_MAX;" << std::endl;
@@ -230,6 +258,22 @@ std::string TilingLib::GenGetAutoFuseTilingInput(bool is_inductor_scene) const {
230 return ss.str();258 return ss.str();
231}259}
232 260 
261+void TilingLib::GenPgoTilingKeySearch(const ascir::FusedScheduledResult &fused_schedule_result,
262+ std::stringstream &ss) const {
263+ if (ascgen_utils::IsSingleGroup(fused_schedule_result)) {
264+ ss << " // 不使用，仅保持接口一致" << std::endl;
265+ ss << " std::unordered_map<int64_t, uint64_t> workspace_map;" << std::endl;
266+ ss << " if (!optiling::PGOSearchTilingKey(tiling_data_list, *tiling, -1, tiling, "
267+ << PGOSearchFuncInputOutputCall(fused_schedule_result) << "stream, *workspaceSize, best_perf, workspace_map)) {"
268+ << std::endl;
269+ } else {
270+ ss << " if (!optiling::PGOSearchTilingKey(tiling_data_list, *tiling, -1, tiling, "
271+ << PGOSearchFuncInputOutputCall(fused_schedule_result) << "stream, *workspaceSize, best_perf)) {" << std::endl;
272+ }
273+ ss << " return -1;" << std::endl;
274+ ss << " }" << std::endl;
275+}
276+ 
233std::string TilingLib::GenPgoTilingSearchPGO(const ascir::FusedScheduledResult &fused_schedule_result,277std::string TilingLib::GenPgoTilingSearchPGO(const ascir::FusedScheduledResult &fused_schedule_result,
234 codegen::PgoShapeStringStream &pgo_shape_dim, const std::string &tiling,278 codegen::PgoShapeStringStream &pgo_shape_dim, const std::string &tiling,
235 bool is_inductor_scene, const std::string &core_num) const {279 bool is_inductor_scene, const std::string &core_num) const {
@@ -241,6 +285,8 @@ std::string TilingLib::GenPgoTilingSearchPGO(const ascir::FusedScheduledResult &
241 << "void *stream=nullptr, ProfilingCallback prof_callback=nullptr, ProfilingBatchCallback "285 << "void *stream=nullptr, ProfilingCallback prof_callback=nullptr, ProfilingBatchCallback "
242 << "prof_batch_callback=nullptr) {" << std::endl;286 << "prof_batch_callback=nullptr) {" << std::endl;
243 287 
288+ ss << " (void)prof_callback;" << std::endl;
289+ ss << " (void)prof_batch_callback;" << std::endl;
244 ss << " const ResLimit *limit = (res_limit == nullptr) ? &g_no_limit_res : res_limit;" << std::endl;290 ss << " const ResLimit *limit = (res_limit == nullptr) ? &g_no_limit_res : res_limit;" << std::endl;
245 ss << " std::vector<AutofuseTilingDataPerf> tiling_data_list;" << std::endl;291 ss << " std::vector<AutofuseTilingDataPerf> tiling_data_list;" << std::endl;
246 ss << pgo_shape_dim.tiling_set_shape_dim.str();292 ss << pgo_shape_dim.tiling_set_shape_dim.str();
@@ -260,18 +306,7 @@ std::string TilingLib::GenPgoTilingSearchPGO(const ascir::FusedScheduledResult &
260 ss << " tiling_data_list.push_back(tiling_perf);" << std::endl;306 ss << " tiling_data_list.push_back(tiling_perf);" << std::endl;
261 ss << " OP_LOGD(OP_NAME, \"axesreorder solution base perf is %lf\", best_perf);" << std::endl;307 ss << " OP_LOGD(OP_NAME, \"axesreorder solution base perf is %lf\", best_perf);" << std::endl;
262 ss << " tiling->set_block_dim(max_block_dim);" << std::endl;308 ss << " tiling->set_block_dim(max_block_dim);" << std::endl;
263- if (ascgen_utils::IsSingleGroup(fused_schedule_result)) {309+ GenPgoTilingKeySearch(fused_schedule_result, ss);
264- ss << " // 不使用，仅保持接口一致" << std::endl;
265- ss << " std::unordered_map<int64_t, uint64_t> workspace_map;" << std::endl;
266- ss << " if (!optiling::PGOSearchTilingKey(tiling_data_list, *tiling, -1, tiling, "
267- << PGOSearchFuncInputOutputCall(fused_schedule_result) << "stream, *workspaceSize, best_perf, workspace_map)) {"
268- << std::endl;
269- } else {
270- ss << " if (!optiling::PGOSearchTilingKey(tiling_data_list, *tiling, -1, tiling, "
271- << PGOSearchFuncInputOutputCall(fused_schedule_result) << "stream, *workspaceSize, best_perf)) {" << std::endl;
272- }
273- ss << " return -1;" << std::endl;
274- ss << " }" << std::endl;
275 ss << " if (optiling::IsEqual(best_perf, DBL_MAX)) {" << std::endl;310 ss << " if (optiling::IsEqual(best_perf, DBL_MAX)) {" << std::endl;
276 ss << " OP_LOGE(OP_NAME, \"pgo solution get perf failed %lf\", best_perf);" << std::endl;311 ss << " OP_LOGE(OP_NAME, \"pgo solution get perf failed %lf\", best_perf);" << std::endl;
277 ss << " return -1;" << std::endl;312 ss << " return -1;" << std::endl;
@@ -308,7 +343,6 @@ std::string TilingLib::GenGetResLimitStru(void) const {
308bool TilingLib::IsMixKernelTaskType(const ascir::FusedScheduledResult &fused_schedule_result) const {343bool TilingLib::IsMixKernelTaskType(const ascir::FusedScheduledResult &fused_schedule_result) const {
309 return fused_schedule_result.workspace_nodes.size() != 0;344 return fused_schedule_result.workspace_nodes.size() != 0;
310}345}
311- 
312std::string TilingLib::GenPGOGetTilingKey(const std::string tiling) const {346std::string TilingLib::GenPGOGetTilingKey(const std::string tiling) const {
313 std::stringstream ss;347 std::stringstream ss;
314 ss << "bool PGOGetTilingKey(const char *config_file_path, " << tiling << " &tiling_data) {" << std::endl;348 ss << "bool PGOGetTilingKey(const char *config_file_path, " << tiling << " &tiling_data) {" << std::endl;
@@ -416,5 +450,4 @@ std::string TilingLib::GenSavePGOConfigTilingDataFunc() const {
416 450 
417 return ss.str();451 return ss.str();
418}452}
419- 
420} // namespace codegen453} // namespace codegen
Mautofuse/compiler/python/ascendc_compile.py+241-21
@@ -10,6 +10,8 @@
10# See LICENSE in the root of the software repository for the full text of the License.10# See LICENSE in the root of the software repository for the full text of the License.
11# -----------------------------------------------------------------------------------------------------------11# -----------------------------------------------------------------------------------------------------------
12import ctypes12import ctypes
13+import hashlib
14+import json
13import os15import os
14import re16import re
15import sys17import sys
@@ -17,8 +19,11 @@ import shutil
17import argparse19import argparse
18import subprocess20import subprocess
19import platform21import platform
22+import tempfile
23+import uuid
20from concurrent.futures import ThreadPoolExecutor, as_completed24from concurrent.futures import ThreadPoolExecutor, as_completed
21import time25import time
26+from dataclasses import dataclass
22from functools import wraps27from functools import wraps
23from typing import List28from typing import List
24from asc_op_compile_base.common.platform.platform_info import get_soc_spec29from asc_op_compile_base.common.platform.platform_info import get_soc_spec
@@ -30,6 +35,9 @@ HOST_LINK_LIBRARIES = ["tiling_api", "platform", "graph_base", "register"]
30CV_HOST_LINK_LIBRARIES = HOST_LINK_LIBRARIES + ["nnopbase"]35CV_HOST_LINK_LIBRARIES = HOST_LINK_LIBRARIES + ["nnopbase"]
31INDUCTOR_COMPILE_TRACE_LABEL = "InductorCompile"36INDUCTOR_COMPILE_TRACE_LABEL = "InductorCompile"
32HOST_COMPILE_MAX_WORKERS = 3237HOST_COMPILE_MAX_WORKERS = 32
38+PGO_BUNDLE_SCHEMA_VERSION = 1
39+PGO_RESULT_PROTOCOL_VERSION = 1
40+PGO_KERNEL_FORMAT = "aicore_binary_elf_v1"
33if not os.path.exists(ASCEND_PATH):41if not os.path.exists(ASCEND_PATH):
34 ASCEND_PATH = os.getenv("ASCEND_HOME_PATH", ASCEND_PATH)42 ASCEND_PATH = os.getenv("ASCEND_HOME_PATH", ASCEND_PATH)
35 43 
@@ -38,17 +46,14 @@ class CompileError(Exception):
38 """Compile failed exception."""46 """Compile failed exception."""
39 47 
40 48 
41-def parse_env_flags(env_name):49+@dataclass(frozen=True)
42- result = {}50+class PgoBundle:
43- flags = os.getenv(env_name)51+ tiling_file: str
44- if not flags:52+ runner_file: str
45- return result53+ kernel_file: str
46- params = flags.split(";")54+ output_file: str
47- for param in params:55+ generation: str
48- if "=" in param:56+ ld_preload: str = ""
49- key_part, value_part = param.split("=", 1)
50- result[key_part.lstrip("-")] = value_part
51- return result
52 57 
53 58 
54def record_inductor_compile_duration(stage, step, graph_name, start, duration):59def record_inductor_compile_duration(stage, step, graph_name, start, duration):
@@ -127,6 +132,180 @@ def link_shared(target_file, obj_files, link_libraries=None):
127 return target_file132 return target_file
128 133 
129 134 
135+def link_pgo_executable(target_file, obj_files, mspti_link_flags):
136+ link_command = [f"{ASCEND_PATH}/tools/bisheng_compiler/bin/bisheng", *obj_files]
137+ link_command.extend(["-fPIC", "-o", target_file])
138+ link_command.extend(["-L", f"{ASCEND_PATH}/lib64"])
139+ link_command.extend(["-L", f"{ASCEND_PATH}/{machine}-linux/lib64"])
140+ link_command.extend([f"-l{link_library}" for link_library in HOST_LINK_LIBRARIES])
141+ link_command.extend(
142+ ["-lascendcl", "-lruntime", "-lunified_dlog", "-lascendalog", "-lc_sec", "-lm"]
143+ )
144+ link_command.extend(
145+ f"-Wl,-rpath,{option[2:]}"
146+ for option in mspti_link_flags
147+ if option.startswith("-L") and len(option) > 2
148+ )
149+ link_command.extend(mspti_link_flags)
150+ link_command.extend(["-ldl", "-lpthread"])
151+ run_compile_command(link_command, "LinkPgoExecutable")
152+ return target_file
153+ 
154+ 
155+def extract_aicore_binary(device_obj_file, output_file):
156+ objcopy = shutil.which("llvm-objcopy")
157+ if objcopy is None:
158+ objcopy = os.path.join(
159+ ASCEND_PATH, "tools", "bisheng_compiler", "bin", "llvm-objcopy"
160+ )
161+ if not os.path.isfile(objcopy):
162+ raise CompileError("llvm-objcopy is required for Inductor PGO device binary")
163+ run_compile_command(
164+ [objcopy, "--dump-section", f".aicore_binary={output_file}", device_obj_file],
165+ "ExtractPgoDeviceBinary",
166+ )
167+ if not os.path.isfile(output_file) or os.path.getsize(output_file) == 0:
168+ raise CompileError("extracted Inductor PGO device binary is empty")
169+ return output_file
170+ 
171+ 
172+def build_pgo_sidecars(args, temp_dir):
173+ mspti_dir, preload_files, link_flags = args.pgo_mspti_config
174+ args.pgo_mspti_dir = mspti_dir
175+ args.pgo_ld_preload = ":".join(preload_files)
176+ runner_obj = compile_host_obj_file(args, temp_dir, args.pgo_runner_file)
177+ runner_file = os.path.join(temp_dir, "pgo_runner")
178+ link_pgo_executable(runner_file, [runner_obj], link_flags)
179+ args.device_files = args.pgo_device_file
180+ device_obj = compile_device_obj(args, temp_dir)
181+ kernel_file = os.path.join(temp_dir, f"pgo_kernel.{PGO_KERNEL_FORMAT}")
182+ extract_aicore_binary(device_obj, kernel_file)
183+ return runner_file, kernel_file
184+ 
185+ 
186+def get_pgo_sidecar_paths(output_file, generation):
187+ generation_dir = f"{os.path.realpath(output_file)}.pgo.{generation}"
188+ output_name = os.path.basename(output_file)
189+ return {
190+ "generation_dir": generation_dir,
191+ "tiling_so": os.path.join(generation_dir, output_name),
192+ "runner": os.path.join(generation_dir, f"{output_name}.pgo_runner"),
193+ "kernel": os.path.join(
194+ generation_dir, f"{output_name}.pgo_kernel.{PGO_KERNEL_FORMAT}"
195+ ),
196+ "manifest": os.path.join(generation_dir, "manifest.json"),
197+ }
198+ 
199+ 
200+def file_sha256(path):
201+ digest = hashlib.sha256()
202+ with open(path, "rb") as file:
203+ for chunk in iter(lambda: file.read(1024 * 1024), b""):
204+ digest.update(chunk)
205+ return digest.hexdigest()
206+ 
207+ 
208+def build_pgo_manifest(bundle):
209+ return {
210+ "bundle_schema_version": PGO_BUNDLE_SCHEMA_VERSION,
211+ "generation": bundle.generation,
212+ "result_protocol_version": PGO_RESULT_PROTOCOL_VERSION,
213+ "ld_preload": bundle.ld_preload,
214+ "artifacts": {
215+ "tiling_so": {
216+ "file": os.path.basename(bundle.output_file),
217+ "sha256": file_sha256(bundle.tiling_file),
218+ },
219+ "runner": {
220+ "file": os.path.basename(bundle.runner_file),
221+ "sha256": file_sha256(bundle.runner_file),
222+ },
223+ "kernel": {
224+ "file": os.path.basename(bundle.kernel_file),
225+ "sha256": file_sha256(bundle.kernel_file),
226+ },
227+ },
228+ }
229+ 
230+ 
231+def write_pgo_manifest(path, manifest):
232+ with open(path, "w", encoding="utf-8") as file:
233+ json.dump(manifest, file, sort_keys=True)
234+ file.flush()
235+ os.fsync(file.fileno())
236+ 
237+ 
238+def cleanup_stale_pgo_generations(output_file, current_generation_dir):
239+ output_file = os.path.realpath(output_file)
240+ output_dir = os.path.dirname(output_file)
241+ generation_prefix = os.path.basename(output_file) + ".pgo."
242+ previous_generations = []
243+ try:
244+ for entry in os.scandir(output_dir):
245+ if not entry.name.startswith(generation_prefix) or not entry.is_dir(
246+ follow_symlinks=False
247+ ):
248+ continue
249+ if os.path.realpath(entry.path) == os.path.realpath(current_generation_dir):
250+ continue
251+ previous_generations.append(
252+ (entry.stat(follow_symlinks=False).st_mtime_ns, entry.name, entry.path)
253+ )
254+ except OSError:
255+ return
256+ previous_generations.sort(reverse=True)
257+ for _, _, generation_dir in previous_generations[1:]:
258+ shutil.rmtree(generation_dir, ignore_errors=True)
259+ 
260+ 
261+def publish_pgo_bundle(bundle):
262+ output_file = os.path.realpath(bundle.output_file)
263+ output_dir = os.path.dirname(output_file)
264+ os.makedirs(output_dir, exist_ok=True)
265+ paths = get_pgo_sidecar_paths(output_file, bundle.generation)
266+ staging_dir = tempfile.mkdtemp(
267+ prefix=f".{os.path.basename(output_file)}.pgo.", dir=output_dir
268+ )
269+ staged_tiling = os.path.join(
270+ output_dir, f".{os.path.basename(output_file)}.{bundle.generation}.tmp"
271+ )
272+ generation_published = False
273+ try:
274+ staged_generation_tiling = os.path.join(
275+ staging_dir, os.path.basename(paths["tiling_so"])
276+ )
277+ staged_runner = os.path.join(staging_dir, os.path.basename(paths["runner"]))
278+ staged_kernel = os.path.join(staging_dir, os.path.basename(paths["kernel"]))
279+ shutil.copy2(bundle.tiling_file, staged_generation_tiling)
280+ shutil.copy2(bundle.runner_file, staged_runner)
281+ shutil.copy2(bundle.kernel_file, staged_kernel)
282+ shutil.copy2(staged_generation_tiling, staged_tiling)
283+ manifest = build_pgo_manifest(
284+ PgoBundle(
285+ staged_generation_tiling,
286+ staged_runner,
287+ staged_kernel,
288+ output_file,
289+ bundle.generation,
290+ bundle.ld_preload,
291+ )
292+ )
293+ write_pgo_manifest(os.path.join(staging_dir, "manifest.json"), manifest)
294+ os.replace(staging_dir, paths["generation_dir"])
295+ generation_published = True
296+ os.replace(staged_tiling, output_file)
297+ cleanup_stale_pgo_generations(output_file, paths["generation_dir"])
298+ return paths
299+ except OSError as ex:
300+ if generation_published:
301+ shutil.rmtree(paths["generation_dir"], ignore_errors=True)
302+ raise CompileError(f"publish Inductor PGO bundle failed: {ex}") from ex
303+ finally:
304+ shutil.rmtree(staging_dir, ignore_errors=True)
305+ if os.path.exists(staged_tiling):
306+ os.remove(staged_tiling)
307+ 
308+ 
130def iter_compile_source_files(args: argparse.Namespace):309def iter_compile_source_files(args: argparse.Namespace):
131 for source_files in (310 for source_files in (
132 getattr(args, "host_files", None),311 getattr(args, "host_files", None),
@@ -201,7 +380,7 @@ def build_host_include_options(temp_dir):
201 380 
202def build_host_base_options(args: argparse.Namespace, temp_dir):381def build_host_base_options(args: argparse.Namespace, temp_dir):
203 soc_version = get_soc_type(args)382 soc_version = get_soc_type(args)
204- return [383+ options = [
205 f"{ASCEND_PATH}/tools/bisheng_compiler/bin/bisheng",384 f"{ASCEND_PATH}/tools/bisheng_compiler/bin/bisheng",
206 "-D",385 "-D",
207 "kernel_EXPORTS",386 "kernel_EXPORTS",
@@ -214,6 +393,13 @@ def build_host_base_options(args: argparse.Namespace, temp_dir):
214 "-Wfloat-equal",393 "-Wfloat-equal",
215 "-fvisibility=default",394 "-fvisibility=default",
216 ]395 ]
396+ mspti_dir = getattr(args, "pgo_mspti_dir", None)
397+ if mspti_dir:
398+ options.extend(["-I", os.path.join(mspti_dir, "include")])
399+ pgo_generation = getattr(args, "pgo_generation", None)
400+ if pgo_generation:
401+ options.extend(["-D", f'AUTOFUSE_PGO_GENERATION="{pgo_generation}"'])
402+ return options
217 403 
218 404 
219def build_host_output_options(source_file, obj_file):405def build_host_output_options(source_file, obj_file):
@@ -618,6 +804,8 @@ def link_host_target(args, temp_dir):
618 link_libraries = (804 link_libraries = (
619 CV_HOST_LINK_LIBRARIES if is_cv_fusion_compile(args) else HOST_LINK_LIBRARIES805 CV_HOST_LINK_LIBRARIES if is_cv_fusion_compile(args) else HOST_LINK_LIBRARIES
620 )806 )
807+ if getattr(args, "pgo_runner_file", None) is not None:
808+ link_libraries = link_libraries + ["ascendcl", "runtime"]
621 with InductorCompileDuration(args, "LinkHostSo"):809 with InductorCompileDuration(args, "LinkHostSo"):
622 link_shared(so_file, host_obj_paths, link_libraries=link_libraries)810 link_shared(so_file, host_obj_paths, link_libraries=link_libraries)
623 return so_file811 return so_file
@@ -661,21 +849,53 @@ def copy_so_to_output(so_file, args, src_directory):
661 os.chdir(src_directory)849 os.chdir(src_directory)
662 850 
663 851 
852+def build_host_output(args):
853+ should_build_sidecars = (
854+ getattr(args, "pgo_runner_file", None) is not None
855+ and getattr(args, "pgo_mspti_config", None) is not None
856+ )
857+ if should_build_sidecars:
858+ args.pgo_generation = uuid.uuid4().hex
859+ so_file = link_host_target(args, args.temp_dir)
860+ if not should_build_sidecars:
861+ return so_file
862+ try:
863+ runner_file, kernel_file = build_pgo_sidecars(args, args.temp_dir)
864+ publish_pgo_bundle(
865+ PgoBundle(
866+ so_file,
867+ runner_file,
868+ kernel_file,
869+ args.output_file,
870+ args.pgo_generation,
871+ getattr(args, "pgo_ld_preload", ""),
872+ )
873+ )
874+ return None
875+ except CompileError as ex:
876+ print(f"[PGO] Inductor PGO sidecar build failed, skip PGO: {ex}")
877+ return so_file
878+ 
879+ 
664def main(args):880def main(args):
665 print("compile args:", args)881 print("compile args:", args)
666 src_directory = os.getcwd()882 src_directory = os.getcwd()
667 os.chdir(args.temp_dir)883 os.chdir(args.temp_dir)
668 print("change work dir:", os.getcwd())884 print("change work dir:", os.getcwd())
885+ try:
886+ if args.stage == "host":
887+ so_file = build_host_output(args)
888+ if so_file is None:
889+ return
890+ elif args.stage == "device":
891+ so_file = link_kernel_target(args, None, args.temp_dir)
892+ else: # all
893+ host_obj_paths = compile_host_objs(args, args.temp_dir)
894+ so_file = link_kernel_target(args, host_obj_paths, args.temp_dir)
669 895 
670- if args.stage == "host":896+ copy_so_to_output(so_file, args, src_directory)
671- so_file = link_host_target(args, args.temp_dir)897+ finally:
672- elif args.stage == "device":898+ os.chdir(src_directory)
673- so_file = link_kernel_target(args, None, args.temp_dir)
674- else: # all
675- host_obj_paths = compile_host_objs(args, args.temp_dir)
676- so_file = link_kernel_target(args, host_obj_paths, args.temp_dir)
677- 
678- copy_so_to_output(so_file, args, src_directory)
679 899 
680 900 
681def main_with_except(argv: List[str]):901def main_with_except(argv: List[str]):