已合并
fix: 统一管理 Python 解释器生命周期 #3631
duhua创建于 6月22日
fix: 统一管理 Python 解释器生命周期 #3631
已合并
共 31 个文件变更+1016-88
| @@ -42,6 +42,7 @@ | |||
| 42 | 42 | ||
| 43 | 43 | ||
| 44 | 44 | ||
| 45 | + | ||
| 45 | 46 | ||
| 46 | 47 | ||
| 47 | 48 | ||
| @@ -811,6 +812,18 @@ class GFlagUtils { | |||
| 811 | return ge::SUCCESS; | 812 | return ge::SUCCESS; |
| 812 | } | 813 | } |
| 813 | 814 | ||
| 815 | + static Status CheckPrecisionRelatedFlags() { | ||
| 816 | + GE_ASSERT_SUCCESS(CheckIsWeightClipParamValid(FLAGS_is_weight_clip), "[Check][is_weight_clip]failed!"); | ||
| 817 | + GE_ASSERT_SUCCESS(CheckPrecisionModeParamValid(FLAGS_precision_mode), "[Check][PrecisionMode]failed!"); | ||
| 818 | + GE_ASSERT_SUCCESS(CheckPrecisionModeV2ParamValid(FLAGS_precision_mode_v2), "[Check][PrecisionModeV2]failed!"); | ||
| 819 | + GE_ASSERT_SUCCESS(CheckPrecisionModeV2Conflict(FLAGS_precision_mode, FLAGS_precision_mode_v2), | ||
| 820 | + "[Check][PrecisionModeV2Conflict]failed!"); | ||
| 821 | + if (CheckModifyMixlistParamValid(FLAGS_precision_mode, FLAGS_precision_mode_v2, FLAGS_modify_mixlist) != SUCCESS) { | ||
| 822 | + return FAILED; | ||
| 823 | + } | ||
| 824 | + return SUCCESS; | ||
| 825 | + } | ||
| 826 | + | ||
| 814 | static Status CheckFlags() { | 827 | static Status CheckFlags() { |
| 815 | const bool is_mode_om = ((FLAGS_mode == static_cast<int32_t>(RunMode::GEN_OM_MODEL)) || | 828 | const bool is_mode_om = ((FLAGS_mode == static_cast<int32_t>(RunMode::GEN_OM_MODEL)) || |
| 816 | (FLAGS_mode == static_cast<int32_t>(RunMode::GEN_EXE_OM_FOR_NANO)) || | 829 | (FLAGS_mode == static_cast<int32_t>(RunMode::GEN_EXE_OM_FOR_NANO)) || |
| @@ -858,14 +871,7 @@ class GFlagUtils { | |||
| 858 | GELOGE(FAILED, "[Check][op_precision_mode] %s not found", FLAGS_op_precision_mode.c_str()); | 871 | GELOGE(FAILED, "[Check][op_precision_mode] %s not found", FLAGS_op_precision_mode.c_str()); |
| 859 | return FAILED; | 872 | return FAILED; |
| 860 | } | 873 | } |
| 861 | - GE_ASSERT_SUCCESS(CheckIsWeightClipParamValid(FLAGS_is_weight_clip), "[Check][is_weight_clip]failed!"); | 874 | + GE_ASSERT_SUCCESS(CheckPrecisionRelatedFlags()); |
| 862 | - GE_ASSERT_SUCCESS(CheckPrecisionModeParamValid(FLAGS_precision_mode), "[Check][PrecisionMode]failed!"); | ||
| 863 | - GE_ASSERT_SUCCESS(CheckPrecisionModeV2ParamValid(FLAGS_precision_mode_v2), "[Check][PrecisionModeV2]failed!"); | ||
| 864 | - GE_ASSERT_SUCCESS(CheckPrecisionModeV2Conflict(FLAGS_precision_mode, FLAGS_precision_mode_v2), | ||
| 865 | - "[Check][PrecisionModeV2Conflict]failed!"); | ||
| 866 | - if (CheckModifyMixlistParamValid(FLAGS_precision_mode, FLAGS_precision_mode_v2, FLAGS_modify_mixlist) != SUCCESS) { | ||
| 867 | - return FAILED; | ||
| 868 | - } | ||
| 869 | 875 | ||
| 870 | if (CheckAndTransferInputShapeToRange(FLAGS_input_shape, FLAGS_input_shape_range, FLAGS_dynamic_batch_size, | 876 | if (CheckAndTransferInputShapeToRange(FLAGS_input_shape, FLAGS_input_shape_range, FLAGS_dynamic_batch_size, |
| 871 | FLAGS_dynamic_image_size, FLAGS_dynamic_dims) != SUCCESS) { | 877 | FLAGS_dynamic_image_size, FLAGS_dynamic_dims) != SUCCESS) { |
| @@ -1417,6 +1423,7 @@ Status GenerateInfershapeJson() { | |||
| 1417 | DOMI_LOGE("GeGenerator initialize failed!"); | 1423 | DOMI_LOGE("GeGenerator initialize failed!"); |
| 1418 | return FAILED; | 1424 | return FAILED; |
| 1419 | } | 1425 | } |
| 1426 | + GE_MAKE_GUARD(release_python_runtime, []() { (void)GePythonRuntimeManager::Instance().ShutdownProcess(); }); | ||
| 1420 | 1427 | ||
| 1421 | Graph graph; | 1428 | Graph graph; |
| 1422 | std::map<std::string, std::string> atc_params; | 1429 | std::map<std::string, std::string> atc_params; |
| @@ -1631,7 +1638,15 @@ Status GenerateModel(std::map<std::string, std::string> &options, const std::str | |||
| 1631 | GeGenerator ge_generator; | 1638 | GeGenerator ge_generator; |
| 1632 | Status ret = SUCCESS; | 1639 | Status ret = SUCCESS; |
| 1633 | std::shared_ptr<GELib> instance_ptr = GELib::GetInstance(); | 1640 | std::shared_ptr<GELib> instance_ptr = GELib::GetInstance(); |
| 1641 | + bool release_python_runtime = false; | ||
| 1642 | + GE_DISMISSABLE_GUARD(release_python_runtime_guard, ([&release_python_runtime]() { | ||
| 1643 | + if (release_python_runtime) { | ||
| 1644 | + (void)GePythonRuntimeManager::Instance().ShutdownProcess(); | ||
| 1645 | + } | ||
| 1646 | + })); | ||
| 1634 | if (instance_ptr == nullptr || !instance_ptr->InitFlag()) { | 1647 | if (instance_ptr == nullptr || !instance_ptr->InitFlag()) { |
| 1648 | + GE_ASSERT_SUCCESS(GePythonRuntimeManager::Instance().EnsureReady()); | ||
| 1649 | + release_python_runtime = true; | ||
| 1635 | ret = GELib::Initialize(options); | 1650 | ret = GELib::Initialize(options); |
| 1636 | if (ret != SUCCESS) { | 1651 | if (ret != SUCCESS) { |
| 1637 | DOMI_LOGE("GE initialize failed!"); | 1652 | DOMI_LOGE("GE initialize failed!"); |
| @@ -1647,8 +1662,10 @@ Status GenerateModel(std::map<std::string, std::string> &options, const std::str | |||
| 1647 | const std::function<void()> callback = [&ge_generator]() { | 1662 | const std::function<void()> callback = [&ge_generator]() { |
| 1648 | (void)ge_generator.Finalize(); | 1663 | (void)ge_generator.Finalize(); |
| 1649 | (void)GELib::GetInstance()->Finalize(); | 1664 | (void)GELib::GetInstance()->Finalize(); |
| 1665 | + (void)GePythonRuntimeManager::Instance().ShutdownProcess(); | ||
| 1650 | }; | 1666 | }; |
| 1651 | GE_MAKE_GUARD(release, callback); | 1667 | GE_MAKE_GUARD(release, callback); |
| 1668 | + release_python_runtime = false; | ||
 | |||
| 1652 | GELOGD("Current input is single graph to generate model."); | 1669 | GELOGD("Current input is single graph to generate model."); |
| 1653 | return GenerateModelBySingleGraph(ge_generator, output, options); | 1670 | return GenerateModelBySingleGraph(ge_generator, output, options); |
| 1654 | } | 1671 | } |
| @@ -1697,7 +1714,7 @@ static void SetEnvForSingleOp(std::map<std::string, std::string> &options) { | |||
| 1697 | SetOptionNameMap(options); | 1714 | SetOptionNameMap(options); |
| 1698 | } | 1715 | } |
| 1699 | 1716 | ||
| 1700 | -Status GenerateSingleOp(const std::string &json_file_path) { | 1717 | +Status CheckSingleOpOptions() { |
| 1701 | if ((!GFlagUtils::IsRequiredParameterExists("--output", FLAGS_output)) || | 1718 | if ((!GFlagUtils::IsRequiredParameterExists("--output", FLAGS_output)) || |
| 1702 | (!GFlagUtils::IsRequiredParameterExists("--soc_version", FLAGS_soc_version))) { | 1719 | (!GFlagUtils::IsRequiredParameterExists("--soc_version", FLAGS_soc_version))) { |
| 1703 | return FAILED; | 1720 | return FAILED; |
| @@ -1720,15 +1737,34 @@ Status GenerateSingleOp(const std::string &json_file_path) { | |||
| 1720 | return FAILED; | 1737 | return FAILED; |
| 1721 | } | 1738 | } |
| 1722 | 1739 | ||
| 1723 | - GE_ASSERT_SUCCESS(CheckIsWeightClipParamValid(FLAGS_is_weight_clip), "[Check][is_weight_clip]failed!"); | 1740 | + GE_ASSERT_SUCCESS(GFlagUtils::CheckPrecisionRelatedFlags()); |
| 1724 | - GE_ASSERT_SUCCESS(CheckPrecisionModeParamValid(FLAGS_precision_mode), "[Check][PrecisionMode]failed!"); | ||
| 1725 | - GE_ASSERT_SUCCESS(CheckPrecisionModeV2ParamValid(FLAGS_precision_mode_v2), "[Check][PrecisionModeV2]failed!"); | ||
| 1726 | - GE_ASSERT_SUCCESS(CheckPrecisionModeV2Conflict(FLAGS_precision_mode, FLAGS_precision_mode_v2), | ||
| 1727 | - "[Check][PrecisionModeV2Conflict]failed!"); | ||
| 1728 | - if (CheckModifyMixlistParamValid(FLAGS_precision_mode, FLAGS_precision_mode_v2, FLAGS_modify_mixlist) != SUCCESS) { | ||
| 1729 | - return FAILED; | ||
| 1730 | - } | ||
| 1731 | GE_ASSERT_SUCCESS(CheckAllowHF32ParamValid(FLAGS_allow_hf32), "[Check][AllowHF32]failed!"); | 1741 | GE_ASSERT_SUCCESS(CheckAllowHF32ParamValid(FLAGS_allow_hf32), "[Check][AllowHF32]failed!"); |
| 1742 | + return SUCCESS; | ||
| 1743 | +} | ||
| 1744 | + | ||
| 1745 | +Status BuildSingleOpModels(GeGenerator &generator, std::vector<SingleOpBuildParam> &build_params) { | ||
| 1746 | + Status ret = SUCCESS; | ||
| 1747 | + int32_t index = 0; | ||
| 1748 | + for (auto ¶m : build_params) { | ||
| 1749 | + std::string output_path; | ||
| 1750 | + if (!FLAGS_output.empty()) { | ||
| 1751 | + output_path = FLAGS_output + "/"; | ||
| 1752 | + } | ||
| 1753 | + output_path += param.file_name; | ||
| 1754 | + ret = generator.BuildSingleOpModel(param.op_desc, param.inputs, param.outputs, output_path, param.compile_flag); | ||
| 1755 | + if (ret != SUCCESS) { | ||
| 1756 | + DOMI_LOGE("Compile op failed. ge ret = %u, op index = %d", ret, index); | ||
| 1757 | + ret = FAILED; | ||
| 1758 | + } else { | ||
| 1759 | + GELOGI("Compile op success. op index = %d, output = %s", index, output_path.c_str()); | ||
| 1760 | + } | ||
| 1761 | + index += 1; | ||
| 1762 | + } | ||
| 1763 | + return ret; | ||
| 1764 | +} | ||
| 1765 | + | ||
| 1766 | +Status GenerateSingleOp(const std::string &json_file_path) { | ||
| 1767 | + GE_ASSERT_SUCCESS(CheckSingleOpOptions()); | ||
| 1732 | GE_ASSERT_GRAPH_SUCCESS(OpLibRegistry::GetInstance().PreProcessForCustomOp()); | 1768 | GE_ASSERT_GRAPH_SUCCESS(OpLibRegistry::GetInstance().PreProcessForCustomOp()); |
| 1733 | std::map<std::string, std::string> options; | 1769 | std::map<std::string, std::string> options; |
| 1734 | // need to be changed when ge.ini plan is done | 1770 | // need to be changed when ge.ini plan is done |
| @@ -1736,6 +1772,8 @@ Status GenerateSingleOp(const std::string &json_file_path) { | |||
| 1736 | // print single op option map | 1772 | // print single op option map |
| 1737 | PrintOptionMap(options, "single op option"); | 1773 | PrintOptionMap(options, "single op option"); |
| 1738 | 1774 | ||
| 1775 | + GE_ASSERT_SUCCESS(GePythonRuntimeManager::Instance().EnsureReady()); | ||
| 1776 | + GE_MAKE_GUARD(release_python_runtime, []() { (void)GePythonRuntimeManager::Instance().ShutdownProcess(); }); | ||
| 1739 | auto ret = GELib::Initialize(options); | 1777 | auto ret = GELib::Initialize(options); |
| 1740 | if (ret != SUCCESS) { | 1778 | if (ret != SUCCESS) { |
| 1741 | DOMI_LOGE("GE initialize failed!"); | 1779 | DOMI_LOGE("GE initialize failed!"); |
| @@ -1758,23 +1796,7 @@ Status GenerateSingleOp(const std::string &json_file_path) { | |||
| 1758 | return FAILED; | 1796 | return FAILED; |
| 1759 | } | 1797 | } |
| 1760 | 1798 | ||
| 1761 | - int32_t index = 0; | 1799 | + ret = BuildSingleOpModels(generator, build_params); |
| 1762 | - for (auto ¶m : build_params) { | ||
| 1763 | - std::string output_path; | ||
| 1764 | - if (!FLAGS_output.empty()) { | ||
| 1765 | - output_path = FLAGS_output + "/"; | ||
| 1766 | - } | ||
| 1767 | - output_path += param.file_name; | ||
| 1768 | - ret = generator.BuildSingleOpModel(param.op_desc, param.inputs, param.outputs, output_path, param.compile_flag); | ||
| 1769 | - if (ret != SUCCESS) { | ||
| 1770 | - DOMI_LOGE("Compile op failed. ge ret = %u, op index = %d", ret, index); | ||
| 1771 | - ret = FAILED; | ||
| 1772 | - } else { | ||
| 1773 | - GELOGI("Compile op success. op index = %d, output = %s", index, output_path.c_str()); | ||
| 1774 | - } | ||
| 1775 | - index += 1; | ||
| 1776 | - } | ||
| 1777 | - | ||
| 1778 | (void)generator.Finalize(); | 1800 | (void)generator.Finalize(); |
| 1779 | (void)GELib::GetInstance()->Finalize(); | 1801 | (void)GELib::GetInstance()->Finalize(); |
| 1780 | return ret; | 1802 | return ret; |
| @@ -19,6 +19,7 @@ | |||
| 19 | 19 | ||
| 20 | 20 | ||
| 21 | 21 | ||
| 22 | + | ||
| 22 | 23 | ||
| 23 | 24 | ||
| 24 | 25 | ||
| @@ -281,6 +282,9 @@ static Status GEInitializeImpl(const std::map<std::string, std::string> &options | |||
| 281 | std::vector<const char *>({opsproto_path.c_str(), "failed to load the OpsProto lib plugin"})); | 282 | std::vector<const char *>({opsproto_path.c_str(), "failed to load the OpsProto lib plugin"})); |
| 282 | return FAILED; | 283 | return FAILED; |
| 283 | } | 284 | } |
| 285 | + GE_ASSERT_SUCCESS(GePythonRuntimeManager::Instance().EnsureReady()); | ||
| 286 | + GE_DISMISSABLE_GUARD(release_python_runtime, | ||
| 287 | + ([]() { (void)GePythonRuntimeManager::Instance().ShutdownProcess(); })); | ||
| 284 | GE_ASSERT_SUCCESS(fusion::LoadPassPlugins()); | 288 | GE_ASSERT_SUCCESS(fusion::LoadPassPlugins()); |
| 285 | 289 | ||
| 286 | ge::GetContext().Init(); | 290 | ge::GetContext().Init(); |
| @@ -334,6 +338,7 @@ static Status GEInitializeImpl(const std::map<std::string, std::string> &options | |||
| 334 | if (!g_ge_initialized) { | 338 | if (!g_ge_initialized) { |
| 335 | g_ge_initialized = true; | 339 | g_ge_initialized = true; |
| 336 | } | 340 | } |
| 341 | + GE_DISMISS_GUARD(release_python_runtime); | ||
| 337 | 342 | ||
| 338 | GELOGT(TRACE_STOP, "GEInitialize finished"); | 343 | GELOGT(TRACE_STOP, "GEInitialize finished"); |
| 339 | GE_TIMESTAMP_EVENT_END(GEInitializeAll, "GEInitialize::All"); | 344 | GE_TIMESTAMP_EVENT_END(GEInitializeAll, "GEInitialize::All"); |
| @@ -408,6 +413,7 @@ Status GEFinalizeV2() { | |||
| 408 | ret = middle_ret; | 413 | ret = middle_ret; |
| 409 | } | 414 | } |
| 410 | 415 | ||
| 416 | + (void)GePythonRuntimeManager::Instance().ShutdownProcess(); | ||
| 411 | if (g_ge_initialized && (ret == SUCCESS)) { | 417 | if (g_ge_initialized && (ret == SUCCESS)) { |
| 412 | // Unified destruct rt_context | 418 | // Unified destruct rt_context |
| 413 | RtContextUtil::GetInstance().DestroyAllRtContexts(); | 419 | RtContextUtil::GetInstance().DestroyAllRtContexts(); |
| @@ -37,6 +37,7 @@ set(SRC_FORMAT_TRANSFERS | |||
| 37 | ) | 37 | ) |
| 38 | 38 | ||
| 39 | set(SRC_LIST | 39 | set(SRC_LIST |
| 40 | + "common/python_runtime/ge_python_runtime_manager.cc" | ||
| 40 | "common/b_cast/b_cast.cc" | 41 | "common/b_cast/b_cast.cc" |
| 41 | "common/plugin/runtime_plugin_loader.cc" | 42 | "common/plugin/runtime_plugin_loader.cc" |
| 42 | "common/plugin/plugin_caller.cc" | 43 | "common/plugin/plugin_caller.cc" |
| @@ -0,0 +1,117 @@ | |||
| 1 | +/** | ||
| 2 | + * Copyright (c) 2026 Huawei Technologies Co., Ltd. | ||
| 3 | + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of | ||
| 4 | + * CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 5 | + * Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 6 | + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 7 | + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 8 | + * See LICENSE in the root of the software repository for the full text of the License. | ||
| 9 | + */ | ||
| 10 | + | ||
| 11 | + | ||
| 12 | + | ||
| 13 | + | ||
| 14 | + | ||
| 15 | + | ||
| 16 | + | ||
| 17 | + | ||
| 18 | + | ||
| 19 | + | ||
| 20 | +namespace ge { | ||
| 21 | + | ||
| 22 | +GePythonRuntimeManager &GePythonRuntimeManager::Instance() { | ||
| 23 | + static GePythonRuntimeManager instance; | ||
| 24 | + return instance; | ||
| 25 | +} | ||
| 26 | + | ||
| 27 | +void GePythonRuntimeManager::AttachExistingLocked() { | ||
| 28 | + manager_is_owner_ = false; | ||
| 29 | + const char *version = g_python_api.py_get_version(); | ||
| 30 | + GELOGI("[GePythonRuntime] Attach existing interpreter, version[%s], owner=0.", version); | ||
| 31 | +} | ||
| 32 | + | ||
| 33 | +Status GePythonRuntimeManager::EnsureReadyLocked() { | ||
| 34 | + if (ready_) { | ||
| 35 | + return SUCCESS; | ||
| 36 | + } | ||
| 37 | + if (!ResolvePythonCApi(nullptr, false)) { | ||
| 38 | + if (!EnsureLibpythonLoaded(&libpython_handle_)) { | ||
| 39 | + GELOGE(FAILED, "[GePythonRuntime] Load libpython failed."); | ||
| 40 | + return FAILED; | ||
| 41 | + } | ||
| 42 | + if (!ResolvePythonCApi(libpython_handle_, true)) { | ||
| 43 | + GELOGE(FAILED, "[GePythonRuntime] Resolve Python C API failed after loading libpython."); | ||
| 44 | + ResetStateLocked(); | ||
| 45 | + return FAILED; | ||
| 46 | + } | ||
| 47 | + } | ||
| 48 | + | ||
| 49 | + if (g_python_api.py_is_initialized() != 0) { | ||
| 50 | + AttachExistingLocked(); | ||
| 51 | + ready_ = true; | ||
| 52 | + return SUCCESS; | ||
| 53 | + } | ||
| 54 | + | ||
| 55 | + g_python_api.py_initialize(); | ||
| 56 | + if (g_python_api.py_eval_threads_initialized() == 0) { | ||
| 57 | + g_python_api.py_eval_init_threads(); | ||
| 58 | + } | ||
| 59 | + ReleaseInitThreadGilLocked(); | ||
| 60 | + manager_is_owner_ = true; | ||
| 61 | + ready_ = true; | ||
| 62 | + const char *version = g_python_api.py_get_version(); | ||
| 63 | + GELOGI("[GePythonRuntime] Initialize owned interpreter, version[%s].", version); | ||
| 64 | + return SUCCESS; | ||
| 65 | +} | ||
| 66 | + | ||
| 67 | +Status GePythonRuntimeManager::EnsureReady() { | ||
| 68 | + std::lock_guard<std::mutex> lock(mutex_); | ||
| 69 | + return EnsureReadyLocked(); | ||
| 70 | +} | ||
| 71 | + | ||
| 72 | +void GePythonRuntimeManager::ReleaseInitThreadGilLocked() { | ||
| 73 | + if (g_python_api.py_gil_state_check() != 0) { | ||
| 74 | + py_thread_state_ = g_python_api.py_eval_save_thread(); | ||
| 75 | + GELOGI("[GePythonRuntime] Saved init thread state and released GIL."); | ||
| 76 | + } | ||
| 77 | +} | ||
| 78 | + | ||
| 79 | +void GePythonRuntimeManager::ResetStateLocked() { | ||
| 80 | + if (libpython_handle_ != nullptr) { | ||
| 81 | + // GE 初始化/反初始化是进程级的，因此这里不主动 dlclose，直到进程退出时系统统一回收 | ||
| 82 | + libpython_handle_ = nullptr; | ||
| 83 | + } | ||
| 84 | + ResetPythonCApi(); | ||
| 85 | + py_thread_state_ = nullptr; | ||
| 86 | + ready_ = false; | ||
| 87 | + manager_is_owner_ = false; | ||
| 88 | +} | ||
| 89 | + | ||
| 90 | +Status GePythonRuntimeManager::FinalizeOwnedInterpreterLocked() { | ||
| 91 | + if (!manager_is_owner_) { | ||
| 92 | + GELOGI("[GePythonRuntime] Skip finalize because manager is not interpreter owner."); | ||
| 93 | + ResetStateLocked(); | ||
| 94 | + return SUCCESS; | ||
| 95 | + } | ||
| 96 | + if (g_python_api.py_is_initialized() == 0) { | ||
| 97 | + GELOGI("[GePythonRuntime] Skip finalize because interpreter is not initialized."); | ||
| 98 | + ResetStateLocked(); | ||
| 99 | + return SUCCESS; | ||
| 100 | + } | ||
| 101 | + if (py_thread_state_ != nullptr) { | ||
| 102 | + g_python_api.py_eval_restore_thread(py_thread_state_); | ||
| 103 | + py_thread_state_ = nullptr; | ||
| 104 | + GELOGI("[GePythonRuntime] Restored init thread state before finalize."); | ||
| 105 | + } | ||
| 106 | + g_python_api.py_finalize(); | ||
| 107 | + ResetStateLocked(); | ||
| 108 | + GELOGI("[GePythonRuntime] Finalize owned interpreter done."); | ||
| 109 | + return SUCCESS; | ||
| 110 | +} | ||
| 111 | + | ||
| 112 | +Status GePythonRuntimeManager::ShutdownProcess() { | ||
| 113 | + std::lock_guard<std::mutex> lock(mutex_); | ||
| 114 | + return FinalizeOwnedInterpreterLocked(); | ||
| 115 | +} | ||
| 116 | + | ||
| 117 | +} // namespace ge | ||
| @@ -0,0 +1,45 @@ | |||
| 1 | +/** | ||
| 2 | + * Copyright (c) 2026 Huawei Technologies Co., Ltd. | ||
| 3 | + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of | ||
| 4 | + * CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 5 | + * Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 6 | + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 7 | + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 8 | + * See LICENSE in the root of the software repository for the full text of the License. | ||
| 9 | + */ | ||
| 10 | + | ||
| 11 | + | ||
| 12 | + | ||
| 13 | + | ||
| 14 | + | ||
| 15 | + | ||
| 16 | + | ||
| 17 | + | ||
| 18 | +namespace ge { | ||
| 19 | + | ||
| 20 | +class GePythonRuntimeManager { | ||
| 21 | + public: | ||
| 22 | + static GePythonRuntimeManager &Instance(); | ||
| 23 | + | ||
| 24 | + Status EnsureReady(); | ||
| 25 | + Status ShutdownProcess(); | ||
| 26 | + | ||
| 27 | + private: | ||
| 28 | + GePythonRuntimeManager() = default; | ||
| 29 | + | ||
| 30 | + Status EnsureReadyLocked(); | ||
| 31 | + void AttachExistingLocked(); | ||
| 32 | + void ReleaseInitThreadGilLocked(); | ||
| 33 | + void ResetStateLocked(); | ||
| 34 | + Status FinalizeOwnedInterpreterLocked(); | ||
| 35 | + | ||
| 36 | + mutable std::mutex mutex_; | ||
| 37 | + bool ready_{false}; | ||
| 38 | + bool manager_is_owner_{false}; | ||
| 39 | + void *libpython_handle_{nullptr}; | ||
| 40 | + void *py_thread_state_{nullptr}; | ||
| 41 | +}; | ||
| 42 | + | ||
| 43 | +} // namespace ge | ||
| 44 | + | ||
| 45 | + | ||
| @@ -0,0 +1,223 @@ | |||
| 1 | +/** | ||
| 2 | + * Copyright (c) 2026 Huawei Technologies Co., Ltd. | ||
| 3 | + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of | ||
| 4 | + * CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 5 | + * Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 6 | + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 7 | + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 8 | + * See LICENSE in the root of the software repository for the full text of the License. | ||
| 9 | + */ | ||
| 10 | + | ||
| 11 | + | ||
| 12 | + | ||
| 13 | + | ||
| 14 | + | ||
| 15 | + | ||
| 16 | + | ||
| 17 | + | ||
| 18 | + | ||
| 19 | + | ||
| 20 | + | ||
| 21 | +namespace ge { | ||
| 22 | + | ||
| 23 | +using PyIsInitializedFn = int (*)(); | ||
| 24 | +using PyInitializeFn = void (*)(); | ||
| 25 | +using PyFinalizeFn = void (*)(); | ||
| 26 | +using PyGetVersionFn = const char *(*)(); | ||
| 27 | +using PyEvalInitThreadsFn = void (*)(); | ||
| 28 | +using PyEvalThreadsInitializedFn = int (*)(); | ||
| 29 | +using PyEvalSaveThreadFn = void *(*)(); | ||
| 30 | +using PyEvalRestoreThreadFn = void (*)(void *); | ||
| 31 | +using PyGILStateCheckFn = int (*)(); | ||
| 32 | + | ||
| 33 | +struct PythonCApi { | ||
| 34 | + PyIsInitializedFn py_is_initialized{nullptr}; | ||
| 35 | + PyInitializeFn py_initialize{nullptr}; | ||
| 36 | + PyFinalizeFn py_finalize{nullptr}; | ||
| 37 | + PyGetVersionFn py_get_version{nullptr}; | ||
| 38 | + PyEvalInitThreadsFn py_eval_init_threads{nullptr}; | ||
| 39 | + PyEvalThreadsInitializedFn py_eval_threads_initialized{nullptr}; | ||
| 40 | + PyEvalSaveThreadFn py_eval_save_thread{nullptr}; | ||
| 41 | + PyEvalRestoreThreadFn py_eval_restore_thread{nullptr}; | ||
| 42 | + PyGILStateCheckFn py_gil_state_check{nullptr}; | ||
| 43 | +}; | ||
| 44 | + | ||
| 45 | +struct PythonProbeResult { | ||
| 46 | + std::string python_command; | ||
| 47 | + std::string python_tag; | ||
| 48 | + std::string libpython_path; | ||
| 49 | +}; | ||
| 50 | + | ||
| 51 | +constexpr const char *kPyIsInitializedSymbol = "Py_IsInitialized"; | ||
| 52 | +constexpr const char *kPyInitializeSymbol = "Py_Initialize"; | ||
| 53 | +constexpr const char *kPyFinalizeSymbol = "Py_Finalize"; | ||
| 54 | +constexpr const char *kPyGetVersionSymbol = "Py_GetVersion"; | ||
| 55 | +constexpr const char *kPyEvalInitThreadsSymbol = "PyEval_InitThreads"; | ||
| 56 | +constexpr const char *kPyEvalThreadsInitializedSymbol = "PyEval_ThreadsInitialized"; | ||
| 57 | +constexpr const char *kPyEvalSaveThreadSymbol = "PyEval_SaveThread"; | ||
| 58 | +constexpr const char *kPyEvalRestoreThreadSymbol = "PyEval_RestoreThread"; | ||
| 59 | +constexpr const char *kPyGILStateCheckSymbol = "PyGILState_Check"; | ||
| 60 | +constexpr const char *kPythonRuntimeProbeScript = | ||
| 61 | + " -c \"import sys; print('cp%d%d' % sys.version_info[:2])\" 2>/dev/null"; | ||
| 62 | +constexpr const char *kLibpythonProbeScript = | ||
| 63 | + " -c \"import os, sysconfig\n" | ||
| 64 | + "version = sysconfig.get_config_var('VERSION') or ''\n" | ||
| 65 | + "libdir = sysconfig.get_config_var('LIBDIR') or ''\n" | ||
| 66 | + "candidates = [sysconfig.get_config_var('LDLIBRARY'), sysconfig.get_config_var('INSTSONAME'), " | ||
| 67 | + "sysconfig.get_config_var('LIBRARY')]\n" | ||
| 68 | + "candidates.extend([('libpython%s.so.1.0' % version) if version else '', " | ||
| 69 | + "('libpython%s.so' % version) if version else ''])\n" | ||
| 70 | + "seen = []\n" | ||
| 71 | + "for item in candidates:\n" | ||
| 72 | + " if item and item not in seen:\n" | ||
| 73 | + " seen.append(item)\n" | ||
| 74 | + "for item in seen:\n" | ||
| 75 | + " if '.so' not in item:\n" | ||
| 76 | + " continue\n" | ||
| 77 | + " path = os.path.join(libdir, item) if libdir else ''\n" | ||
| 78 | + " if path and os.path.exists(path):\n" | ||
| 79 | + " print(path)\n" | ||
| 80 | + " break\" 2>/dev/null"; | ||
| 81 | +constexpr int kOpenFlags = RTLD_NOW | RTLD_GLOBAL; | ||
| 82 | + | ||
| 83 | +inline PythonCApi g_python_api; | ||
| 84 | + | ||
| 85 | + | ||
| 86 | + do { \ | ||
| 87 | + resolved.func_field = reinterpret_cast<decltype(resolved.func_field)>(dlsym(handle, py_symbol)); \ | ||
| 88 | + if (resolved.func_field == nullptr) { \ | ||
| 89 | + if (log_failure) { \ | ||
| 90 | + GELOGW("[GePythonRuntime] Failed to dlsym python symbol[%s].", py_symbol); \ | ||
| 91 | + } \ | ||
| 92 | + return false; \ | ||
| 93 | + } \ | ||
| 94 | + } while (0) | ||
| 95 | + | ||
| 96 | +inline bool ResolvePythonCApi(void *libpython_handle, const bool log_failure) { | ||
| 97 | + void *handle = (libpython_handle != nullptr) ? libpython_handle : RTLD_DEFAULT; | ||
| 98 | + PythonCApi resolved; | ||
| 99 | + GE_BIND_PY_SYMBOL(py_is_initialized, kPyIsInitializedSymbol); | ||
| 100 | + GE_BIND_PY_SYMBOL(py_initialize, kPyInitializeSymbol); | ||
| 101 | + GE_BIND_PY_SYMBOL(py_finalize, kPyFinalizeSymbol); | ||
| 102 | + GE_BIND_PY_SYMBOL(py_get_version, kPyGetVersionSymbol); | ||
| 103 | + GE_BIND_PY_SYMBOL(py_eval_init_threads, kPyEvalInitThreadsSymbol); | ||
| 104 | + GE_BIND_PY_SYMBOL(py_eval_threads_initialized, kPyEvalThreadsInitializedSymbol); | ||
| 105 | + GE_BIND_PY_SYMBOL(py_eval_save_thread, kPyEvalSaveThreadSymbol); | ||
| 106 | + GE_BIND_PY_SYMBOL(py_eval_restore_thread, kPyEvalRestoreThreadSymbol); | ||
| 107 | + GE_BIND_PY_SYMBOL(py_gil_state_check, kPyGILStateCheckSymbol); | ||
| 108 | + g_python_api = resolved; | ||
| 109 | + return true; | ||
| 110 | +} | ||
| 111 | + | ||
| 112 | + | ||
| 113 | + | ||
| 114 | +inline void ResetPythonCApi() { | ||
| 115 | + g_python_api = PythonCApi {}; | ||
| 116 | +} | ||
| 117 | + | ||
| 118 | +inline std::string FirstLine(const std::string &text) { | ||
| 119 | + const auto pos = text.find('\n'); | ||
| 120 | + if (pos == std::string::npos) { | ||
| 121 | + return text; | ||
| 122 | + } | ||
| 123 | + return text.substr(0U, pos); | ||
| 124 | +} | ||
| 125 | + | ||
| 126 | +inline bool ReadCommandOutput(const std::string &command, std::string &output) { | ||
| 127 | + FILE *fp = popen(command.c_str(), "r"); | ||
| 128 | + if (fp == nullptr) { | ||
| 129 | + return false; | ||
| 130 | + } | ||
| 131 | + char buffer[256] = {0}; | ||
| 132 | + while (fgets(buffer, sizeof(buffer), fp) != nullptr) { | ||
| 133 | + output += buffer; | ||
| 134 | + } | ||
| 135 | + const auto ret = pclose(fp); | ||
| 136 | + return (ret == 0) && (!output.empty()); | ||
| 137 | +} | ||
| 138 | + | ||
| 139 | +inline bool ProbePythonRuntimeFromCommand(const char *python_command, PythonProbeResult &result) { | ||
| 140 | + if ((python_command == nullptr) || (python_command[0] == '\0')) { | ||
| 141 | + return false; | ||
| 142 | + } | ||
| 143 | + std::string output; | ||
| 144 | + if (!ReadCommandOutput(std::string(python_command) + kPythonRuntimeProbeScript, output)) { | ||
| 145 | + return false; | ||
| 146 | + } | ||
| 147 | + const auto python_tag = FirstLine(output); | ||
| 148 | + if (python_tag.empty()) { | ||
| 149 | + return false; | ||
| 150 | + } | ||
| 151 | + result.python_command = python_command; | ||
| 152 | + result.python_tag = python_tag; | ||
| 153 | + return true; | ||
| 154 | +} | ||
| 155 | + | ||
| 156 | +inline std::string ResolveLibpythonPathFromCommand(const std::string &python_command) { | ||
| 157 | + if (python_command.empty()) { | ||
| 158 | + return ""; | ||
| 159 | + } | ||
| 160 | + std::string output; | ||
| 161 | + if (!ReadCommandOutput(python_command + kLibpythonProbeScript, output)) { | ||
| 162 | + return ""; | ||
| 163 | + } | ||
| 164 | + const auto libpython_path = FirstLine(output); | ||
| 165 | + if (libpython_path.empty()) { | ||
| 166 | + return ""; | ||
| 167 | + } | ||
| 168 | + const auto real_path = RealPath(libpython_path.c_str()); | ||
| 169 | + return real_path.empty() ? libpython_path : real_path; | ||
| 170 | +} | ||
| 171 | + | ||
| 172 | +inline bool ProbeRuntimeCandidate(const char *candidate, PythonProbeResult &probed) { | ||
| 173 | + if (!ProbePythonRuntimeFromCommand(candidate, probed)) { | ||
| 174 | + return false; | ||
| 175 | + } | ||
| 176 | + probed.libpython_path = ResolveLibpythonPathFromCommand(probed.python_command); | ||
| 177 | + if (probed.libpython_path.empty()) { | ||
| 178 | + GELOGW("[GePythonRuntime] Skip python command[%s] because libpython path is unresolved.", | ||
| 179 | + probed.python_command.c_str()); | ||
| 180 | + return false; | ||
| 181 | + } | ||
| 182 | + return true; | ||
| 183 | +} | ||
| 184 | + | ||
| 185 | +inline bool EnsureLibpythonLoaded(void **handle) { | ||
| 186 | + if (handle == nullptr) { | ||
| 187 | + return false; | ||
| 188 | + } | ||
| 189 | + if (*handle != nullptr) { | ||
| 190 | + return true; | ||
| 191 | + } | ||
| 192 | + | ||
| 193 | + bool has_probe_result = false; | ||
| 194 | + for (const char *candidate : {"python3", "python"}) { | ||
| 195 | + PythonProbeResult probe_result; | ||
| 196 | + if (!ProbeRuntimeCandidate(candidate, probe_result)) { | ||
| 197 | + continue; | ||
| 198 | + } | ||
| 199 | + has_probe_result = true; | ||
| 200 | + void *opened_handle = dlopen(probe_result.libpython_path.c_str(), kOpenFlags); | ||
| 201 | + if (opened_handle == nullptr) { | ||
| 202 | + const char *open_error = dlerror(); | ||
| 203 | + GELOGW("[GePythonRuntime] dlopen libpython[%s] failed: %s, try next candidate.", | ||
| 204 | + probe_result.libpython_path.c_str(), open_error == nullptr ? "" : open_error); | ||
| 205 | + continue; | ||
| 206 | + } | ||
| 207 | + *handle = opened_handle; | ||
| 208 | + GELOGI("[GePythonRuntime] Loaded libpython[%s] for python tag[%s].", probe_result.libpython_path.c_str(), | ||
| 209 | + probe_result.python_tag.c_str()); | ||
| 210 | + return true; | ||
| 211 | + } | ||
| 212 | + | ||
| 213 | + if (!has_probe_result) { | ||
| 214 | + GELOGE(FAILED, "[GePythonRuntime] Probe target python runtime failed."); | ||
| 215 | + return false; | ||
| 216 | + } | ||
| 217 | + GELOGE(FAILED, "[GePythonRuntime] Load libpython failed for all probed python runtime candidates."); | ||
| 218 | + return false; | ||
| 219 | +} | ||
| 220 | + | ||
| 221 | +} // namespace ge | ||
| 222 | + | ||
| 223 | + | ||
| @@ -60,6 +60,7 @@ | |||
| 60 | 60 | ||
| 61 | 61 | ||
| 62 | 62 | ||
| 63 | + | ||
| 63 | 64 | ||
| 64 | 65 | ||
| 65 | 66 | ||
| @@ -440,6 +441,8 @@ static graphStatus aclgrphBuildInitializeImpl(std::map<std::string, std::string> | |||
| 440 | ge::PrintOptionMap(global_options, "global option"); | 441 | ge::PrintOptionMap(global_options, "global option"); |
| 441 | GE_ASSERT_GRAPH_SUCCESS(OpLibRegistry::GetInstance().PreProcessForCustomOp()); | 442 | GE_ASSERT_GRAPH_SUCCESS(OpLibRegistry::GetInstance().PreProcessForCustomOp()); |
| 442 | LoadOpsProto(); | 443 | LoadOpsProto(); |
| 444 | + GE_ASSERT_SUCCESS(GePythonRuntimeManager::Instance().EnsureReady()); | ||
| 445 | + GE_DISMISSABLE_GUARD(release_python_runtime, []() { (void)GePythonRuntimeManager::Instance().ShutdownProcess(); }); | ||
| 443 | GE_ASSERT_SUCCESS(fusion::LoadPassPlugins()); | 446 | GE_ASSERT_SUCCESS(fusion::LoadPassPlugins()); |
| 444 | 447 | ||
| 445 | std::shared_ptr<ge::GELib> instance_ptr = ge::GELib::GetInstance(); | 448 | std::shared_ptr<ge::GELib> instance_ptr = ge::GELib::GetInstance(); |
| @@ -457,6 +460,7 @@ static graphStatus aclgrphBuildInitializeImpl(std::map<std::string, std::string> | |||
| 457 | GELOGW("gelib has been initialized!"); | 460 | GELOGW("gelib has been initialized!"); |
| 458 | Status ret = static_cast<uint32_t>(error_message::ErrMgrInit(error_message::ErrorMessageMode::INTERNAL_MODE)); | 461 | Status ret = static_cast<uint32_t>(error_message::ErrMgrInit(error_message::ErrorMessageMode::INTERNAL_MODE)); |
| 459 | GE_ASSERT_SUCCESS(ret, "ErrorManager init failed!"); | 462 | GE_ASSERT_SUCCESS(ret, "ErrorManager init failed!"); |
| 463 | + GE_DISMISS_GUARD(release_python_runtime); | ||
| 460 | return GRAPH_SUCCESS; | 464 | return GRAPH_SUCCESS; |
| 461 | } | 465 | } |
| 462 | 466 | ||
| @@ -483,9 +487,10 @@ void aclgrphBuildFinalize() { | |||
| 483 | (void)fusion::ShutdownPassPluginsForProcess(); | 487 | (void)fusion::ShutdownPassPluginsForProcess(); |
| 484 | if (ge::GELib::GetInstance() != nullptr && ge::GELib::GetInstance()->InitFlag()) { | 488 | if (ge::GELib::GetInstance() != nullptr && ge::GELib::GetInstance()->InitFlag()) { |
| 485 | (void)ge::GELib::GetInstance()->Finalize(); | 489 | (void)ge::GELib::GetInstance()->Finalize(); |
| 486 | - return; | 490 | + } else { |
| 491 | + GELOGW("[Notice] gelib has not been initialized!do nothing!"); | ||
| 487 | } | 492 | } |
| 488 | - GELOGW("[Notice] gelib has not been initialized!do nothing!"); | 493 | + (void)GePythonRuntimeManager::Instance().ShutdownProcess(); |
| 489 | } | 494 | } |
| 490 | 495 | ||
| 491 | class Impl { | 496 | class Impl { |
| @@ -52,6 +52,7 @@ | |||
| 52 | 52 | ||
| 53 | 53 | ||
| 54 | 54 | ||
| 55 | + | ||
| 55 | 56 | ||
| 56 | namespace { | 57 | namespace { |
| 57 | 58 | ||
| @@ -466,6 +467,9 @@ Status GeGenerator::Initialize(const std::map<std::string, std::string> &options | |||
| 466 | std::map<std::string, std::string> option_tmp; | 467 | std::map<std::string, std::string> option_tmp; |
| 467 | option_tmp.emplace(std::pair<std::string, std::string>(string("ge.opsProtoLibPath"), opsproto_path)); | 468 | option_tmp.emplace(std::pair<std::string, std::string>(string("ge.opsProtoLibPath"), opsproto_path)); |
| 468 | (void)manager->Initialize(option_tmp); | 469 | (void)manager->Initialize(option_tmp); |
| 470 | + GE_ASSERT_SUCCESS(GePythonRuntimeManager::Instance().EnsureReady()); | ||
| 471 | + GE_DISMISSABLE_GUARD(release_python_runtime, | ||
| 472 | + ([]() { (void)GePythonRuntimeManager::Instance().ShutdownProcess(); })); | ||
| 469 | GE_ASSERT_SUCCESS(fusion::LoadPassPlugins()); | 473 | GE_ASSERT_SUCCESS(fusion::LoadPassPlugins()); |
| 470 | 474 | ||
| 471 | ret = impl_->graph_manager_.Initialize(options); | 475 | ret = impl_->graph_manager_.Initialize(options); |
| @@ -485,6 +489,7 @@ Status GeGenerator::Initialize(const std::map<std::string, std::string> &options | |||
| 485 | if (iter != options.end()) { | 489 | if (iter != options.end()) { |
| 486 | impl_->build_step_ = iter->second; | 490 | impl_->build_step_ = iter->second; |
| 487 | } | 491 | } |
| 492 | + GE_DISMISS_GUARD(release_python_runtime); | ||
| 488 | return SUCCESS; | 493 | return SUCCESS; |
| 489 | } | 494 | } |
| 490 | 495 | ||
| @@ -61,6 +61,7 @@ target_include_directories(helper_runtime_test PRIVATE | |||
| 61 | target_link_options(helper_runtime_test PRIVATE | 61 | target_link_options(helper_runtime_test PRIVATE |
| 62 | -Wl,--disable-new-dtags | 62 | -Wl,--disable-new-dtags |
| 63 | -Wl,-Bsymbolic | 63 | -Wl,-Bsymbolic |
| 64 | + -rdynamic | ||
| 64 | ) | 65 | ) |
| 65 | 66 | ||
| 66 | target_compile_options(helper_runtime_test PRIVATE | 67 | target_compile_options(helper_runtime_test PRIVATE |
| @@ -15,6 +15,7 @@ | |||
| 15 | 15 | ||
| 16 | 16 | ||
| 17 | 17 | ||
| 18 | + | ||
| 18 | 19 | ||
| 19 | namespace ge { | 20 | namespace ge { |
| 20 | 21 | ||
| @@ -22,6 +23,7 @@ inline void InitGe() { | |||
| 22 | std::map<AscendString, AscendString> options; | 23 | std::map<AscendString, AscendString> options; |
| 23 | options[ge::OPTION_HOST_ENV_OS] = "linux"; | 24 | options[ge::OPTION_HOST_ENV_OS] = "linux"; |
| 24 | options[ge::OPTION_HOST_ENV_CPU] = "x86_64"; | 25 | options[ge::OPTION_HOST_ENV_CPU] = "x86_64"; |
| 26 | + ScopedUnsetLdPreload guard; | ||
| 25 | auto init_status = ge::GEInitialize(options); | 27 | auto init_status = ge::GEInitialize(options); |
| 26 | if (init_status != SUCCESS) { | 28 | if (init_status != SUCCESS) { |
| 27 | std::cout << "ge init failed , ret code:" << init_status << std::endl; | 29 | std::cout << "ge init failed , ret code:" << init_status << std::endl; |
| @@ -41,6 +43,7 @@ inline void ReInitGe() { | |||
| 41 | std::map<AscendString, AscendString> options; | 43 | std::map<AscendString, AscendString> options; |
| 42 | options[ge::OPTION_HOST_ENV_OS] = "linux"; | 44 | options[ge::OPTION_HOST_ENV_OS] = "linux"; |
| 43 | options[ge::OPTION_HOST_ENV_CPU] = "x86_64"; | 45 | options[ge::OPTION_HOST_ENV_CPU] = "x86_64"; |
| 46 | + ScopedUnsetLdPreload guard; | ||
| 44 | auto init_status = ge::GEInitializeV2(options); | 47 | auto init_status = ge::GEInitializeV2(options); |
| 45 | if (init_status != SUCCESS) { | 48 | if (init_status != SUCCESS) { |
| 46 | std::cout << "ge init failed , ret code:" << init_status << std::endl; | 49 | std::cout << "ge init failed , ret code:" << init_status << std::endl; |
| @@ -20,6 +20,20 @@ | |||
| 20 | using namespace std; | 20 | using namespace std; |
| 21 | using namespace ge; | 21 | using namespace ge; |
| 22 | 22 | ||
| 23 | +extern "C" const char *__lsan_default_suppressions() { | ||
| 24 | + return "leak:_PyObject_Malloc\n" | ||
| 25 | + "leak:_PyObject_Realloc\n" | ||
| 26 | + "leak:PyType_GenericAlloc\n" | ||
| 27 | + "leak:PyType_Ready\n" | ||
| 28 | + "leak:PyObject_GC_New\n" | ||
| 29 | + "leak:PyObject_GC_NewVar\n" | ||
| 30 | + "leak:_PyObject_GC_Malloc\n" | ||
| 31 | + "leak:PyUnicode_New\n" | ||
| 32 | + "leak:Py_InitializeEx\n" | ||
| 33 | + "leak:PyImport_ImportModuleLevelObject\n" | ||
| 34 | + "leak:libpython\n"; | ||
| 35 | +} | ||
| 36 | + | ||
| 23 | int main(int argc, char **argv) { | 37 | int main(int argc, char **argv) { |
| 24 | setenv("GE_PROFILING_TO_STD_OUT", "1", true); | 38 | setenv("GE_PROFILING_TO_STD_OUT", "1", true); |
| 25 | // Init running dir env | 39 | // Init running dir env |
| @@ -227,6 +227,11 @@ class MockMmpa : public ge::MmpaStubApiGe { | |||
| 227 | class UtestDflowApi : public testing::Test { | 227 | class UtestDflowApi : public testing::Test { |
| 228 | protected: | 228 | protected: |
| 229 | static void SetUpTestSuite() { | 229 | static void SetUpTestSuite() { |
| 230 | + const auto env_ptr = getenv("LD_PRELOAD"); | ||
| 231 | + if (env_ptr != nullptr) { | ||
| 232 | + env = env_ptr; | ||
| 233 | + unsetenv("LD_PRELOAD"); | ||
| 234 | + } | ||
| 230 | // Init running dir env | 235 | // Init running dir env |
| 231 | ge::DirEnv::GetInstance().InitEngineConfJson(); | 236 | ge::DirEnv::GetInstance().InitEngineConfJson(); |
| 232 | const std::map<AscendString, AscendString> options{}; | 237 | const std::map<AscendString, AscendString> options{}; |
| @@ -239,6 +244,9 @@ class UtestDflowApi : public testing::Test { | |||
| 239 | static void TearDownTestSuite() { | 244 | static void TearDownTestSuite() { |
| 240 | ge::GEFinalizeV2(); | 245 | ge::GEFinalizeV2(); |
| 241 | unsetenv("RESOURCE_CONFIG_PATH"); | 246 | unsetenv("RESOURCE_CONFIG_PATH"); |
| 247 | + if (!env.empty()) { | ||
| 248 | + setenv("LD_PRELOAD", env.c_str(), 1); | ||
| 249 | + } | ||
| 242 | } | 250 | } |
| 243 | 251 | ||
| 244 | void SetUp() override { | 252 | void SetUp() override { |
| @@ -278,8 +286,12 @@ class UtestDflowApi : public testing::Test { | |||
| 278 | ge::ExecutionRuntime::SetExecutionRuntime(nullptr); | 286 | ge::ExecutionRuntime::SetExecutionRuntime(nullptr); |
| 279 | ge::OpsKernelBuilderRegistry::GetInstance().UnregisterAll(); | 287 | ge::OpsKernelBuilderRegistry::GetInstance().UnregisterAll(); |
| 280 | } | 288 | } |
| 289 | + | ||
| 290 | + static std::string env; | ||
| 281 | }; | 291 | }; |
| 282 | 292 | ||
| 293 | +std::string UtestDflowApi::env; | ||
| 294 | + | ||
| 283 | TEST_F(UtestDflowApi, DFlowInitialize) { | 295 | TEST_F(UtestDflowApi, DFlowInitialize) { |
| 284 | std::map<AscendString, AscendString> options = {}; | 296 | std::map<AscendString, AscendString> options = {}; |
| 285 | std::string empty_key = ""; | 297 | std::string empty_key = ""; |
| @@ -98,6 +98,11 @@ class GeFakeOpsKernelBuilder : public OpsKernelBuilder { | |||
| 98 | class UtestGeApiDflow : public testing::Test { | 98 | class UtestGeApiDflow : public testing::Test { |
| 99 | protected: | 99 | protected: |
| 100 | static void SetUpTestSuite() { | 100 | static void SetUpTestSuite() { |
| 101 | + const auto env_ptr = getenv("LD_PRELOAD"); | ||
| 102 | + if (env_ptr != nullptr) { | ||
| 103 | + suite_env = env_ptr; | ||
| 104 | + unsetenv("LD_PRELOAD"); | ||
| 105 | + } | ||
| 101 | // Init running dir env | 106 | // Init running dir env |
| 102 | ge::DirEnv::GetInstance().InitEngineConfJson(); | 107 | ge::DirEnv::GetInstance().InitEngineConfJson(); |
| 103 | const std::map<AscendString, AscendString> options{}; | 108 | const std::map<AscendString, AscendString> options{}; |
| @@ -109,6 +114,9 @@ class UtestGeApiDflow : public testing::Test { | |||
| 109 | static void TearDownTestSuite() { | 114 | static void TearDownTestSuite() { |
| 110 | ge::GEFinalizeV2(); | 115 | ge::GEFinalizeV2(); |
| 111 | unsetenv("RESOURCE_CONFIG_PATH"); | 116 | unsetenv("RESOURCE_CONFIG_PATH"); |
| 117 | + if (!suite_env.empty()) { | ||
| 118 | + setenv("LD_PRELOAD", suite_env.c_str(), 1); | ||
| 119 | + } | ||
| 112 | } | 120 | } |
| 113 | 121 | ||
| 114 | void SetUp() override { | 122 | void SetUp() override { |
| @@ -137,8 +145,11 @@ class UtestGeApiDflow : public testing::Test { | |||
| 137 | } | 145 | } |
| 138 | 146 | ||
| 139 | std::string env; | 147 | std::string env; |
| 148 | + static std::string suite_env; | ||
| 140 | }; | 149 | }; |
| 141 | 150 | ||
| 151 | +std::string UtestGeApiDflow::suite_env; | ||
| 152 | + | ||
| 142 | TEST_F(UtestGeApiDflow, Feed_test_not_init) { | 153 | TEST_F(UtestGeApiDflow, Feed_test_not_init) { |
| 143 | std::map<std::string, std::string> options; | 154 | std::map<std::string, std::string> options; |
| 144 | Session session(options); | 155 | Session session(options); |
| @@ -172,6 +172,8 @@ target_compile_definitions(ut_libge_helper_utest PRIVATE | |||
| 172 | target_link_options(ut_libge_helper_utest PRIVATE | 172 | target_link_options(ut_libge_helper_utest PRIVATE |
| 173 | -Wl,--disable-new-dtags | 173 | -Wl,--disable-new-dtags |
| 174 | -Wl,-rpath,${CMAKE_CURRENT_LIST_DIR} | 174 | -Wl,-rpath,${CMAKE_CURRENT_LIST_DIR} |
| 175 | + -rdynamic | ||
| 176 | + -Wl,-Bsymbolic | ||
| 175 | ) | 177 | ) |
| 176 | 178 | ||
| 177 | target_link_libraries(ut_libge_helper_utest | 179 | target_link_libraries(ut_libge_helper_utest |
| @@ -15,6 +15,18 @@ | |||
| 15 | using namespace std; | 15 | using namespace std; |
| 16 | using namespace ge; | 16 | using namespace ge; |
| 17 | 17 | ||
| 18 | +extern "C" const char *__lsan_default_suppressions() { | ||
| 19 | + return "leak:_PyObject_Malloc\n" | ||
| 20 | + "leak:_PyObject_Realloc\n" | ||
| 21 | + "leak:PyType_GenericAlloc\n" | ||
| 22 | + "leak:PyType_Ready\n" | ||
| 23 | + "leak:PyObject_GC_New\n" | ||
| 24 | + "leak:PyObject_GC_NewVar\n" | ||
| 25 | + "leak:_PyObject_GC_Malloc\n" | ||
| 26 | + "leak:PyUnicode_New\n" | ||
| 27 | + "leak:libpython\n"; | ||
| 28 | +} | ||
| 29 | + | ||
| 18 | int main(int argc, char **argv) { | 30 | int main(int argc, char **argv) { |
| 19 | // init the logging | 31 | // init the logging |
| 20 | testing::InitGoogleTest(&argc, argv); | 32 | testing::InitGoogleTest(&argc, argv); |
| @@ -0,0 +1,39 @@ | |||
| 1 | +/** | ||
| 2 | + * Copyright (c) 2026 Huawei Technologies Co., Ltd. | ||
| 3 | + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of | ||
| 4 | + * CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 5 | + * Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 6 | + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 7 | + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 8 | + * See LICENSE in the root of the software repository for the full text of the License. | ||
| 9 | + */ | ||
| 10 | + | ||
| 11 | + | ||
| 12 | + | ||
| 13 | + | ||
| 14 | + | ||
| 15 | + | ||
| 16 | + | ||
| 17 | +class ScopedUnsetLdPreload { | ||
| 18 | + public: | ||
| 19 | + ScopedUnsetLdPreload() { | ||
| 20 | + const char *ld_preload = std::getenv("LD_PRELOAD"); | ||
| 21 | + if (ld_preload != nullptr) { | ||
| 22 | + has_ld_preload_ = true; | ||
| 23 | + ld_preload_ = ld_preload; | ||
| 24 | + (void)unsetenv("LD_PRELOAD"); | ||
| 25 | + } | ||
| 26 | + } | ||
| 27 | + | ||
| 28 | + ~ScopedUnsetLdPreload() { | ||
| 29 | + if (has_ld_preload_) { | ||
| 30 | + (void)setenv("LD_PRELOAD", ld_preload_.c_str(), 1); | ||
| 31 | + } | ||
| 32 | + } | ||
| 33 | + | ||
| 34 | + private: | ||
| 35 | + bool has_ld_preload_{false}; | ||
| 36 | + std::string ld_preload_; | ||
| 37 | +}; | ||
| 38 | + | ||
| 39 | + | ||
| @@ -23,6 +23,8 @@ target_compile_options(ge_running_env_test PRIVATE | |||
| 23 | 23 | ||
| 24 | target_link_options(ge_running_env_test PRIVATE | 24 | target_link_options(ge_running_env_test PRIVATE |
| 25 | -Wl,--disable-new-dtags | 25 | -Wl,--disable-new-dtags |
| 26 | + -rdynamic | ||
| 27 | + -Wl,-Bsymbolic | ||
| 26 | ) | 28 | ) |
| 27 | 29 | ||
| 28 | target_link_libraries(ge_running_env_test PUBLIC intf_llt_pub GTestShared::gtest ge_running_env ge_common_base | 30 | target_link_libraries(ge_running_env_test PUBLIC intf_llt_pub GTestShared::gtest ge_running_env ge_common_base |
| @@ -13,14 +13,30 @@ | |||
| 13 | 13 | ||
| 14 | 14 | ||
| 15 | 15 | ||
| 16 | + | ||
| 16 | 17 | ||
| 17 | using namespace std; | 18 | using namespace std; |
| 18 | using namespace ge; | 19 | using namespace ge; |
| 19 | 20 | ||
| 21 | +extern "C" const char *__lsan_default_suppressions() { | ||
| 22 | + return "leak:_PyObject_Malloc\n" | ||
| 23 | + "leak:_PyObject_Realloc\n" | ||
| 24 | + "leak:PyType_GenericAlloc\n" | ||
| 25 | + "leak:PyType_Ready\n" | ||
| 26 | + "leak:PyObject_GC_New\n" | ||
| 27 | + "leak:PyObject_GC_NewVar\n" | ||
| 28 | + "leak:_PyObject_GC_Malloc\n" | ||
| 29 | + "leak:PyUnicode_New\n" | ||
| 30 | + "leak:Py_InitializeEx\n" | ||
| 31 | + "leak:PyImport_ImportModuleLevelObject\n" | ||
| 32 | + "leak:libpython\n"; | ||
| 33 | +} | ||
| 34 | + | ||
| 20 | int main(int argc, char **argv) { | 35 | int main(int argc, char **argv) { |
| 21 | map<AscendString, AscendString> options; | 36 | map<AscendString, AscendString> options; |
| 22 | options.insert({AscendString("ge.exec.opWaitTimeout"), AscendString("11")}); | 37 | options.insert({AscendString("ge.exec.opWaitTimeout"), AscendString("11")}); |
| 23 | options.insert({AscendString("ge.exec.opExecuteTimeout"), AscendString("11")}); | 38 | options.insert({AscendString("ge.exec.opExecuteTimeout"), AscendString("11")}); |
| 39 | + ScopedUnsetLdPreload guard; | ||
| 24 | ge::GEInitialize(options); | 40 | ge::GEInitialize(options); |
| 25 | GeRunningEnvFaker::BackupEnv(); | 41 | GeRunningEnvFaker::BackupEnv(); |
| 26 | testing::InitGoogleTest(&argc, argv); | 42 | testing::InitGoogleTest(&argc, argv); |
| @@ -138,6 +138,8 @@ target_include_directories(graph_engine_test PRIVATE | |||
| 138 | 138 | ||
| 139 | target_link_options(graph_engine_test PRIVATE | 139 | target_link_options(graph_engine_test PRIVATE |
| 140 | -Wl,--disable-new-dtags | 140 | -Wl,--disable-new-dtags |
| 141 | + -rdynamic | ||
| 142 | + -Wl,-Bsymbolic | ||
| 141 | ) | 143 | ) |
| 142 | 144 | ||
| 143 | target_link_libraries(graph_engine_test | 145 | target_link_libraries(graph_engine_test |
| @@ -226,6 +228,8 @@ target_include_directories(ge_common_atc PRIVATE | |||
| 226 | 228 | ||
| 227 | target_link_options(ge_common_atc PRIVATE | 229 | target_link_options(ge_common_atc PRIVATE |
| 228 | -Wl,--disable-new-dtags | 230 | -Wl,--disable-new-dtags |
| 231 | + -rdynamic | ||
| 232 | + -Wl,-Bsymbolic | ||
| 229 | ) | 233 | ) |
| 230 | 234 | ||
| 231 | target_link_libraries(ge_common_atc | 235 | target_link_libraries(ge_common_atc |
| @@ -401,6 +405,7 @@ target_link_options(graph_engine_compile_test PRIVATE | |||
| 401 | -Wl,--disable-new-dtags | 405 | -Wl,--disable-new-dtags |
| 402 | -Wl,-rpath,${CMAKE_BINARY_DIR}/compiler | 406 | -Wl,-rpath,${CMAKE_BINARY_DIR}/compiler |
| 403 | -rdynamic | 407 | -rdynamic |
| 408 | + -Wl,-Bsymbolic | ||
| 404 | ) | 409 | ) |
| 405 | target_link_libraries(graph_engine_compile_test | 410 | target_link_libraries(graph_engine_compile_test |
| 406 | intf_llt_pub | 411 | intf_llt_pub |
| @@ -20,6 +20,7 @@ | |||
| 20 | 20 | ||
| 21 | 21 | ||
| 22 | 22 | ||
| 23 | + | ||
| 23 | 24 | ||
| 24 | 25 | ||
| 25 | 26 | ||
| @@ -57,6 +58,7 @@ template <bool dynamic> | |||
| 57 | class DumpST : public ::testing::Test { | 58 | class DumpST : public ::testing::Test { |
| 58 | public: | 59 | public: |
| 59 | static void SetUpTestSuite() { | 60 | static void SetUpTestSuite() { |
| 61 | + ScopedUnsetLdPreload guard; | ||
| 60 | const std::map<AscendString, AscendString> options = { | 62 | const std::map<AscendString, AscendString> options = { |
| 61 | {OPTION_HOST_ENV_OS, "linux"}, | 63 | {OPTION_HOST_ENV_OS, "linux"}, |
| 62 | {OPTION_HOST_ENV_CPU, "x86_64"}, | 64 | {OPTION_HOST_ENV_CPU, "x86_64"}, |
| @@ -14,6 +14,20 @@ | |||
| 14 | using namespace std; | 14 | using namespace std; |
| 15 | using namespace ge; | 15 | using namespace ge; |
| 16 | 16 | ||
| 17 | +extern "C" const char *__lsan_default_suppressions() { | ||
| 18 | + return "leak:_PyObject_Malloc\n" | ||
| 19 | + "leak:_PyObject_Realloc\n" | ||
| 20 | + "leak:PyType_GenericAlloc\n" | ||
| 21 | + "leak:PyType_Ready\n" | ||
| 22 | + "leak:PyObject_GC_New\n" | ||
| 23 | + "leak:PyObject_GC_NewVar\n" | ||
| 24 | + "leak:_PyObject_GC_Malloc\n" | ||
| 25 | + "leak:PyUnicode_New\n" | ||
| 26 | + "leak:Py_InitializeEx\n" | ||
| 27 | + "leak:PyImport_ImportModuleLevelObject\n" | ||
| 28 | + "leak:libpython\n"; | ||
| 29 | +} | ||
| 30 | + | ||
| 17 | int main(int argc, char **argv) { | 31 | int main(int argc, char **argv) { |
| 18 | gert::LoadDefaultSpaceRegistry(); | 32 | gert::LoadDefaultSpaceRegistry(); |
| 19 | testing::InitGoogleTest(&argc, argv); | 33 | testing::InitGoogleTest(&argc, argv); |
| @@ -20,6 +20,25 @@ | |||
| 20 | using namespace std; | 20 | using namespace std; |
| 21 | using namespace ge; | 21 | using namespace ge; |
| 22 | 22 | ||
| 23 | +extern "C" const char *__lsan_default_suppressions() { | ||
| 24 | + return "leak:_PyObject_Malloc\n" | ||
| 25 | + "leak:_PyObject_Realloc\n" | ||
| 26 | + "leak:PyType_GenericAlloc\n" | ||
| 27 | + "leak:PyType_Ready\n" | ||
| 28 | + "leak:PyObject_GC_New\n" | ||
| 29 | + "leak:PyObject_GC_NewVar\n" | ||
| 30 | + "leak:_PyObject_GC_Malloc\n" | ||
| 31 | + "leak:PyUnicode_New\n" | ||
| 32 | + "leak:Py_InitializeEx\n" | ||
| 33 | + "leak:PyImport_ImportModuleLevelObject\n" | ||
| 34 | + "leak:PyThread_allocate_lock\n" | ||
| 35 | + "leak:pybind11::detail::make_static_property_type\n" | ||
| 36 | + "leak:pybind11::detail::make_object_base_type\n" | ||
| 37 | + "leak:pybind11::detail::get_internals\n" | ||
| 38 | + "leak:pybind11::detail::get_local_internals\n" | ||
| 39 | + "leak:libpython\n"; | ||
| 40 | +} | ||
| 41 | + | ||
| 23 | int main(int argc, char **argv) { | 42 | int main(int argc, char **argv) { |
| 24 | setenv("GE_PROFILING_TO_STD_OUT", "1", true); | 43 | setenv("GE_PROFILING_TO_STD_OUT", "1", true); |
| 25 | // Init running dir env | 44 | // Init running dir env |
| @@ -384,6 +384,7 @@ set(BASE_COMMON_TEST_FILES | |||
| 384 | "common/properties_manager_unittest.cc" | 384 | "common/properties_manager_unittest.cc" |
| 385 | "common/kernel_store_unittest.cc" | 385 | "common/kernel_store_unittest.cc" |
| 386 | "common/ge_auth_file_saver_unittest.cc" | 386 | "common/ge_auth_file_saver_unittest.cc" |
| 387 | + "common/ge_python_runtime_unittest.cc" | ||
| 387 | "common/tbe_plugin_manager_unittest.cc" | 388 | "common/tbe_plugin_manager_unittest.cc" |
| 388 | "common/file_constant_unittest.cc" | 389 | "common/file_constant_unittest.cc" |
| 389 | "common/graph_compile_summary_impl_unittest.cc" | 390 | "common/graph_compile_summary_impl_unittest.cc" |
| @@ -713,6 +714,8 @@ target_include_directories(ut_libge_multiparts_utest PRIVATE | |||
| 713 | target_link_options(ut_libge_multiparts_utest PRIVATE | 714 | target_link_options(ut_libge_multiparts_utest PRIVATE |
| 714 | -Wl,--disable-new-dtags | 715 | -Wl,--disable-new-dtags |
| 715 | -Wl,-rpath,${CMAKE_CURRENT_LIST_DIR} | 716 | -Wl,-rpath,${CMAKE_CURRENT_LIST_DIR} |
| 717 | + -rdynamic | ||
| 718 | + -Wl,-Bsymbolic | ||
| 716 | ) | 719 | ) |
| 717 | 720 | ||
| 718 | target_link_libraries(ut_libge_multiparts_utest | 721 | target_link_libraries(ut_libge_multiparts_utest |
| @@ -924,6 +927,8 @@ target_link_options(ut_libge_others_utest PRIVATE | |||
| 924 | -Wl,--disable-new-dtags | 927 | -Wl,--disable-new-dtags |
| 925 | -Wl,-rpath,${CMAKE_CURRENT_LIST_DIR} | 928 | -Wl,-rpath,${CMAKE_CURRENT_LIST_DIR} |
| 926 | -Wl,-rpath,${CMAKE_BINARY_DIR}/compiler | 929 | -Wl,-rpath,${CMAKE_BINARY_DIR}/compiler |
| 930 | + -rdynamic | ||
| 931 | + -Wl,-Bsymbolic | ||
| 927 | ) | 932 | ) |
| 928 | 933 | ||
| 929 | target_link_libraries(ut_libge_others_utest | 934 | target_link_libraries(ut_libge_others_utest |
| @@ -979,6 +984,7 @@ target_link_libraries(ut_python_pass_artifact_selector_utest | |||
| 979 | ge_common_base | 984 | ge_common_base |
| 980 | opp_registry | 985 | opp_registry |
| 981 | register | 986 | register |
| 987 | + unified_dlog | ||
| 982 | ) | 988 | ) |
| 983 | 989 | ||
| 984 | target_link_options(ut_python_pass_artifact_selector_utest PRIVATE | 990 | target_link_options(ut_python_pass_artifact_selector_utest PRIVATE |
| @@ -1167,6 +1173,8 @@ target_include_directories(ut_libge_label_maker_utest PRIVATE | |||
| 1167 | target_link_options(ut_libge_label_maker_utest PRIVATE | 1173 | target_link_options(ut_libge_label_maker_utest PRIVATE |
| 1168 | -Wl,--disable-new-dtags | 1174 | -Wl,--disable-new-dtags |
| 1169 | -Wl,-rpath,${CMAKE_CURRENT_LIST_DIR} | 1175 | -Wl,-rpath,${CMAKE_CURRENT_LIST_DIR} |
| 1176 | + -rdynamic | ||
| 1177 | + -Wl,-Bsymbolic | ||
| 1170 | ) | 1178 | ) |
| 1171 | 1179 | ||
| 1172 | # TODO:ge_ut_common need shrink; it has too many depends so | 1180 | # TODO:ge_ut_common need shrink; it has too many depends so |
| @@ -0,0 +1,215 @@ | |||
| 1 | +/** | ||
| 2 | + * Copyright (c) 2026 Huawei Technologies Co., Ltd. | ||
| 3 | + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of | ||
| 4 | + * CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 5 | + * Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 6 | + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 7 | + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 8 | + * See LICENSE in the root of the software repository for the full text of the License. | ||
| 9 | + */ | ||
| 10 | + | ||
| 11 | + | ||
| 12 | + | ||
| 13 | + | ||
| 14 | + | ||
| 15 | + | ||
| 16 | +namespace ge { | ||
| 17 | +namespace { | ||
| 18 | +int g_py_initialized = 0; | ||
| 19 | +int g_threads_initialized = 0; | ||
| 20 | +int g_py_initialize_count = 0; | ||
| 21 | +int g_py_finalize_count = 0; | ||
| 22 | +int g_py_eval_init_threads_count = 0; | ||
| 23 | +int g_py_eval_save_thread_count = 0; | ||
| 24 | +int g_py_eval_restore_thread_count = 0; | ||
| 25 | +int g_py_gil_state_check = 0; | ||
| 26 | + | ||
| 27 | +int FakePyIsInitialized() { | ||
| 28 | + return g_py_initialized; | ||
| 29 | +} | ||
| 30 | + | ||
| 31 | +void FakePyInitialize() { | ||
| 32 | + ++g_py_initialize_count; | ||
| 33 | + g_py_initialized = 1; | ||
| 34 | +} | ||
| 35 | + | ||
| 36 | +void FakePyFinalize() { | ||
| 37 | + ++g_py_finalize_count; | ||
| 38 | + g_py_initialized = 0; | ||
| 39 | +} | ||
| 40 | + | ||
| 41 | +const char *FakePyGetVersion() { | ||
| 42 | + return "3.13.0"; | ||
| 43 | +} | ||
| 44 | + | ||
| 45 | +void FakePyEvalInitThreads() { | ||
| 46 | + ++g_py_eval_init_threads_count; | ||
| 47 | + g_threads_initialized = 1; | ||
| 48 | +} | ||
| 49 | + | ||
| 50 | +int FakePyEvalThreadsInitialized() { | ||
| 51 | + return g_threads_initialized; | ||
| 52 | +} | ||
| 53 | + | ||
| 54 | +void *FakePyEvalSaveThread() { | ||
| 55 | + ++g_py_eval_save_thread_count; | ||
| 56 | + return reinterpret_cast<void *>(0x100); | ||
| 57 | +} | ||
| 58 | + | ||
| 59 | +void FakePyEvalRestoreThread(void *thread_state) { | ||
| 60 | + (void)thread_state; | ||
| 61 | + ++g_py_eval_restore_thread_count; | ||
| 62 | +} | ||
| 63 | + | ||
| 64 | +int FakePyGILStateCheck() { | ||
| 65 | + return g_py_gil_state_check; | ||
| 66 | +} | ||
| 67 | + | ||
| 68 | +extern "C" int Py_IsInitialized() { | ||
| 69 | + return FakePyIsInitialized(); | ||
| 70 | +} | ||
| 71 | + | ||
| 72 | +extern "C" void Py_Initialize() { | ||
| 73 | + FakePyInitialize(); | ||
| 74 | +} | ||
| 75 | + | ||
| 76 | +extern "C" void Py_Finalize() { | ||
| 77 | + FakePyFinalize(); | ||
| 78 | +} | ||
| 79 | + | ||
| 80 | +extern "C" const char *Py_GetVersion() { | ||
| 81 | + return FakePyGetVersion(); | ||
| 82 | +} | ||
| 83 | + | ||
| 84 | +extern "C" void PyEval_InitThreads() { | ||
| 85 | + FakePyEvalInitThreads(); | ||
| 86 | +} | ||
| 87 | + | ||
| 88 | +extern "C" int PyEval_ThreadsInitialized() { | ||
| 89 | + return FakePyEvalThreadsInitialized(); | ||
| 90 | +} | ||
| 91 | + | ||
| 92 | +extern "C" void *PyEval_SaveThread() { | ||
| 93 | + return FakePyEvalSaveThread(); | ||
| 94 | +} | ||
| 95 | + | ||
| 96 | +extern "C" void PyEval_RestoreThread(void *thread_state) { | ||
| 97 | + FakePyEvalRestoreThread(thread_state); | ||
| 98 | +} | ||
| 99 | + | ||
| 100 | +extern "C" int PyGILState_Check() { | ||
| 101 | + return FakePyGILStateCheck(); | ||
| 102 | +} | ||
| 103 | + | ||
| 104 | +void ResetFakePythonState() { | ||
| 105 | + g_py_initialized = 0; | ||
| 106 | + g_threads_initialized = 0; | ||
| 107 | + g_py_initialize_count = 0; | ||
| 108 | + g_py_finalize_count = 0; | ||
| 109 | + g_py_eval_init_threads_count = 0; | ||
| 110 | + g_py_eval_save_thread_count = 0; | ||
| 111 | + g_py_eval_restore_thread_count = 0; | ||
| 112 | + g_py_gil_state_check = 0; | ||
| 113 | +} | ||
| 114 | + | ||
| 115 | +class UtestGePythonRuntimeManager : public testing::Test { | ||
| 116 | + protected: | ||
| 117 | + void SetUp() override { | ||
| 118 | + auto &manager = GePythonRuntimeManager::Instance(); | ||
| 119 | + (void)manager.ShutdownProcess(); | ||
| 120 | + ResetFakePythonState(); | ||
| 121 | + g_py_gil_state_check = 1; | ||
| 122 | + } | ||
| 123 | + | ||
| 124 | + void TearDown() override { | ||
| 125 | + auto &manager = GePythonRuntimeManager::Instance(); | ||
| 126 | + (void)manager.ShutdownProcess(); | ||
| 127 | + ResetFakePythonState(); | ||
| 128 | + } | ||
| 129 | +}; | ||
| 130 | + | ||
| 131 | +TEST_F(UtestGePythonRuntimeManager, EnsureReadyAttachesExistingInterpreter) { | ||
| 132 | + g_py_initialized = 1; | ||
| 133 | + | ||
| 134 | + auto &manager = GePythonRuntimeManager::Instance(); | ||
| 135 | + EXPECT_EQ(manager.EnsureReady(), SUCCESS); | ||
| 136 | + | ||
| 137 | + EXPECT_EQ(g_py_initialize_count, 0); | ||
| 138 | + EXPECT_EQ(g_py_eval_init_threads_count, 0); | ||
| 139 | + EXPECT_EQ(g_py_eval_save_thread_count, 0); | ||
| 140 | + | ||
| 141 | + EXPECT_EQ(manager.ShutdownProcess(), SUCCESS); | ||
| 142 | + EXPECT_EQ(g_py_finalize_count, 0); | ||
| 143 | + EXPECT_EQ(g_py_initialized, 1); | ||
| 144 | +} | ||
| 145 | + | ||
| 146 | +TEST_F(UtestGePythonRuntimeManager, EnsureReadyInitializesOwnedInterpreterAndShutdownFinalizes) { | ||
| 147 | + auto &manager = GePythonRuntimeManager::Instance(); | ||
| 148 | + EXPECT_EQ(manager.EnsureReady(), SUCCESS); | ||
| 149 | + | ||
| 150 | + EXPECT_EQ(g_py_initialize_count, 1); | ||
| 151 | + EXPECT_EQ(g_py_eval_init_threads_count, 1); | ||
| 152 | + EXPECT_EQ(g_py_eval_save_thread_count, 1); | ||
| 153 | + | ||
| 154 | + EXPECT_EQ(manager.EnsureReady(), SUCCESS); | ||
| 155 | + EXPECT_EQ(g_py_initialize_count, 1); | ||
| 156 | + EXPECT_EQ(g_py_eval_save_thread_count, 1); | ||
| 157 | + | ||
| 158 | + EXPECT_EQ(manager.ShutdownProcess(), SUCCESS); | ||
| 159 | + EXPECT_EQ(g_py_finalize_count, 1); | ||
| 160 | + EXPECT_EQ(g_py_eval_restore_thread_count, 1); | ||
| 161 | + | ||
| 162 | + EXPECT_EQ(manager.EnsureReady(), SUCCESS); | ||
| 163 | + EXPECT_EQ(g_py_initialize_count, 2); | ||
| 164 | + EXPECT_EQ(g_py_eval_save_thread_count, 2); | ||
| 165 | +} | ||
| 166 | + | ||
| 167 | +TEST_F(UtestGePythonRuntimeManager, FinalizeOwnedInterpreterSkipsWhenNotInitialized) { | ||
| 168 | + auto &manager = GePythonRuntimeManager::Instance(); | ||
| 169 | + EXPECT_EQ(manager.EnsureReady(), SUCCESS); | ||
| 170 | + | ||
| 171 | + g_py_initialized = 0; | ||
| 172 | + | ||
| 173 | + EXPECT_EQ(manager.ShutdownProcess(), SUCCESS); | ||
| 174 | + EXPECT_EQ(g_py_finalize_count, 0); | ||
| 175 | +} | ||
| 176 | + | ||
| 177 | +TEST_F(UtestGePythonRuntimeManager, ShutdownProcessWhenNotReady) { | ||
| 178 | + auto &manager = GePythonRuntimeManager::Instance(); | ||
| 179 | + EXPECT_EQ(manager.ShutdownProcess(), SUCCESS); | ||
| 180 | + EXPECT_EQ(g_py_finalize_count, 0); | ||
| 181 | +} | ||
| 182 | + | ||
| 183 | +TEST_F(UtestGePythonRuntimeManager, EnsureReadyWithThreadsAlreadyInitialized) { | ||
| 184 | + g_py_initialized = 0; | ||
| 185 | + g_threads_initialized = 1; | ||
| 186 | + | ||
| 187 | + auto &manager = GePythonRuntimeManager::Instance(); | ||
| 188 | + EXPECT_EQ(manager.EnsureReady(), SUCCESS); | ||
| 189 | + | ||
| 190 | + EXPECT_EQ(g_py_initialize_count, 1); | ||
| 191 | + EXPECT_EQ(g_py_eval_init_threads_count, 0); | ||
| 192 | +} | ||
| 193 | + | ||
| 194 | +TEST_F(UtestGePythonRuntimeManager, EnsureReadyIdempotentAfterShutdown) { | ||
| 195 | + auto &manager = GePythonRuntimeManager::Instance(); | ||
| 196 | + EXPECT_EQ(manager.EnsureReady(), SUCCESS); | ||
| 197 | + | ||
| 198 | + EXPECT_EQ(manager.ShutdownProcess(), SUCCESS); | ||
| 199 | + | ||
| 200 | + EXPECT_EQ(manager.EnsureReady(), SUCCESS); | ||
| 201 | + EXPECT_EQ(g_py_initialize_count, 2); | ||
| 202 | +} | ||
| 203 | + | ||
| 204 | +TEST_F(UtestGePythonRuntimeManager, FinalizeNonOwnerPreservesInterpreterState) { | ||
| 205 | + g_py_initialized = 1; | ||
| 206 | + | ||
| 207 | + auto &manager = GePythonRuntimeManager::Instance(); | ||
| 208 | + EXPECT_EQ(manager.EnsureReady(), SUCCESS); | ||
| 209 | + | ||
| 210 | + EXPECT_EQ(manager.ShutdownProcess(), SUCCESS); | ||
| 211 | + EXPECT_EQ(g_py_finalize_count, 0); | ||
| 212 | + EXPECT_EQ(g_py_initialized, 1); | ||
| 213 | +} | ||
| 214 | +} // namespace | ||
| 215 | +} // namespace ge | ||
| @@ -390,6 +390,8 @@ void ResetPatternFusionRuntimeSnapshot() { | |||
| 390 | 390 | ||
| 391 | std::atomic<bool> g_force_py_is_initialized_init_failure{false}; | 391 | std::atomic<bool> g_force_py_is_initialized_init_failure{false}; |
| 392 | std::atomic<uint32_t> g_py_is_initialized_call_count{0U}; | 392 | std::atomic<uint32_t> g_py_is_initialized_call_count{0U}; |
| 393 | +std::atomic<bool> g_force_py_get_version_probe_failure{false}; | ||
| 394 | +constexpr const char *kProbeFailurePyVersion = "invalid-runtime"; | ||
| 393 | 395 | ||
| 394 | int QueryRealPyIsInitializedForUt() { | 396 | int QueryRealPyIsInitializedForUt() { |
| 395 | using PyIsInitializedFn = int (*)(); | 397 | using PyIsInitializedFn = int (*)(); |
| @@ -405,6 +407,19 @@ extern "C" int Py_IsInitialized() { | |||
| 405 | return QueryRealPyIsInitializedForUt(); | 407 | return QueryRealPyIsInitializedForUt(); |
| 406 | } | 408 | } |
| 407 | 409 | ||
| 410 | +const char *QueryRealPyGetVersionForUt() { | ||
| 411 | + using PyGetVersionFn = const char *(*)(); | ||
| 412 | + auto *py_get_version = reinterpret_cast<PyGetVersionFn>(dlsym(RTLD_NEXT, "Py_GetVersion")); | ||
| 413 | + return (py_get_version == nullptr) ? "" : py_get_version(); | ||
| 414 | +} | ||
| 415 | + | ||
| 416 | +extern "C" const char *Py_GetVersion() { | ||
| 417 | + if (g_force_py_get_version_probe_failure.load(std::memory_order_acquire)) { | ||
| 418 | + return kProbeFailurePyVersion; | ||
| 419 | + } | ||
| 420 | + return QueryRealPyGetVersionForUt(); | ||
| 421 | +} | ||
| 422 | + | ||
| 408 | class ScopedPyIsInitializedInitFailureForUt { | 423 | class ScopedPyIsInitializedInitFailureForUt { |
| 409 | public: | 424 | public: |
| 410 | ScopedPyIsInitializedInitFailureForUt() { | 425 | ScopedPyIsInitializedInitFailureForUt() { |
| @@ -418,6 +433,21 @@ class ScopedPyIsInitializedInitFailureForUt { | |||
| 418 | } | 433 | } |
| 419 | }; | 434 | }; |
| 420 | 435 | ||
| 436 | +class ScopedPyRuntimeProbeFailureForUt { | ||
| 437 | + public: | ||
| 438 | + ScopedPyRuntimeProbeFailureForUt() { | ||
| 439 | + g_force_py_is_initialized_init_failure.store(true, std::memory_order_release); | ||
| 440 | + g_py_is_initialized_call_count.store(0U, std::memory_order_release); | ||
| 441 | + g_force_py_get_version_probe_failure.store(true, std::memory_order_release); | ||
| 442 | + } | ||
| 443 | + | ||
| 444 | + ~ScopedPyRuntimeProbeFailureForUt() { | ||
| 445 | + g_force_py_get_version_probe_failure.store(false, std::memory_order_release); | ||
| 446 | + g_force_py_is_initialized_init_failure.store(false, std::memory_order_release); | ||
| 447 | + g_py_is_initialized_call_count.store(0U, std::memory_order_release); | ||
| 448 | + } | ||
| 449 | +}; | ||
| 450 | + | ||
| 421 | void *OpenLibraryFailedForUt(const char *path, int flags) { | 451 | void *OpenLibraryFailedForUt(const char *path, int flags) { |
| 422 | (void)path; | 452 | (void)path; |
| 423 | (void)flags; | 453 | (void)flags; |
| @@ -595,6 +625,57 @@ void ForgetNativeModuleForUt() { | |||
| 595 | " sys.modules.pop(name, None)\n"); | 625 | " sys.modules.pop(name, None)\n"); |
| 596 | } | 626 | } |
| 597 | 627 | ||
| 628 | +class ScopedLoadedPythonPassForUt { | ||
| 629 | + public: | ||
| 630 | + ~ScopedLoadedPythonPassForUt() { | ||
| 631 | + UnloadPythonPasses(); | ||
| 632 | + } | ||
| 633 | +}; | ||
| 634 | + | ||
| 635 | +const std::string &GetBridgeConfigDirForUt() { | ||
| 636 | + static ScopedTempDir temp_dir; | ||
| 637 | + static const std::string dir = []() { | ||
| 638 | + temp_dir.MakeDir("ge/passes"); | ||
| 639 | + WriteFile(temp_dir.CreateFilePath("ge/__init__.py"), ""); | ||
| 640 | + WriteFile(temp_dir.CreateFilePath("ge/passes/__init__.py"), | ||
| 641 | + "def clear_registered_passes():\n" | ||
| 642 | + " pass\n"); | ||
| 643 | + WriteFile(temp_dir.CreateFilePath("ge/passes/fake_native.py"), "configured_native_loaded = True\n"); | ||
| 644 | + WriteFile(temp_dir.CreateFilePath("ge/passes/_bridge.py"), | ||
| 645 | + "def clear_pass_holders():\n" | ||
| 646 | + " pass\n" | ||
| 647 | + "\n" | ||
| 648 | + "def clear_loaded_pass_modules():\n" | ||
| 649 | + " pass\n"); | ||
| 650 | + return temp_dir.FilePath(""); | ||
| 651 | + }(); | ||
| 652 | + return dir; | ||
| 653 | +} | ||
| 654 | + | ||
| 655 | +const std::string &GetBridgeNativeModulePathForUt() { | ||
| 656 | + static const std::string path = GetBridgeConfigDirForUt() + "/ge/passes/fake_native.py"; | ||
| 657 | + return path; | ||
| 658 | +} | ||
| 659 | + | ||
| 660 | +void RestoreBridgeConfigForUt(const PythonFusionPassBridgeApi &api) { | ||
| 661 | + if (api.set_artifact_config == nullptr) { | ||
| 662 | + return; | ||
| 663 | + } | ||
| 664 | + | ||
| 665 | + PythonFusionPassBridgeArtifactConfig config = {nullptr, GetBridgeNativeModulePathForUt().c_str()}; | ||
| 666 | + (void)api.set_artifact_config(&config); | ||
| 667 | +} | ||
| 668 | + | ||
| 669 | +void CleanupBridgeStateForUt(const PythonFusionPassBridgeApi &api) { | ||
| 670 | + ScopedEnvVar scoped_python_path("PYTHONPATH", GetBridgeConfigDirForUt()); | ||
| 671 | + ScopedPythonPathForUt scoped_sys_path(GetBridgeConfigDirForUt()); | ||
| 672 | + RestoreBridgeConfigForUt(api); | ||
| 673 | + if (api.reset_bridge_state != nullptr) { | ||
| 674 | + api.reset_bridge_state(); | ||
| 675 | + } | ||
| 676 | + ForgetNativeModuleForUt(); | ||
| 677 | +} | ||
| 678 | + | ||
| 598 | int g_direct_bridge_registered_count = 0; | 679 | int g_direct_bridge_registered_count = 0; |
| 599 | 680 | ||
| 600 | bool RecordPythonPassFromDirectBridge(const PythonPassDescriptor *pass_desc, | 681 | bool RecordPythonPassFromDirectBridge(const PythonPassDescriptor *pass_desc, |
| @@ -709,8 +790,6 @@ class UtestFusionPassExecutor : public testing::Test { | |||
| 709 | void SetUp() override { | 790 | void SetUp() override { |
| 710 | PreparePythonPathForSt(); | 791 | PreparePythonPathForSt(); |
| 711 | (void)unsetenv(kEnvPythonPassPath); | 792 | (void)unsetenv(kEnvPythonPassPath); |
| 712 | - // 逐 case 只清理 Python pass 业务态，避免在同一测试进程内反复 finalize/dlclose。 | ||
| 713 | - UnloadPythonPasses(); | ||
| 714 | PassRegistry::GetInstance().name_2_fusion_pass_regs_.clear(); | 793 | PassRegistry::GetInstance().name_2_fusion_pass_regs_.clear(); |
| 715 | PassRegistry::GetInstance().descriptor_key_2_python_pass_descs_.clear(); | 794 | PassRegistry::GetInstance().descriptor_key_2_python_pass_descs_.clear(); |
| 716 | PassRegistry::GetInstance().pass_name_2_python_pass_create_contexts_.clear(); | 795 | PassRegistry::GetInstance().pass_name_2_python_pass_create_contexts_.clear(); |
| @@ -725,7 +804,6 @@ class UtestFusionPassExecutor : public testing::Test { | |||
| 725 | } | 804 | } |
| 726 | void TearDown() override { | 805 | void TearDown() override { |
| 727 | (void)unsetenv(kEnvPythonPassPath); | 806 | (void)unsetenv(kEnvPythonPassPath); |
| 728 | - UnloadPythonPasses(); | ||
| 729 | PassRegistry::GetInstance().name_2_fusion_pass_regs_.clear(); | 807 | PassRegistry::GetInstance().name_2_fusion_pass_regs_.clear(); |
| 730 | PassRegistry::GetInstance().descriptor_key_2_python_pass_descs_.clear(); | 808 | PassRegistry::GetInstance().descriptor_key_2_python_pass_descs_.clear(); |
| 731 | PassRegistry::GetInstance().pass_name_2_python_pass_create_contexts_.clear(); | 809 | PassRegistry::GetInstance().pass_name_2_python_pass_create_contexts_.clear(); |
| @@ -753,14 +831,17 @@ class UtestFusionPassExecutor : public testing::Test { | |||
| 753 | const std::string new_python_path = std::string(ST_FUSION_PASS_PY_INSTALL_DIR) + ":" + python_path_bak_; | 831 | const std::string new_python_path = std::string(ST_FUSION_PASS_PY_INSTALL_DIR) + ":" + python_path_bak_; |
| 754 | (void)setenv("PYTHONPATH", new_python_path.c_str(), 1); | 832 | (void)setenv("PYTHONPATH", new_python_path.c_str(), 1); |
| 755 | } | 833 | } |
| 834 | + PrependPythonPathForUt(ST_FUSION_PASS_PY_INSTALL_DIR); | ||
| 756 | return; | 835 | return; |
| 757 | } | 836 | } |
| 758 | (void)setenv("PYTHONPATH", ST_FUSION_PASS_PY_INSTALL_DIR, 1); | 837 | (void)setenv("PYTHONPATH", ST_FUSION_PASS_PY_INSTALL_DIR, 1); |
| 838 | + PrependPythonPathForUt(ST_FUSION_PASS_PY_INSTALL_DIR); | ||
| 759 | 839 | ||
| 760 | } | 840 | } |
| 761 | 841 | ||
| 762 | void RestorePythonPathForSt() { | 842 | void RestorePythonPathForSt() { |
| 763 | 843 | ||
| 844 | + RemovePythonPathForUt(ST_FUSION_PASS_PY_INSTALL_DIR); | ||
| 764 | if (has_python_path_bak_) { | 845 | if (has_python_path_bak_) { |
| 765 | (void)setenv("PYTHONPATH", python_path_bak_.c_str(), 1); | 846 | (void)setenv("PYTHONPATH", python_path_bak_.c_str(), 1); |
| 766 | } else { | 847 | } else { |
| @@ -1388,7 +1469,9 @@ TEST_F(UtestFusionPassExecutor, PythonPassBridgeLoader_ProbesPathPythonRuntime) | |||
| 1388 | ASSERT_EQ(chmod(fake_python.c_str(), 0700), 0); | 1469 | ASSERT_EQ(chmod(fake_python.c_str(), 0700), 0); |
| 1389 | 1470 | ||
| 1390 | ScopedEnvVar scoped_path("PATH", temp_dir.FilePath("")); | 1471 | ScopedEnvVar scoped_path("PATH", temp_dir.FilePath("")); |
| 1472 | + ScopedPyRuntimeProbeFailureForUt scoped_probe_failure; | ||
| 1391 | EXPECT_EQ(RegisterPythonPassesFromPlugin(), FAILED); | 1473 | EXPECT_EQ(RegisterPythonPassesFromPlugin(), FAILED); |
| 1474 | + ShutdownPythonPassesForProcess(); | ||
| 1392 | } | 1475 | } |
| 1393 | 1476 | ||
| 1394 | TEST_F(UtestFusionPassExecutor, PythonPassBridgeLoader_SkipsBrokenPrebuiltCandidate) { | 1477 | TEST_F(UtestFusionPassExecutor, PythonPassBridgeLoader_SkipsBrokenPrebuiltCandidate) { |
| @@ -1483,7 +1566,7 @@ TEST_F(UtestFusionPassExecutor, PythonPassBridgeCApi_ConfiguredNativeModulePathF | |||
| 1483 | g_direct_bridge_registered_count = 0; | 1566 | g_direct_bridge_registered_count = 0; |
| 1484 | EXPECT_EQ(api->register_passes(®istrar), FAILED); | 1567 | EXPECT_EQ(api->register_passes(®istrar), FAILED); |
| 1485 | EXPECT_EQ(g_direct_bridge_registered_count, 0); | 1568 | EXPECT_EQ(g_direct_bridge_registered_count, 0); |
| 1486 | - ForgetNativeModuleForUt(); | 1569 | + CleanupBridgeStateForUt(*api); |
| 1487 | } | 1570 | } |
| 1488 | 1571 | ||
| 1489 | TEST_F(UtestFusionPassExecutor, PythonPassPybindBridge_InterpreterInitializationFailed) { | 1572 | TEST_F(UtestFusionPassExecutor, PythonPassPybindBridge_InterpreterInitializationFailed) { |
| @@ -1536,6 +1619,7 @@ TEST_F(UtestFusionPassExecutor, PythonPassPybindBridge_InterpreterInitialization | |||
| 1536 | api->reset_bridge_state(); | 1619 | api->reset_bridge_state(); |
| 1537 | ForgetNativeModuleForUt(); | 1620 | ForgetNativeModuleForUt(); |
| 1538 | EXPECT_EQ(g_direct_bridge_registered_count, 0); | 1621 | EXPECT_EQ(g_direct_bridge_registered_count, 0); |
| 1622 | + CleanupBridgeStateForUt(*api); | ||
| 1539 | } | 1623 | } |
| 1540 | 1624 | ||
| 1541 | TEST_F(UtestFusionPassExecutor, PythonPassBridgeCApi_LoadsConfiguredNativeModulePath) { | 1625 | TEST_F(UtestFusionPassExecutor, PythonPassBridgeCApi_LoadsConfiguredNativeModulePath) { |
| @@ -1593,7 +1677,7 @@ TEST_F(UtestFusionPassExecutor, PythonPassBridgeCApi_LoadsConfiguredNativeModule | |||
| 1593 | 1677 | ||
| 1594 | api->reset_bridge_state(); | 1678 | api->reset_bridge_state(); |
| 1595 | EXPECT_EQ(api->set_artifact_config(&empty_config), SUCCESS); | 1679 | EXPECT_EQ(api->set_artifact_config(&empty_config), SUCCESS); |
| 1596 | - ForgetNativeModuleForUt(); | 1680 | + CleanupBridgeStateForUt(*api); |
| 1597 | } | 1681 | } |
| 1598 | 1682 | ||
| 1599 | TEST_F(UtestFusionPassExecutor, PythonFusionBasePass_PybindBridge_RunSuccess) { | 1683 | TEST_F(UtestFusionPassExecutor, PythonFusionBasePass_PybindBridge_RunSuccess) { |
| @@ -1603,6 +1687,7 @@ TEST_F(UtestFusionPassExecutor, PythonFusionBasePass_PybindBridge_RunSuccess) { | |||
| 1603 | 1687 | ||
| 1604 | ScopedEnvVar scoped_py_pass_path(kEnvPythonPassPath, GetSharedPybindPassFilePath()); | 1688 | ScopedEnvVar scoped_py_pass_path(kEnvPythonPassPath, GetSharedPybindPassFilePath()); |
| 1605 | ASSERT_EQ(RegisterPythonPassesFromPlugin(), SUCCESS); | 1689 | ASSERT_EQ(RegisterPythonPassesFromPlugin(), SUCCESS); |
| 1690 | + ScopedLoadedPythonPassForUt loaded_python_pass; | ||
| 1606 | 1691 | ||
| 1607 | auto target_compute_graph = gert::ShareGraph::BuildSingleNodeGraph(); | 1692 | auto target_compute_graph = gert::ShareGraph::BuildSingleNodeGraph(); |
| 1608 | const auto expected_graph_name = target_compute_graph->GetName(); | 1693 | const auto expected_graph_name = target_compute_graph->GetName(); |
| @@ -1616,6 +1701,7 @@ TEST_F(UtestFusionPassExecutor, PythonFusionBasePass_PybindBridge_RunFailedOnPyt | |||
| 1616 | EnsureSharedPybindPassFile(); | 1701 | EnsureSharedPybindPassFile(); |
| 1617 | ScopedEnvVar scoped_py_pass_path(kEnvPythonPassPath, GetSharedPybindPassFilePath()); | 1702 | ScopedEnvVar scoped_py_pass_path(kEnvPythonPassPath, GetSharedPybindPassFilePath()); |
| 1618 | ASSERT_EQ(RegisterPythonPassesFromPlugin(), SUCCESS); | 1703 | ASSERT_EQ(RegisterPythonPassesFromPlugin(), SUCCESS); |
| 1704 | + ScopedLoadedPythonPassForUt loaded_python_pass; | ||
| 1619 | 1705 | ||
| 1620 | auto target_compute_graph = gert::ShareGraph::BuildSingleNodeGraph(); | 1706 | auto target_compute_graph = gert::ShareGraph::BuildSingleNodeGraph(); |
| 1621 | FusionPassExecutor pass_executor; | 1707 | FusionPassExecutor pass_executor; |
| @@ -1629,6 +1715,7 @@ TEST_F(UtestFusionPassExecutor, PythonPatternFusionPass_PybindBridge_RunSuccess) | |||
| 1629 | 1715 | ||
| 1630 | ScopedEnvVar scoped_py_pass_path(kEnvPythonPassPath, GetSharedPybindPatternPassFilePath()); | 1716 | ScopedEnvVar scoped_py_pass_path(kEnvPythonPassPath, GetSharedPybindPatternPassFilePath()); |
| 1631 | ASSERT_EQ(RegisterPythonPassesFromPlugin(), SUCCESS); | 1717 | ASSERT_EQ(RegisterPythonPassesFromPlugin(), SUCCESS); |
| 1718 | + ScopedLoadedPythonPassForUt loaded_python_pass; | ||
| 1632 | 1719 | ||
| 1633 | auto target_graph = ge::es::EsGraphBuilder("python_pattern_target"); | 1720 | auto target_graph = ge::es::EsGraphBuilder("python_pattern_target"); |
| 1634 | auto *esb_graph = target_graph.GetCGraphBuilder(); | 1721 | auto *esb_graph = target_graph.GetCGraphBuilder(); |
| @@ -1666,6 +1753,7 @@ TEST_F(UtestFusionPassExecutor, PythonPatternFusionPass_PybindBridge_MatcherConf | |||
| 1666 | 1753 | ||
| 1667 | ScopedEnvVar scoped_py_pass_path(kEnvPythonPassPath, GetSharedPybindPatternMatcherConfigPassFilePath()); | 1754 | ScopedEnvVar scoped_py_pass_path(kEnvPythonPassPath, GetSharedPybindPatternMatcherConfigPassFilePath()); |
| 1668 | ASSERT_EQ(RegisterPythonPassesFromPlugin(), SUCCESS); | 1755 | ASSERT_EQ(RegisterPythonPassesFromPlugin(), SUCCESS); |
| 1756 | + ScopedLoadedPythonPassForUt loaded_python_pass; | ||
| 1669 | 1757 | ||
| 1670 | auto target_graph = ge::es::EsGraphBuilder("python_pattern_matcher_config_target"); | 1758 | auto target_graph = ge::es::EsGraphBuilder("python_pattern_matcher_config_target"); |
| 1671 | auto *esb_graph = target_graph.GetCGraphBuilder(); | 1759 | auto *esb_graph = target_graph.GetCGraphBuilder(); |
| @@ -1702,6 +1790,7 @@ TEST_F(UtestFusionPassExecutor, PythonDecomposePass_PybindBridge_RunSuccess) { | |||
| 1702 | 1790 | ||
| 1703 | ScopedEnvVar scoped_py_pass_path(kEnvPythonPassPath, GetSharedPybindDecomposePassFilePath()); | 1791 | ScopedEnvVar scoped_py_pass_path(kEnvPythonPassPath, GetSharedPybindDecomposePassFilePath()); |
| 1704 | ASSERT_EQ(RegisterPythonPassesFromPlugin(), SUCCESS); | 1792 | ASSERT_EQ(RegisterPythonPassesFromPlugin(), SUCCESS); |
| 1793 | + ScopedLoadedPythonPassForUt loaded_python_pass; | ||
| 1705 | 1794 | ||
| 1706 | auto target_graph = ge::es::EsGraphBuilder("python_decompose_target"); | 1795 | auto target_graph = ge::es::EsGraphBuilder("python_decompose_target"); |
| 1707 | auto *esb_graph = target_graph.GetCGraphBuilder(); | 1796 | auto *esb_graph = target_graph.GetCGraphBuilder(); |
| @@ -1990,25 +2079,3 @@ TEST_F(UtestFusionPassExecutor, RunPasses_SkipOrphanSubgraph) { | |||
| 1990 | } | 2079 | } |
| 1991 | } // namespace fusion | 2080 | } // namespace fusion |
| 1992 | } // namespace ge | 2081 | } // namespace ge |
| 1993 | - | ||
| 1994 | -// CPython 内部分配器（_PyObject_Malloc / PyThread_allocate_lock）在 Py_Finalize() | ||
| 1995 | -// 后仍有残余内存不被回收，这是 CPython 的已知行为，不是业务代码的泄漏。 | ||
| 1996 | -// pybind11 的内部类型对象（make_static_property_type / make_object_base_type） | ||
| 1997 | -// 在 get_internals() 中创建，在解释器关闭时也不会被释放，这是 pybind11 的已知行为。 | ||
| 1998 | -// _ge_pass_native.so 中的 pybind11 绑定在模块卸载时的少量残余内存是 Python C 扩展的已知行为。 | ||
| 1999 | -// 通过 LSan 抑制规则让 ut_fusion_pass_executor_utest 不因此失败。 | ||
| 2000 | -extern "C" const char *__lsan_default_suppressions() { | ||
| 2001 | - return "leak:_PyObject_Malloc\n" | ||
| 2002 | - "leak:PyThread_allocate_lock\n" | ||
| 2003 | - "leak:_PyObject_Realloc\n" | ||
| 2004 | - "leak:PyType_GenericAlloc\n" | ||
| 2005 | - "leak:PyType_Ready\n" | ||
| 2006 | - "leak:PyObject_GC_New\n" | ||
| 2007 | - "leak:PyObject_GC_NewVar\n" | ||
| 2008 | - "leak:_PyObject_GC_Malloc\n" | ||
| 2009 | - "leak:pybind11::detail::make_static_property_type\n" | ||
| 2010 | - "leak:pybind11::detail::make_object_base_type\n" | ||
| 2011 | - "leak:pybind11::detail::get_internals\n" | ||
| 2012 | - "leak:pybind11::detail::get_local_internals\n" | ||
| 2013 | - "leak:libpython\n"; | ||
| 2014 | -} | ||
| @@ -10,6 +10,7 @@ | |||
| 10 | 10 | ||
| 11 | 11 | ||
| 12 | 12 | ||
| 13 | + | ||
| 13 | 14 | ||
| 14 | 15 | ||
| 15 | 16 | ||
| @@ -44,6 +45,11 @@ const char *const kKernelLibName = "DNN_VM_GE_LOCAL"; | |||
| 44 | class UtestGeGenerator : public testing::Test { | 45 | class UtestGeGenerator : public testing::Test { |
| 45 | protected: | 46 | protected: |
| 46 | void SetUp() { | 47 | void SetUp() { |
| 48 | + const auto env_ptr = getenv("LD_PRELOAD"); | ||
| 49 | + if (env_ptr != nullptr) { | ||
| 50 | + env = env_ptr; | ||
| 51 | + unsetenv("LD_PRELOAD"); | ||
| 52 | + } | ||
| 47 | std::string opp_path = __FILE__; | 53 | std::string opp_path = __FILE__; |
| 48 | opp_path = opp_path.substr(0, opp_path.rfind("/") + 1); | 54 | opp_path = opp_path.substr(0, opp_path.rfind("/") + 1); |
| 49 | mmSetEnv(kEnvName, opp_path.c_str(), 1); | 55 | mmSetEnv(kEnvName, opp_path.c_str(), 1); |
| @@ -81,6 +87,10 @@ class UtestGeGenerator : public testing::Test { | |||
| 81 | system(("rm -rf " + path_so).c_str()); | 87 | system(("rm -rf " + path_so).c_str()); |
| 82 | OperatorFactoryImpl::operator_infershape_funcs_->erase("Data"); | 88 | OperatorFactoryImpl::operator_infershape_funcs_->erase("Data"); |
| 83 | OperatorFactoryImpl::operator_infershape_funcs_->erase("NetOutput"); | 89 | OperatorFactoryImpl::operator_infershape_funcs_->erase("NetOutput"); |
| 90 | + (void)GePythonRuntimeManager::Instance().ShutdownProcess(); | ||
| 91 | + if (!env.empty()) { | ||
| 92 | + setenv("LD_PRELOAD", env.c_str(), 1); | ||
| 93 | + } | ||
| 84 | } | 94 | } |
| 85 | 95 | ||
| 86 | class FakeOpsKernelInfoStore : public OpsKernelInfoStore { | 96 | class FakeOpsKernelInfoStore : public OpsKernelInfoStore { |
| @@ -106,6 +116,8 @@ class UtestGeGenerator : public testing::Test { | |||
| 106 | }; | 116 | }; |
| 107 | }; | 117 | }; |
| 108 | 118 | ||
| 119 | + std::string env; | ||
| 120 | + | ||
| 109 | class FakeOpsKernelBuilder : public OpsKernelBuilder { | 121 | class FakeOpsKernelBuilder : public OpsKernelBuilder { |
| 110 | public: | 122 | public: |
| 111 | FakeOpsKernelBuilder() = default; | 123 | FakeOpsKernelBuilder() = default; |
| @@ -9,7 +9,9 @@ | |||
| 9 | */ | 9 | */ |
| 10 | 10 | ||
| 11 | 11 | ||
| 12 | + | ||
| 12 | 13 | ||
| 14 | + | ||
| 13 | 15 | ||
| 14 | 16 | ||
| 15 | 17 | ||
| @@ -29,12 +31,18 @@ | |||
| 29 | 31 | ||
| 30 | 32 | ||
| 31 | 33 | ||
| 34 | + | ||
| 32 | 35 | ||
| 33 | using namespace testing; | 36 | using namespace testing; |
| 34 | using namespace domi; | 37 | using namespace domi; |
| 35 | 38 | ||
| 36 | namespace ge { | 39 | namespace ge { |
| 37 | namespace { | 40 | namespace { |
| 41 | +static Status GEInitializeWithoutLdPreload(const std::map<AscendString, AscendString> &options) { | ||
| 42 | + ScopedUnsetLdPreload guard; | ||
| 43 | + return GEInitialize(options); | ||
| 44 | +} | ||
| 45 | + | ||
| 38 | class TestOpsKernelInfoStore : public OpsKernelInfoStore { | 46 | class TestOpsKernelInfoStore : public OpsKernelInfoStore { |
| 39 | public: | 47 | public: |
| 40 | TestOpsKernelInfoStore() = default; | 48 | TestOpsKernelInfoStore() = default; |
| @@ -101,7 +109,7 @@ class UtestGraphPassesAtomicAddrCleanPass : public Test { | |||
| 101 | ofs.close(); | 109 | ofs.close(); |
| 102 | rename(json_tmp_path.c_str(), json_path.c_str()); | 110 | rename(json_tmp_path.c_str(), json_path.c_str()); |
| 103 | std::map<AscendString, AscendString> options; | 111 | std::map<AscendString, AscendString> options; |
| 104 | - GEInitialize(options); | 112 | + GEInitializeWithoutLdPreload(options); |
| 105 | { | 113 | { |
| 106 | std::ifstream ifs(json_backup_path); | 114 | std::ifstream ifs(json_backup_path); |
| 107 | if (ifs.is_open()) { | 115 | if (ifs.is_open()) { |
| @@ -824,7 +832,7 @@ TEST_F(UtestGraphPassesAtomicAddrCleanPass, test_ge_init_fail) { | |||
| 824 | AtomicAddrCleanPass atomi_addr_clean_pass; | 832 | AtomicAddrCleanPass atomi_addr_clean_pass; |
| 825 | EXPECT_EQ(atomi_addr_clean_pass.CallCompileOp(node_list), ge::GE_CLI_GE_NOT_INITIALIZED); | 833 | EXPECT_EQ(atomi_addr_clean_pass.CallCompileOp(node_list), ge::GE_CLI_GE_NOT_INITIALIZED); |
| 826 | std::map<AscendString, AscendString> options; | 834 | std::map<AscendString, AscendString> options; |
| 827 | - GEInitialize(options); | 835 | + GEInitializeWithoutLdPreload(options); |
| 828 | } | 836 | } |
| 829 | 837 | ||
| 830 | } // namespace ge | 838 | } // namespace ge |
| @@ -22,11 +22,17 @@ | |||
| 22 | 22 | ||
| 23 | 23 | ||
| 24 | 24 | ||
| 25 | + | ||
| 25 | 26 | ||
| 26 | using namespace std; | 27 | using namespace std; |
| 27 | using namespace ge; | 28 | using namespace ge; |
| 28 | 29 | ||
| 29 | namespace { | 30 | namespace { |
| 31 | +static Status GEInitializeWithoutLdPreload(const std::map<AscendString, AscendString> &options) { | ||
| 32 | + ScopedUnsetLdPreload guard; | ||
| 33 | + return GEInitialize(options); | ||
| 34 | +} | ||
| 35 | + | ||
| 30 | class TestOpsKernelInfoStore : public OpsKernelInfoStore { | 36 | class TestOpsKernelInfoStore : public OpsKernelInfoStore { |
| 31 | public: | 37 | public: |
| 32 | TestOpsKernelInfoStore() = default; | 38 | TestOpsKernelInfoStore() = default; |
| @@ -93,7 +99,7 @@ class UtestCompileNodesPass : public testing::Test { | |||
| 93 | protected: | 99 | protected: |
| 94 | void SetUp() { | 100 | void SetUp() { |
| 95 | std::map<AscendString, AscendString> options; | 101 | std::map<AscendString, AscendString> options; |
| 96 | - GEInitialize(options); | 102 | + GEInitializeWithoutLdPreload(options); |
| 97 | ge::GELib::GetInstance()->OpsKernelManagerObj().ops_kernel_store_.clear(); | 103 | ge::GELib::GetInstance()->OpsKernelManagerObj().ops_kernel_store_.clear(); |
| 98 | } | 104 | } |
| 99 | void TearDown() { | 105 | void TearDown() { |
| @@ -144,7 +150,7 @@ TEST_F(UtestCompileNodesPass, not_init_fail) { | |||
| 144 | EXPECT_EQ(pass_.Run(graph_), GE_CLI_GE_NOT_INITIALIZED); | 150 | EXPECT_EQ(pass_.Run(graph_), GE_CLI_GE_NOT_INITIALIZED); |
| 145 | EXPECT_EQ(graph_->GetDirectNodesSize(), 3); | 151 | EXPECT_EQ(graph_->GetDirectNodesSize(), 3); |
| 146 | std::map<AscendString, AscendString> options; | 152 | std::map<AscendString, AscendString> options; |
| 147 | - GEInitialize(options); | 153 | + GEInitializeWithoutLdPreload(options); |
| 148 | } | 154 | } |
| 149 | 155 | ||
| 150 | TEST_F(UtestCompileNodesPass, no_kernel_fail) { | 156 | TEST_F(UtestCompileNodesPass, no_kernel_fail) { |
| @@ -9,6 +9,7 @@ | |||
| 9 | */ | 9 | */ |
| 10 | 10 | ||
| 11 | 11 | ||
| 12 | + | ||
| 12 | 13 | ||
| 13 | 14 | ||
| 14 | 15 | ||
| @@ -47,6 +48,7 @@ | |||
| 47 | 48 | ||
| 48 | 49 | ||
| 49 | 50 | ||
| 51 | + | ||
| 50 | 52 | ||
| 51 | const string AddNYes = "AddNYes"; | 53 | const string AddNYes = "AddNYes"; |
| 52 | const char *const kEnvName = "ASCEND_OPP_PATH"; | 54 | const char *const kEnvName = "ASCEND_OPP_PATH"; |
| @@ -104,6 +106,12 @@ class UtestIrBuild : public testing::Test { | |||
| 104 | void TearDown() {} | 106 | void TearDown() {} |
| 105 | }; | 107 | }; |
| 106 | 108 | ||
| 109 | +template <typename Options> | ||
| 110 | +static ge::graphStatus AclgrphBuildInitializeWithoutLdPreload(Options &options) { | ||
| 111 | + ScopedUnsetLdPreload guard; | ||
| 112 | + return ge::aclgrphBuildInitialize(options); | ||
| 113 | +} | ||
| 114 | + | ||
| 107 | static std::vector<uint8_t> ReadFileToVector(const std::string &path) { | 115 | static std::vector<uint8_t> ReadFileToVector(const std::string &path) { |
| 108 | std::ifstream file(path, std::ios::binary); | 116 | std::ifstream file(path, std::ios::binary); |
| 109 | if (!file.is_open()) { | 117 | if (!file.is_open()) { |
| @@ -845,7 +853,7 @@ TEST(UtestIrBuild, check_aclgrphBundle) { | |||
| 845 | global_options[ge::OPTION_EXEC_HCCL_FLAG] = "0"; | 853 | global_options[ge::OPTION_EXEC_HCCL_FLAG] = "0"; |
| 846 | global_options[ge::OPTION_HOST_ENV_OS] = "linux"; | 854 | global_options[ge::OPTION_HOST_ENV_OS] = "linux"; |
| 847 | global_options[ge::OPTION_HOST_ENV_CPU] = "x86_64"; | 855 | global_options[ge::OPTION_HOST_ENV_CPU] = "x86_64"; |
| 848 | - ge::aclgrphBuildInitialize(global_options); | 856 | + AclgrphBuildInitializeWithoutLdPreload(global_options); |
| 849 | 857 | ||
| 850 | Graph graph = BuildIrConstGraph1(); | 858 | Graph graph = BuildIrConstGraph1(); |
| 851 | WeightRefreshableGraphs split_graphs; | 859 | WeightRefreshableGraphs split_graphs; |
| @@ -1300,7 +1308,7 @@ TEST(UtestIrBuild, aclgrphBuildInitialize_test) { | |||
| 1300 | global_options[ge::OPTION_EXEC_HCCL_FLAG] = "0"; | 1308 | global_options[ge::OPTION_EXEC_HCCL_FLAG] = "0"; |
| 1301 | global_options[ge::OPTION_HOST_ENV_OS] = "linux"; | 1309 | global_options[ge::OPTION_HOST_ENV_OS] = "linux"; |
| 1302 | global_options[ge::OPTION_HOST_ENV_CPU] = "x86_64"; | 1310 | global_options[ge::OPTION_HOST_ENV_CPU] = "x86_64"; |
| 1303 | - ge::graphStatus ret = ge::aclgrphBuildInitialize(global_options); | 1311 | + ge::graphStatus ret = AclgrphBuildInitializeWithoutLdPreload(global_options); |
| 1304 | ge::aclgrphBuildFinalize(); | 1312 | ge::aclgrphBuildFinalize(); |
| 1305 | EXPECT_EQ(ret, ge::GRAPH_SUCCESS); | 1313 | EXPECT_EQ(ret, ge::GRAPH_SUCCESS); |
| 1306 | 1314 | ||
| @@ -1309,13 +1317,13 @@ TEST(UtestIrBuild, aclgrphBuildInitialize_test) { | |||
| 1309 | global_options1[ge::OPTION_HOST_ENV_OS] = "linux"; | 1317 | global_options1[ge::OPTION_HOST_ENV_OS] = "linux"; |
| 1310 | global_options1[ge::OPTION_HOST_ENV_CPU] = "x86_64"; | 1318 | global_options1[ge::OPTION_HOST_ENV_CPU] = "x86_64"; |
| 1311 | global_options1[ge::OPTION_SCREEN_PRINT_MODE] = "enable"; | 1319 | global_options1[ge::OPTION_SCREEN_PRINT_MODE] = "enable"; |
| 1312 | - ret = ge::aclgrphBuildInitialize(global_options1); | 1320 | + ret = AclgrphBuildInitializeWithoutLdPreload(global_options1); |
| 1313 | ge::aclgrphBuildFinalize(); | 1321 | ge::aclgrphBuildFinalize(); |
| 1314 | EXPECT_EQ(ret, ge::GRAPH_SUCCESS); | 1322 | EXPECT_EQ(ret, ge::GRAPH_SUCCESS); |
| 1315 | 1323 | ||
| 1316 | std::map<AscendString, AscendString> global_options2; | 1324 | std::map<AscendString, AscendString> global_options2; |
| 1317 | global_options1["ge.autoTuneMode"] = "RA"; | 1325 | global_options1["ge.autoTuneMode"] = "RA"; |
| 1318 | - ret = ge::aclgrphBuildInitialize(global_options1); | 1326 | + ret = AclgrphBuildInitializeWithoutLdPreload(global_options1); |
| 1319 | ge::aclgrphBuildFinalize(); | 1327 | ge::aclgrphBuildFinalize(); |
| 1320 | EXPECT_NE(ret, ge::GRAPH_SUCCESS); | 1328 | EXPECT_NE(ret, ge::GRAPH_SUCCESS); |
| 1321 | 1329 | ||
| @@ -1324,7 +1332,7 @@ TEST(UtestIrBuild, aclgrphBuildInitialize_test) { | |||
| 1324 | global_options3["ge.deterministicLevel"] = "1"; | 1332 | global_options3["ge.deterministicLevel"] = "1"; |
| 1325 | global_options3[ge::OPTION_HOST_ENV_OS] = "linux"; | 1333 | global_options3[ge::OPTION_HOST_ENV_OS] = "linux"; |
| 1326 | global_options3[ge::OPTION_HOST_ENV_CPU] = "x86_64"; | 1334 | global_options3[ge::OPTION_HOST_ENV_CPU] = "x86_64"; |
| 1327 | - ret = ge::aclgrphBuildInitialize(global_options3); | 1335 | + ret = AclgrphBuildInitializeWithoutLdPreload(global_options3); |
| 1328 | auto &options = GetMutableGlobalOptions(); | 1336 | auto &options = GetMutableGlobalOptions(); |
| 1329 | auto it = options.find(ge::DETERMINISTIC); | 1337 | auto it = options.find(ge::DETERMINISTIC); |
| 1330 | EXPECT_NE(it, options.end()); | 1338 | EXPECT_NE(it, options.end()); |
| @@ -1343,7 +1351,7 @@ TEST(UtestIrBuild, aclgrphBuildInitialize_test) { | |||
| 1343 | global_options4[ge::OP_PRECISION_MODE] = "op_precision.ini"; | 1351 | global_options4[ge::OP_PRECISION_MODE] = "op_precision.ini"; |
| 1344 | global_options4[ge::OPTION_HOST_ENV_OS] = "linux"; | 1352 | global_options4[ge::OPTION_HOST_ENV_OS] = "linux"; |
| 1345 | global_options4[ge::OPTION_HOST_ENV_CPU] = "x86_64"; | 1353 | global_options4[ge::OPTION_HOST_ENV_CPU] = "x86_64"; |
| 1346 | - ret = ge::aclgrphBuildInitialize(global_options4); | 1354 | + ret = AclgrphBuildInitializeWithoutLdPreload(global_options4); |
| 1347 | options = GetMutableGlobalOptions(); | 1355 | options = GetMutableGlobalOptions(); |
| 1348 | it = options.find(ge::OP_PRECISION_MODE); | 1356 | it = options.find(ge::OP_PRECISION_MODE); |
| 1349 | EXPECT_NE(it, options.end()); | 1357 | EXPECT_NE(it, options.end()); |
| @@ -1355,13 +1363,13 @@ TEST(UtestIrBuild, aclgrphBuildInitialize_test) { | |||
| 1355 | 1363 | ||
| 1356 | std::map<std::string, std::string> global_options5; | 1364 | std::map<std::string, std::string> global_options5; |
| 1357 | global_options5[ge::ALLOW_HF32] = "1"; | 1365 | global_options5[ge::ALLOW_HF32] = "1"; |
| 1358 | - ret = ge::aclgrphBuildInitialize(global_options5); | 1366 | + ret = AclgrphBuildInitializeWithoutLdPreload(global_options5); |
| 1359 | ge::aclgrphBuildFinalize(); | 1367 | ge::aclgrphBuildFinalize(); |
| 1360 | EXPECT_EQ(ret, ge::GRAPH_PARAM_INVALID); | 1368 | EXPECT_EQ(ret, ge::GRAPH_PARAM_INVALID); |
| 1361 | 1369 | ||
| 1362 | std::map<std::string, std::string> global_options6; | 1370 | std::map<std::string, std::string> global_options6; |
| 1363 | global_options6[ge::OPTION_SCREEN_PRINT_MODE] = "0"; | 1371 | global_options6[ge::OPTION_SCREEN_PRINT_MODE] = "0"; |
| 1364 | - ret = ge::aclgrphBuildInitialize(global_options6); | 1372 | + ret = AclgrphBuildInitializeWithoutLdPreload(global_options6); |
| 1365 | ge::aclgrphBuildFinalize(); | 1373 | ge::aclgrphBuildFinalize(); |
| 1366 | EXPECT_EQ(ret, ge::GRAPH_PARAM_INVALID); | 1374 | EXPECT_EQ(ret, ge::GRAPH_PARAM_INVALID); |
| 1367 | } | 1375 | } |
| @@ -1371,7 +1379,7 @@ TEST(UtestIrBuild, aclgrphBuildInitialize_test_fail) { | |||
| 1371 | global_options2["ge.optionInvalid"] = "invalid"; | 1379 | global_options2["ge.optionInvalid"] = "invalid"; |
| 1372 | global_options2[ge::OPTION_HOST_ENV_OS] = "linux"; | 1380 | global_options2[ge::OPTION_HOST_ENV_OS] = "linux"; |
| 1373 | global_options2[ge::OPTION_HOST_ENV_CPU] = "x86_64"; | 1381 | global_options2[ge::OPTION_HOST_ENV_CPU] = "x86_64"; |
| 1374 | - EXPECT_EQ(ge::aclgrphBuildInitialize(global_options2), ge::GRAPH_SUCCESS); | 1382 | + EXPECT_EQ(AclgrphBuildInitializeWithoutLdPreload(global_options2), ge::GRAPH_SUCCESS); |
| 1375 | ge::aclgrphBuildFinalize(); | 1383 | ge::aclgrphBuildFinalize(); |
| 1376 | } | 1384 | } |
| 1377 | 1385 | ||
| @@ -1380,7 +1388,7 @@ TEST(UtestIrBuild, check_compression_optimize_conf_test) { | |||
| 1380 | global_options2[ge::COMPRESSION_OPTIMIZE_CONF] = "0"; | 1388 | global_options2[ge::COMPRESSION_OPTIMIZE_CONF] = "0"; |
| 1381 | global_options2[ge::OPTION_HOST_ENV_OS] = "linux"; | 1389 | global_options2[ge::OPTION_HOST_ENV_OS] = "linux"; |
| 1382 | global_options2[ge::OPTION_HOST_ENV_CPU] = "x86_64"; | 1390 | global_options2[ge::OPTION_HOST_ENV_CPU] = "x86_64"; |
| 1383 | - ge::graphStatus ret = ge::aclgrphBuildInitialize(global_options2); | 1391 | + ge::graphStatus ret = AclgrphBuildInitializeWithoutLdPreload(global_options2); |
| 1384 | ge::aclgrphBuildFinalize(); | 1392 | ge::aclgrphBuildFinalize(); |
| 1385 | EXPECT_EQ(ret, ge::GRAPH_SUCCESS); | 1393 | EXPECT_EQ(ret, ge::GRAPH_SUCCESS); |
| 1386 | } | 1394 | } |
| @@ -1390,7 +1398,7 @@ TEST(UtestIrBuild, check_virtual_type_test_success) { | |||
| 1390 | global_options2[ge::VIRTUAL_TYPE] = "1"; | 1398 | global_options2[ge::VIRTUAL_TYPE] = "1"; |
| 1391 | global_options2[ge::OPTION_HOST_ENV_OS] = "linux"; | 1399 | global_options2[ge::OPTION_HOST_ENV_OS] = "linux"; |
| 1392 | global_options2[ge::OPTION_HOST_ENV_CPU] = "x86_64"; | 1400 | global_options2[ge::OPTION_HOST_ENV_CPU] = "x86_64"; |
| 1393 | - ge::graphStatus ret = ge::aclgrphBuildInitialize(global_options2); | 1401 | + ge::graphStatus ret = AclgrphBuildInitializeWithoutLdPreload(global_options2); |
| 1394 | ge::aclgrphBuildFinalize(); | 1402 | ge::aclgrphBuildFinalize(); |
| 1395 | EXPECT_EQ(ret, ge::GRAPH_SUCCESS); | 1403 | EXPECT_EQ(ret, ge::GRAPH_SUCCESS); |
| 1396 | } | 1404 | } |
| @@ -1400,7 +1408,7 @@ TEST(UtestIrBuild, aclgrphBuildInitializeCheckJitCompileTrue) { | |||
| 1400 | global_options2[ge::VIRTUAL_TYPE] = "1"; | 1408 | global_options2[ge::VIRTUAL_TYPE] = "1"; |
| 1401 | global_options2[ge::OPTION_HOST_ENV_OS] = "linux"; | 1409 | global_options2[ge::OPTION_HOST_ENV_OS] = "linux"; |
| 1402 | global_options2[ge::OPTION_HOST_ENV_CPU] = "x86_64"; | 1410 | global_options2[ge::OPTION_HOST_ENV_CPU] = "x86_64"; |
| 1403 | - ge::graphStatus ret = ge::aclgrphBuildInitialize(global_options2); | 1411 | + ge::graphStatus ret = AclgrphBuildInitializeWithoutLdPreload(global_options2); |
| 1404 | ge::aclgrphBuildFinalize(); | 1412 | ge::aclgrphBuildFinalize(); |
| 1405 | EXPECT_EQ(ret, ge::GRAPH_SUCCESS); | 1413 | EXPECT_EQ(ret, ge::GRAPH_SUCCESS); |
| 1406 | 1414 | ||
| @@ -1821,7 +1829,7 @@ TEST(UtestIrBuild, aclgrphBuildModelOm2UnsupportedGlobalOptionTest) { | |||
| 1821 | {ge::OPTION_HOST_ENV_OS, "linux"}, | 1829 | {ge::OPTION_HOST_ENV_OS, "linux"}, |
| 1822 | {ge::OPTION_HOST_ENV_CPU, "x86_64"}, | 1830 | {ge::OPTION_HOST_ENV_CPU, "x86_64"}, |
| 1823 | }; | 1831 | }; |
| 1824 | - ASSERT_EQ(ge::aclgrphBuildInitialize(global_options), ge::GRAPH_SUCCESS); | 1832 | + ASSERT_EQ(AclgrphBuildInitializeWithoutLdPreload(global_options), ge::GRAPH_SUCCESS); |
| 1825 | GE_MAKE_GUARD(finalize_guard, [] { ge::aclgrphBuildFinalize(); }); | 1833 | GE_MAKE_GUARD(finalize_guard, [] { ge::aclgrphBuildFinalize(); }); |
| 1826 | 1834 | ||
| 1827 | Graph graph = BuildIrGraph1(); | 1835 | Graph graph = BuildIrGraph1(); |
| @@ -1884,7 +1892,7 @@ TEST(UtestIrBuild, check_os_err) { | |||
| 1884 | global_options2[ge::COMPRESSION_OPTIMIZE_CONF] = "0"; | 1892 | global_options2[ge::COMPRESSION_OPTIMIZE_CONF] = "0"; |
| 1885 | global_options2[ge::OPTION_HOST_ENV_OS] = "Linux"; | 1893 | global_options2[ge::OPTION_HOST_ENV_OS] = "Linux"; |
| 1886 | global_options2[ge::OPTION_HOST_ENV_CPU] = "x86_64"; | 1894 | global_options2[ge::OPTION_HOST_ENV_CPU] = "x86_64"; |
| 1887 | - ge::graphStatus ret = ge::aclgrphBuildInitialize(global_options2); | 1895 | + ge::graphStatus ret = AclgrphBuildInitializeWithoutLdPreload(global_options2); |
| 1888 | ge::aclgrphBuildFinalize(); | 1896 | ge::aclgrphBuildFinalize(); |
| 1889 | EXPECT_EQ(ret, ge::GRAPH_PARAM_INVALID); | 1897 | EXPECT_EQ(ret, ge::GRAPH_PARAM_INVALID); |
| 1890 | } | 1898 | } |
| @@ -1894,7 +1902,7 @@ TEST(UtestIrBuild, check_cpu_err) { | |||
| 1894 | global_options2[ge::COMPRESSION_OPTIMIZE_CONF] = "0"; | 1902 | global_options2[ge::COMPRESSION_OPTIMIZE_CONF] = "0"; |
| 1895 | global_options2[ge::OPTION_HOST_ENV_OS] = "linux"; | 1903 | global_options2[ge::OPTION_HOST_ENV_OS] = "linux"; |
| 1896 | global_options2[ge::OPTION_HOST_ENV_CPU] = "aaaaa"; | 1904 | global_options2[ge::OPTION_HOST_ENV_CPU] = "aaaaa"; |
| 1897 | - ge::graphStatus ret = ge::aclgrphBuildInitialize(global_options2); | 1905 | + ge::graphStatus ret = AclgrphBuildInitializeWithoutLdPreload(global_options2); |
| 1898 | ge::aclgrphBuildFinalize(); | 1906 | ge::aclgrphBuildFinalize(); |
| 1899 | EXPECT_EQ(ret, ge::GRAPH_PARAM_INVALID); | 1907 | EXPECT_EQ(ret, ge::GRAPH_PARAM_INVALID); |
| 1900 | } | 1908 | } |
| @@ -2047,7 +2055,7 @@ TEST(UtestIrBuild, ir_build_oo_init) { | |||
| 2047 | {ge::OPTION_HOST_ENV_CPU, "x86_64"}, | 2055 | {ge::OPTION_HOST_ENV_CPU, "x86_64"}, |
| 2048 | {ge::OO_LEVEL, "O1"}, | 2056 | {ge::OO_LEVEL, "O1"}, |
| 2049 | {OO_CONSTANT_FOLDING, "false"}}; | 2057 | {OO_CONSTANT_FOLDING, "false"}}; |
| 2050 | - EXPECT_EQ(ge::aclgrphBuildInitialize(global_options), GRAPH_SUCCESS); | 2058 | + EXPECT_EQ(AclgrphBuildInitializeWithoutLdPreload(global_options), GRAPH_SUCCESS); |
| 2051 | 2059 | ||
| 2052 | const std::map<std::string, std::string> build_options = {{ge::OO_LEVEL, "O1"}, {OO_CONSTANT_FOLDING, "true"}}; | 2060 | const std::map<std::string, std::string> build_options = {{ge::OO_LEVEL, "O1"}, {OO_CONSTANT_FOLDING, "true"}}; |
| 2053 | EXPECT_NE(ge::aclgrphBuildModel(graph_1, build_options, model_1), GRAPH_SUCCESS); | 2061 | EXPECT_NE(ge::aclgrphBuildModel(graph_1, build_options, model_1), GRAPH_SUCCESS); |
| @@ -2070,14 +2078,14 @@ TEST(UtestIrBuild, ir_build_oo_init_param_invalid) { | |||
| 2070 | 2078 | ||
| 2071 | std::map<std::string, std::string> global_options; | 2079 | std::map<std::string, std::string> global_options; |
| 2072 | global_options[OO_LEVEL] = "O4"; | 2080 | global_options[OO_LEVEL] = "O4"; |
| 2073 | - EXPECT_NE(ge::aclgrphBuildInitialize(global_options), GRAPH_SUCCESS); | 2081 | + EXPECT_NE(AclgrphBuildInitializeWithoutLdPreload(global_options), GRAPH_SUCCESS); |
| 2074 | 2082 | ||
| 2075 | global_options[OO_LEVEL] = "O1"; | 2083 | global_options[OO_LEVEL] = "O1"; |
| 2076 | global_options[OO_CONSTANT_FOLDING] = "False"; | 2084 | global_options[OO_CONSTANT_FOLDING] = "False"; |
| 2077 | - EXPECT_NE(ge::aclgrphBuildInitialize(global_options), GRAPH_SUCCESS); | 2085 | + EXPECT_NE(AclgrphBuildInitializeWithoutLdPreload(global_options), GRAPH_SUCCESS); |
| 2078 | 2086 | ||
| 2079 | global_options[OO_CONSTANT_FOLDING] = "0"; | 2087 | global_options[OO_CONSTANT_FOLDING] = "0"; |
| 2080 | - EXPECT_NE(ge::aclgrphBuildInitialize(global_options), GRAPH_SUCCESS); | 2088 | + EXPECT_NE(AclgrphBuildInitializeWithoutLdPreload(global_options), GRAPH_SUCCESS); |
| 2081 | 2089 | ||
| 2082 | std::map<std::string, std::string> build_options; | 2090 | std::map<std::string, std::string> build_options; |
| 2083 | build_options[OO_LEVEL] = "O4"; | 2091 | build_options[OO_LEVEL] = "O4"; |
| @@ -2112,7 +2120,7 @@ TEST(UtestIrBuild, ir_build_export_compile_stat_valid) { | |||
| 2112 | global_options[OPTION_EXPORT_COMPILE_STAT] = "0"; | 2120 | global_options[OPTION_EXPORT_COMPILE_STAT] = "0"; |
| 2113 | global_options[ge::OPTION_HOST_ENV_OS] = "linux"; | 2121 | global_options[ge::OPTION_HOST_ENV_OS] = "linux"; |
| 2114 | global_options[ge::OPTION_HOST_ENV_CPU] = "x86_64"; | 2122 | global_options[ge::OPTION_HOST_ENV_CPU] = "x86_64"; |
| 2115 | - EXPECT_EQ(ge::aclgrphBuildInitialize(global_options), GRAPH_SUCCESS); | 2123 | + EXPECT_EQ(AclgrphBuildInitializeWithoutLdPreload(global_options), GRAPH_SUCCESS); |
| 2116 | EXPECT_NE(ge::aclgrphBuildModel(graph_1, build_options, model_1), GRAPH_SUCCESS); | 2124 | EXPECT_NE(ge::aclgrphBuildModel(graph_1, build_options, model_1), GRAPH_SUCCESS); |
| 2117 | EXPECT_EQ(GetThreadLocalContext().GetOption(OPTION_EXPORT_COMPILE_STAT, opt_value), ge::GRAPH_SUCCESS); | 2125 | EXPECT_EQ(GetThreadLocalContext().GetOption(OPTION_EXPORT_COMPILE_STAT, opt_value), ge::GRAPH_SUCCESS); |
| 2118 | EXPECT_EQ(opt_value, "0"); | 2126 | EXPECT_EQ(opt_value, "0"); |
| @@ -2140,7 +2148,7 @@ TEST(UtestIrBuild, ir_build_export_compile_stat_invalid) { | |||
| 2140 | std::string opt_value("-1"); | 2148 | std::string opt_value("-1"); |
| 2141 | std::map<std::string, std::string> global_options; | 2149 | std::map<std::string, std::string> global_options; |
| 2142 | global_options[OPTION_EXPORT_COMPILE_STAT] = "3"; | 2150 | global_options[OPTION_EXPORT_COMPILE_STAT] = "3"; |
| 2143 | - EXPECT_NE(ge::aclgrphBuildInitialize(global_options), GRAPH_SUCCESS); | 2151 | + EXPECT_NE(AclgrphBuildInitializeWithoutLdPreload(global_options), GRAPH_SUCCESS); |
| 2144 | 2152 | ||
| 2145 | aclgrphBuildFinalize(); | 2153 | aclgrphBuildFinalize(); |
| 2146 | system(("rm -rf " + opp_path).c_str()); | 2154 | system(("rm -rf " + opp_path).c_str()); |
| @@ -259,6 +259,11 @@ REGISTER_LABEL_MAKER(CASE, FakeLabelMaker); | |||
| 259 | class UtestGeApiV2 : public testing::Test { | 259 | class UtestGeApiV2 : public testing::Test { |
| 260 | protected: | 260 | protected: |
| 261 | void SetUp() override { | 261 | void SetUp() override { |
| 262 | + const auto env_ptr = getenv("LD_PRELOAD"); | ||
| 263 | + if (env_ptr != nullptr) { | ||
| 264 | + env = env_ptr; | ||
| 265 | + unsetenv("LD_PRELOAD"); | ||
| 266 | + } | ||
| 262 | OperatorFactoryImpl::RegisterInferShapeFunc("Data", [](Operator &op) { return GRAPH_SUCCESS; }); | 267 | OperatorFactoryImpl::RegisterInferShapeFunc("Data", [](Operator &op) { return GRAPH_SUCCESS; }); |
| 263 | OperatorFactoryImpl::RegisterInferShapeFunc("Add", [](Operator &op) { return GRAPH_SUCCESS; }); | 268 | OperatorFactoryImpl::RegisterInferShapeFunc("Add", [](Operator &op) { return GRAPH_SUCCESS; }); |
| 264 | OperatorFactoryImpl::RegisterInferShapeFunc("NetOutput", [](Operator &op) { return GRAPH_SUCCESS; }); | 269 | OperatorFactoryImpl::RegisterInferShapeFunc("NetOutput", [](Operator &op) { return GRAPH_SUCCESS; }); |
| @@ -273,6 +278,9 @@ class UtestGeApiV2 : public testing::Test { | |||
| 273 | OperatorFactoryImpl::operator_infershape_funcs_->erase("Add"); | 278 | OperatorFactoryImpl::operator_infershape_funcs_->erase("Add"); |
| 274 | OperatorFactoryImpl::operator_infershape_funcs_->erase("NetOutput"); | 279 | OperatorFactoryImpl::operator_infershape_funcs_->erase("NetOutput"); |
| 275 | RuntimeStub::Reset(); | 280 | RuntimeStub::Reset(); |
| 281 | + if (!env.empty()) { | ||
| 282 | + setenv("LD_PRELOAD", env.c_str(), 1); | ||
| 283 | + } | ||
| 276 | } | 284 | } |
| 277 | 285 | ||
| 278 | void CreateSharedLibrary(const std::string &path) { | 286 | void CreateSharedLibrary(const std::string &path) { |
| @@ -288,6 +296,7 @@ class UtestGeApiV2 : public testing::Test { | |||
| 288 | system(cmd.c_str()); | 296 | system(cmd.c_str()); |
| 289 | std::remove((path + ".cpp").c_str()); | 297 | std::remove((path + ".cpp").c_str()); |
| 290 | } | 298 | } |
| 299 | + std::string env; | ||
| 291 | }; | 300 | }; |
| 292 | 301 | ||
| 293 | TEST_F(UtestGeApiV2, run_graph_with_stream) { | 302 | TEST_F(UtestGeApiV2, run_graph_with_stream) { |
| @@ -21,6 +21,25 @@ | |||
| 21 | using namespace std; | 21 | using namespace std; |
| 22 | using namespace ge; | 22 | using namespace ge; |
| 23 | 23 | ||
| 24 | +extern "C" const char *__lsan_default_suppressions() { | ||
| 25 | + return "leak:_PyObject_Malloc\n" | ||
| 26 | + "leak:_PyObject_Realloc\n" | ||
| 27 | + "leak:PyType_GenericAlloc\n" | ||
| 28 | + "leak:PyType_Ready\n" | ||
| 29 | + "leak:PyObject_GC_New\n" | ||
| 30 | + "leak:PyObject_GC_NewVar\n" | ||
| 31 | + "leak:_PyObject_GC_Malloc\n" | ||
| 32 | + "leak:PyUnicode_New\n" | ||
| 33 | + "leak:Py_InitializeEx\n" | ||
| 34 | + "leak:PyImport_ImportModuleLevelObject\n" | ||
| 35 | + "leak:PyThread_allocate_lock\n" | ||
| 36 | + "leak:pybind11::detail::make_static_property_type\n" | ||
| 37 | + "leak:pybind11::detail::make_object_base_type\n" | ||
| 38 | + "leak:pybind11::detail::get_internals\n" | ||
| 39 | + "leak:pybind11::detail::get_local_internals\n" | ||
| 40 | + "leak:libpython\n"; | ||
| 41 | +} | ||
| 42 | + | ||
| 24 | int main(int argc, char **argv) { | 43 | int main(int argc, char **argv) { |
| 25 | // init the logging | 44 | // init the logging |
| 26 | testing::InitGoogleTest(&argc, argv); | 45 | testing::InitGoogleTest(&argc, argv); |
🔵 Low Priority
在
api/atc/main_impl.cc的GenerateModel函数中，回调 lambda（第 1656–1660 行）无条件调用GePythonRuntimeManager::Instance().ShutdownProcess()。触发条件：当外部组件先调用了
GELib::Initialize()（如通过GEInitializeV2链路），此时instance_ptr->InitFlag()为true，GenerateModel跳过EnsureReady()和GELib::Initialize()，直接进入ge_generator.Initialize()。但回调中的ShutdownProcess()仍然会执行，由于manager_is_owner_为true（由外部的首次EnsureReady()设置），Python 解释器会被错误地 finalize。影响：在正常 ATC 工具流中此路径不可达（因为每次
GenerateModel回调都会 finalizeGELib，下一次调用时InitFlag()为false）。但如果GenerateModel被复用或与其他 GE API 混合使用，会导致 Python 解释器被意外终止。变更引入点：第 1659 行新增的
GePythonRuntimeManager::Instance().ShutdownProcess()调用。建议：将回调中的
ShutdownProcess()调用改为条件执行：仅当本函数调用了EnsureReady()时才 shutdown。可将release_python_runtime标志传入回调，或复用已有的 dismissable guard 机制（将 dismissable guard 的回调改为始终有效，移除 callback 中的 ShutdownProcess，通过 dismissable 变量控制）。具体做法：回调中只在release_python_runtime == true时调用ShutdownProcess()，然后移除release_python_runtime = false行，因为 dismissable guard 本身已通过release_python_runtime控制。