已合并
fix: 修复aclop与图模式混跑时自定义Pass资源被提前释放的问题 #4243
why you创建于 8月3日
fix: 修复aclop与图模式混跑时自定义Pass资源被提前释放的问题 #4243
已合并
共 12 个文件变更+186-141
| @@ -2345,7 +2345,7 @@ int32_t main_impl(int32_t argc, char *argv[]) { | |||
| 2345 | GELOGW("[Ensure][PythonRuntime] failed before running ATC, continue initialization, ret[%u].", python_runtime_ret); | 2345 | GELOGW("[Ensure][PythonRuntime] failed before running ATC, continue initialization, ret[%u].", python_runtime_ret); |
| 2346 | } | 2346 | } |
| 2347 | GE_MAKE_GUARD(release_python_resources, []() { | 2347 | GE_MAKE_GUARD(release_python_resources, []() { |
| 2348 | - (void)ge::fusion::ShutdownPassPluginsForProcess(); | 2348 | + (void)ge::fusion::UnloadPassPlugins(); |
| 2349 | (void)ge::custom_op::ShutdownCustomOpsForProcess(); | 2349 | (void)ge::custom_op::ShutdownCustomOpsForProcess(); |
| 2350 | (void)GePythonRuntimeManager::Instance().ShutdownProcess(); | 2350 | (void)GePythonRuntimeManager::Instance().ShutdownProcess(); |
| 2351 | }); | 2351 | }); |
| @@ -263,7 +263,7 @@ static Status GEInitializeImpl(const std::map<std::string, std::string> &options | |||
| 263 | GELOGW("[Ensure][PythonRuntime] failed, continue initialization, ret[%u].", ret); | 263 | GELOGW("[Ensure][PythonRuntime] failed, continue initialization, ret[%u].", ret); |
| 264 | } | 264 | } |
| 265 | GE_DISMISSABLE_GUARD(release_python_resources, ([]() { | 265 | GE_DISMISSABLE_GUARD(release_python_resources, ([]() { |
| 266 | - (void)fusion::ShutdownPassPluginsForProcess(); | 266 | + (void)fusion::UnloadPassPlugins(); |
| 267 | (void)ge::custom_op::ShutdownCustomOpsForProcess(); | 267 | (void)ge::custom_op::ShutdownCustomOpsForProcess(); |
| 268 | (void)GePythonRuntimeManager::Instance().ShutdownProcess(); | 268 | (void)GePythonRuntimeManager::Instance().ShutdownProcess(); |
| 269 | })); | 269 | })); |
| @@ -401,7 +401,7 @@ Status GEFinalizeV2() { | |||
| 401 | ShutDownProfiling(); | 401 | ShutDownProfiling(); |
| 402 | 402 | ||
| 403 | // 这里是 GE 的进程级 finalization,额外负责显式关闭 Python bridge so。 | 403 | // 这里是 GE 的进程级 finalization,额外负责显式关闭 Python bridge so。 |
| 404 | - (void)fusion::ShutdownPassPluginsForProcess(); | 404 | + (void)fusion::UnloadPassPlugins(); |
| 405 | (void)custom_op::ShutdownCustomOpsForProcess(); | 405 | (void)custom_op::ShutdownCustomOpsForProcess(); |
| 406 | // call Finalize | 406 | // call Finalize |
| 407 | (void)GeExecutor::FinalizeEx(); | 407 | (void)GeExecutor::FinalizeEx(); |
| @@ -435,7 +435,7 @@ static graphStatus aclgrphBuildInitializeImpl(std::map<std::string, std::string> | |||
| 435 | GELOGW("[Ensure][PythonRuntime] failed, continue initialization, ret[%u].", python_runtime_ret); | 435 | GELOGW("[Ensure][PythonRuntime] failed, continue initialization, ret[%u].", python_runtime_ret); |
| 436 | } | 436 | } |
| 437 | GE_DISMISSABLE_GUARD(release_python_resources, []() { | 437 | GE_DISMISSABLE_GUARD(release_python_resources, []() { |
| 438 | - (void)fusion::ShutdownPassPluginsForProcess(); | 438 | + (void)fusion::UnloadPassPlugins(); |
| 439 | (void)ge::custom_op::ShutdownCustomOpsForProcess(); | 439 | (void)ge::custom_op::ShutdownCustomOpsForProcess(); |
| 440 | (void)GePythonRuntimeManager::Instance().ShutdownProcess(); | 440 | (void)GePythonRuntimeManager::Instance().ShutdownProcess(); |
| 441 | }); | 441 | }); |
| @@ -482,7 +482,7 @@ graphStatus aclgrphBuildInitialize(std::map<AscendString, AscendString> &global_ | |||
| 482 | 482 | ||
| 483 | void aclgrphBuildFinalize() { | 483 | void aclgrphBuildFinalize() { |
| 484 | // ge_ir_build 生命周期结束时显式关闭 Python bridge so,避免进程退出前长期悬挂。 | 484 | // ge_ir_build 生命周期结束时显式关闭 Python bridge so,避免进程退出前长期悬挂。 |
| 485 | - (void)fusion::ShutdownPassPluginsForProcess(); | 485 | + (void)fusion::UnloadPassPlugins(); |
| 486 | (void)custom_op::ShutdownCustomOpsForProcess(); | 486 | (void)custom_op::ShutdownCustomOpsForProcess(); |
| 487 | if (ge::GELib::GetInstance() != nullptr && ge::GELib::GetInstance()->InitFlag()) { | 487 | if (ge::GELib::GetInstance() != nullptr && ge::GELib::GetInstance()->InitFlag()) { |
| 488 | (void)ge::GELib::GetInstance()->Finalize(); | 488 | (void)ge::GELib::GetInstance()->Finalize(); |
| @@ -428,6 +428,7 @@ class GeGenerator::Impl { | |||
| 428 | bool is_singleop_unregistered_ = false; | 428 | bool is_singleop_unregistered_ = false; |
| 429 | bool is_fuzz_compile_enable_ = false; | 429 | bool is_fuzz_compile_enable_ = false; |
| 430 | bool jit_compile_ = true; | 430 | bool jit_compile_ = true; |
| 431 | + bool pass_plugins_loaded_ = false; | ||
| 431 | std::string build_mode_; | 432 | std::string build_mode_; |
| 432 | std::string build_step_; | 433 | std::string build_step_; |
| 433 | static std::mutex mutex_; | 434 | static std::mutex mutex_; |
| @@ -471,12 +472,18 @@ Status GeGenerator::Initialize(const std::map<std::string, std::string> &options | |||
| 471 | option_tmp.emplace(std::pair<std::string, std::string>(string("ge.opsProtoLibPath"), opsproto_path)); | 472 | option_tmp.emplace(std::pair<std::string, std::string>(string("ge.opsProtoLibPath"), opsproto_path)); |
| 472 | (void)manager->Initialize(option_tmp); | 473 | (void)manager->Initialize(option_tmp); |
| 473 | GE_ASSERT_SUCCESS(fusion::LoadPassPlugins()); | 474 | GE_ASSERT_SUCCESS(fusion::LoadPassPlugins()); |
| 475 | + impl_->pass_plugins_loaded_ = true; | ||
| 476 | + GE_DISMISSABLE_GUARD(release_pass_plugins, [this]() { | ||
| 477 | + impl_->pass_plugins_loaded_ = false; | ||
| 478 | + (void)fusion::UnloadPassPlugins(); | ||
| 479 | + }); | ||
| 474 | 480 | ||
| 475 | ret = impl_->graph_manager_.Initialize(options); | 481 | ret = impl_->graph_manager_.Initialize(options); |
| 476 | if (ret != SUCCESS) { | 482 | if (ret != SUCCESS) { |
| 477 | GELOGE(GE_GENERATOR_GRAPH_MANAGER_INIT_FAILED, "[Call][Initialize] Graph manager initialize failed."); | 483 | GELOGE(GE_GENERATOR_GRAPH_MANAGER_INIT_FAILED, "[Call][Initialize] Graph manager initialize failed."); |
| 478 | return GE_GENERATOR_GRAPH_MANAGER_INIT_FAILED; | 484 | return GE_GENERATOR_GRAPH_MANAGER_INIT_FAILED; |
| 479 | } | 485 | } |
| 486 | + GE_DISMISS_GUARD(release_pass_plugins); | ||
| 480 | // 将备份的注册信息低优先级merge到当前map | 487 | // 将备份的注册信息低优先级merge到当前map |
| 481 | OperatorFactoryImpl::MergeBackupCreatorsOnce(); | 488 | OperatorFactoryImpl::MergeBackupCreatorsOnce(); |
| 482 | // get build mode | 489 | // get build mode |
| @@ -496,14 +503,17 @@ Status GeGenerator::Finalize() { | |||
| 496 | if (impl_ == nullptr) { | 503 | if (impl_ == nullptr) { |
| 497 | return SUCCESS; | 504 | return SUCCESS; |
| 498 | } | 505 | } |
| 499 | - // GeGenerator::Finalize 对应本轮构建流程结束,这里走进程级 shutdown。 | 506 | + if (impl_->pass_plugins_loaded_) { |
| 500 | - (void)fusion::ShutdownPassPluginsForProcess(); | 507 | + impl_->pass_plugins_loaded_ = false; |
| 508 | + (void)fusion::UnloadPassPlugins(); | ||
| 509 | + } | ||
| 501 | (void)custom_op::ShutdownCustomOpsForProcess(); | 510 | (void)custom_op::ShutdownCustomOpsForProcess(); |
| 502 | Status ret = impl_->graph_manager_.Finalize(); | 511 | Status ret = impl_->graph_manager_.Finalize(); |
| 503 | if (ret != SUCCESS) { | 512 | if (ret != SUCCESS) { |
| 504 | GELOGE(GE_GENERATOR_GRAPH_MANAGER_FINALIZE_FAILED, "[Call][Finalize] Graph manager finalize failed."); | 513 | GELOGE(GE_GENERATOR_GRAPH_MANAGER_FINALIZE_FAILED, "[Call][Finalize] Graph manager finalize failed."); |
| 505 | return GE_GENERATOR_GRAPH_MANAGER_FINALIZE_FAILED; | 514 | return GE_GENERATOR_GRAPH_MANAGER_FINALIZE_FAILED; |
| 506 | } | 515 | } |
| 516 | + impl_ = nullptr; | ||
| 507 | return SUCCESS; | 517 | return SUCCESS; |
| 508 | } | 518 | } |
| 509 | 519 | ||
| @@ -36,79 +36,58 @@ class PassPluginLoader { | |||
| 36 | 36 | ||
| 37 | Status Load() { | 37 | Status Load() { |
| 38 | std::lock_guard<std::mutex> lock(mutex_); | 38 | std::lock_guard<std::mutex> lock(mutex_); |
| 39 | - | 39 | + if (active_users_ == 0U) { |
| 40 | - if (!cpp_pass_loaded_) { | 40 | + if (!cpp_pass_loaded_) { |
| 41 | - const auto ret = CustomPassHelper::Instance().Load(); | 41 | + const auto ret = CustomPassHelper::Instance().Load(); |
| 42 | - if (ret != SUCCESS) { | 42 | + if (ret != SUCCESS) { |
| 43 | - GELOGE(ret, "Load C++ custom pass plugins failed."); | 43 | + GELOGE(ret, "Load C++ custom pass plugins failed."); |
| 44 | - return ret; | 44 | + return ret; |
| 45 | + } | ||
| 46 | + cpp_pass_loaded_ = true; | ||
| 45 | } | 47 | } |
| 46 | - cpp_pass_loaded_ = true; | 48 | + if ((!python_pass_loaded_) && NeedLoadPythonPasses()) { |
| 47 | - } | 49 | + const auto ret = RegisterPythonPassesFromPlugin(); |
| 48 | - | 50 | + if (ret != SUCCESS) { |
| 49 | - if ((!python_pass_loaded_) && NeedLoadPythonPasses()) { | 51 | + GELOGE(ret, "Load Python fusion pass plugins failed."); |
| 50 | - const auto ret = RegisterPythonPassesFromPlugin(); | 52 | + (void)CustomPassHelper::Instance().Unload(); |
| 51 | - if (ret != SUCCESS) { | 53 | + cpp_pass_loaded_ = false; |
| 52 | - GELOGE(ret, "Load Python fusion pass plugins failed."); | 54 | + return ret; |
| 53 | - RollbackPythonPassesLoad(); | 55 | + } |
| 54 | - return ret; | 56 | + python_pass_loaded_ = true; |
| 55 | } | 57 | } |
| 56 | - python_pass_loaded_ = true; | ||
| 57 | } | 58 | } |
| 59 | + active_users_++; | ||
| 58 | return SUCCESS; | 60 | return SUCCESS; |
| 59 | } | 61 | } |
| 60 | 62 | ||
| 61 | Status Unload() { | 63 | Status Unload() { |
| 62 | std::lock_guard<std::mutex> lock(mutex_); | 64 | std::lock_guard<std::mutex> lock(mutex_); |
| 63 | - if (python_pass_loaded_) { | 65 | + if (active_users_ == 0U) { |
| 64 | - UnloadPythonPasses(); | 66 | + GELOGW("UnloadPassPlugins called with no active users, possible reference leak."); |
| 65 | - python_pass_loaded_ = false; | ||
| 66 | - } | ||
| 67 | - | ||
| 68 | - if (!cpp_pass_loaded_) { | ||
| 69 | return SUCCESS; | 67 | return SUCCESS; |
| 70 | } | 68 | } |
| 71 | - | 69 | + active_users_--; |
| 72 | - cpp_pass_loaded_ = false; | 70 | + if (active_users_ == 0U) { |
| 73 | - return CustomPassHelper::Instance().Unload(); | 71 | + if (python_pass_loaded_) { |
| 74 | - } | 72 | + UnloadPythonPasses(); |
| 75 | - | 73 | + python_pass_loaded_ = false; |
| 76 | - Status ShutdownForProcess() { | 74 | + } |
| 77 | - std::lock_guard<std::mutex> lock(mutex_); | 75 | + if (cpp_pass_loaded_) { |
| 78 | - | 76 | + cpp_pass_loaded_ = false; |
| 79 | - // 业务级状态 reset 必须在 shutdown_done_ 检查之前执行, | 77 | + (void)CustomPassHelper::Instance().Unload(); |
| 80 | - // 否则串行 GEInitialize/GEFinalize 场景下 cpp_pass_loaded_ 不会被 reset, | 78 | + } |
| 81 | - // 后续 Load() 会因 cpp_pass_loaded_=true 跳过实际加载。 | 79 | + if (!shutdown_done_) { |
| 82 | - if (python_pass_loaded_) { | 80 | + shutdown_done_ = true; |
| 83 | - GELOGI("[PythonPass] ShutdownForProcess unloading python passes."); | 81 | + ShutdownPythonPassesForProcess(); |
| 84 | - UnloadPythonPasses(); | 82 | + GELOGI("[PythonPass] ShutdownPythonPassesForProcess done."); |
| 85 | - python_pass_loaded_ = false; | 83 | + } |
| 86 | } | 84 | } |
| 87 | - if (cpp_pass_loaded_) { | ||
| 88 | - cpp_pass_loaded_ = false; | ||
| 89 | - (void)CustomPassHelper::Instance().Unload(); | ||
| 90 | - } | ||
| 91 | - | ||
| 92 | - // 进程级资源(Python 解释器、bridge so)只清理一次 | ||
| 93 | - if (shutdown_done_) { | ||
| 94 | - GELOGI("[PythonPass] ShutdownForProcess process-level cleanup already done, skip."); | ||
| 95 | - return SUCCESS; | ||
| 96 | - } | ||
| 97 | - shutdown_done_ = true; | ||
| 98 | - | ||
| 99 | - // 进程级 shutdown 额外负责关闭 bridge so。 | ||
| 100 | - ShutdownPythonPassesForProcess(); | ||
| 101 | - GELOGI("[PythonPass] ShutdownPythonPassesForProcess done."); | ||
| 102 | return SUCCESS; | 85 | return SUCCESS; |
| 103 | } | 86 | } |
| 104 | 87 | ||
| 105 | private: | 88 | private: |
| 106 | - void RollbackPythonPassesLoad() { | ||
| 107 | - UnloadPythonPasses(); | ||
| 108 | - python_pass_loaded_ = false; | ||
| 109 | - } | ||
| 110 | - | ||
| 111 | std::mutex mutex_; | 89 | std::mutex mutex_; |
| 90 | + size_t active_users_{0U}; | ||
| 112 | bool cpp_pass_loaded_{false}; | 91 | bool cpp_pass_loaded_{false}; |
| 113 | bool python_pass_loaded_{false}; | 92 | bool python_pass_loaded_{false}; |
| 114 | bool shutdown_done_{false}; | 93 | bool shutdown_done_{false}; |
| @@ -122,9 +101,5 @@ Status LoadPassPlugins() { | |||
| 122 | Status UnloadPassPlugins() { | 101 | Status UnloadPassPlugins() { |
| 123 | return PassPluginLoader::GetInstance().Unload(); | 102 | return PassPluginLoader::GetInstance().Unload(); |
| 124 | } | 103 | } |
| 125 | - | ||
| 126 | -Status ShutdownPassPluginsForProcess() { | ||
| 127 | - return PassPluginLoader::GetInstance().ShutdownForProcess(); | ||
| 128 | -} | ||
| 129 | } // namespace fusion | 104 | } // namespace fusion |
| 130 | } // namespace ge | 105 | } // namespace ge |
| @@ -17,7 +17,6 @@ namespace ge { | |||
| 17 | namespace fusion { | 17 | namespace fusion { |
| 18 | Status LoadPassPlugins(); | 18 | Status LoadPassPlugins(); |
| 19 | Status UnloadPassPlugins(); | 19 | Status UnloadPassPlugins(); |
| 20 | -Status ShutdownPassPluginsForProcess(); | ||
| 21 | } // namespace fusion | 20 | } // namespace fusion |
| 22 | } // namespace ge | 21 | } // namespace ge |
| 23 | 22 | ||
| @@ -824,53 +824,57 @@ UnloadPassPlugins() | |||
| 824 | 824 | ||
| 825 | Unload does not touch the Python interpreter lifecycle or bridge so handle, ensuring the next `Load()` can directly reuse them. | 825 | Unload does not touch the Python interpreter lifecycle or bridge so handle, ensuring the next `Load()` can directly reuse them. |
| 826 | 826 | ||
| 827 | -##### ShutdownForProcess -- Process-level Shutdown | 827 | +##### Process-level Shutdown |
| 828 | 828 | ||
| 829 | -When the process exits, GE calls `ShutdownPassPluginsForProcess()` to perform complete resource release. Currently there are 3 entry points that can trigger this: | 829 | +When the process exits, GE performs complete resource release through `UnloadPassPlugins()` when the reference count drops to zero. All entry points now uniformly call `UnloadPassPlugins()`: |
| 830 | 830 | ||
| 831 | - `GEFinalizeV2()` -- when the online mode process ends | 831 | - `GEFinalizeV2()` -- when the online mode process ends |
| 832 | - `aclgrphBuildFinalize()` -- when offline compilation ends | 832 | - `aclgrphBuildFinalize()` -- when offline compilation ends |
| 833 | - `GeGenerator::Finalize()` -- when the generator mode ends | 833 | - `GeGenerator::Finalize()` -- when the generator mode ends |
| 834 | +- `atc main_impl` -- when ATC ends | ||
| 834 | 835 | ||
| 835 | -Current implementation chain: | 836 | +`UnloadPassPlugins()` performs process-level cleanup when the reference count reaches zero: |
| 836 | 837 | ||
| 837 | ``` | 838 | ``` |
| 838 | -ShutdownPassPluginsForProcess() | 839 | +UnloadPassPlugins() |
| 839 | - → PassPluginLoader::ShutdownForProcess() [pass_plugin_loader.cc] | 840 | + → PassPluginLoader::Unload() [pass_plugin_loader.cc] |
| 840 | - ├─ One-time guard: if (shutdown_done_) return // Ensure process-level shutdown only executes once | 841 | + ├─ active_users_-- |
| 841 | - ├─ shutdown_done_ = true | 842 | + ├─ if (active_users_ == 0): |
| 842 | - │ | 843 | + │ ├─ if (python_pass_loaded_): |
| 843 | - ├─ if (python_pass_loaded_): | 844 | + │ │ UnloadPythonFusionBasePasses() // Clean up registration state first |
| 844 | - │ UnloadPythonFusionBasePasses() // Clean up registration state first (same as Unload) | 845 | + │ │ → BridgeLoader::Unload() |
| 845 | - │ → BridgeLoader::Unload() | 846 | + │ │ ├─ api_->reset_bridge_state() |
| 846 | - │ ├─ api_->reset_bridge_state() | 847 | + │ │ ├─ ClearPythonFusionBasePassRuntimeRegistry() |
| 847 | - │ ├─ ClearPythonFusionBasePassRuntimeRegistry() | 848 | + │ │ └─ PassRegistry::ClearPythonPasses() |
| 848 | - │ └─ PassRegistry::ClearPythonPasses() | 849 | + │ │ python_pass_loaded_ = false |
| 849 | - │ python_pass_loaded_ = false | 850 | + │ │ |
| 850 | - │ | 851 | + │ ├─ if (cpp_pass_loaded_): |
| 851 | - ├─ ShutdownPythonFusionBasePassesForProcess() // Unconditionally executed | 852 | + │ │ cpp_pass_loaded_ = false |
| 852 | - │ → BridgeLoader::ShutdownForProcess() [bridge_loader.cc] | 853 | + │ │ CustomPassHelper::Unload() // Clean up C++ custom passes |
| 853 | - │ ├─ if (api_ != nullptr): | 854 | + │ │ |
| 854 | - │ │ api_->shutdown_bridge() // Call bridge so exported shutdown | 855 | + │ └─ if (!shutdown_done_): |
| 855 | - │ │ → PybindBridge::Shutdown() [pybind_bridge.cc] | 856 | + │ shutdown_done_ = true |
| 856 | - │ │ ├─ ResetBridgeStateUnlocked() // Clean up Python-side state, release bridge module references and gc.collect() | 857 | + │ ShutdownPythonFusionBasePassesForProcess() // Process-level Python bridge cleanup |
| 857 | - │ │ └─ if (owns_interpreter_): // Only when the interpreter was started by the bridge itself | 858 | + │ → BridgeLoader::ShutdownForProcess() [bridge_loader.cc] |
| 858 | - │ │ py::finalize_interpreter() // Finalize the Python interpreter | 859 | + │ ├─ if (api_ != nullptr): |
| 859 | - │ │ owns_interpreter_ = false | 860 | + │ │ api_->shutdown_bridge() |
| 860 | - │ ├─ api_ = nullptr // Set to null, prevent subsequent calls | 861 | + │ │ → PybindBridge::Shutdown() [pybind_bridge.cc] |
| 861 | - │ ├─ if (handle_ != nullptr): | 862 | + │ │ ├─ ResetBridgeStateUnlocked() |
| 862 | - │ │ dlclose(handle_) // Unload bridge so | 863 | + │ │ └─ if (owns_interpreter_): |
| 863 | - │ │ handle_ = nullptr // Set to null, prevent dlclose duplication | 864 | + │ │ py::finalize_interpreter() |
| 864 | - │ └─ loaded_path_.clear() | 865 | + │ │ owns_interpreter_ = false |
| 865 | - │ | 866 | + │ ├─ api_ = nullptr |
| 866 | - └─ CustomPassHelper::Unload() // Clean up C++ custom passes | 867 | + │ ├─ if (handle_ != nullptr): |
| 868 | + │ │ dlclose(handle_) | ||
| 869 | + │ │ handle_ = nullptr | ||
| 870 | + │ └─ loaded_path_.clear() | ||
| 867 | ``` | 871 | ``` |
| 868 | 872 | ||
| 869 | ##### Idempotency Guarantee | 873 | ##### Idempotency Guarantee |
| 870 | 874 | ||
| 871 | -Since `ShutdownPassPluginsForProcess()` may be called repeatedly from multiple entry points, the entire chain guarantees idempotency through the following guards: | 875 | +Since `UnloadPassPlugins()` may be called repeatedly from multiple entry points, the entire chain guarantees idempotency through the following guards: |
| 872 | 876 | ||
| 873 | -1. **PassPluginLoader layer** -- `shutdown_done_` flag: set to `true` after first execution, subsequent calls directly return `SUCCESS` | 877 | +1. **PassPluginLoader layer** -- `active_users_` reference count: decremented on each call, unloading only occurs when it reaches zero; `shutdown_done_` flag ensures process-level cleanup only executes once |
| 874 | 2. **BridgeLoader layer** -- `api_` / `handle_` null pointer guard: set to `nullptr` after first execution, subsequent calls skip shutdown and dlclose | 878 | 2. **BridgeLoader layer** -- `api_` / `handle_` null pointer guard: set to `nullptr` after first execution, subsequent calls skip shutdown and dlclose |
| 875 | 3. **PybindBridge layer** -- `Py_IsInitialized()` guard: does not enter Python cleanup logic after the interpreter has been finalized; `owns_interpreter_` guard ensures only the self-initialized interpreter is finalized | 879 | 3. **PybindBridge layer** -- `Py_IsInitialized()` guard: does not enter Python cleanup logic after the interpreter has been finalized; `owns_interpreter_` guard ensures only the self-initialized interpreter is finalized |
| 876 | 880 | ||
| @@ -1609,7 +1613,7 @@ Therefore, the shutdown constraints differ under different startup orders: | |||
| 1609 | 1613 | ||
| 1610 | ```text | 1614 | ```text |
| 1611 | TBE initializes Python first: | 1615 | TBE initializes Python first: |
| 1612 | -ShutdownPassPluginsForProcess() | 1616 | +UnloadPassPlugins() (when reference count reaches zero) |
| 1613 | -> reset / clear Python pass holders, modules, registry | 1617 | -> reset / clear Python pass holders, modules, registry |
| 1614 | GELib::Finalize() | 1618 | GELib::Finalize() |
| 1615 | -> TBE / op store finalize | 1619 | -> TBE / op store finalize |
| @@ -1617,7 +1621,7 @@ GELib::Finalize() | |||
| 1617 | -> TBE calls Py_Finalize when it owns the interpreter | 1621 | -> TBE calls Py_Finalize when it owns the interpreter |
| 1618 | 1622 | ||
| 1619 | Python pass bridge initializes Python first: | 1623 | Python pass bridge initializes Python first: |
| 1620 | -ShutdownPassPluginsForProcess() | 1624 | +UnloadPassPlugins() (when reference count reaches zero) |
| 1621 | -> reset / clear Python pass holders, modules, registry | 1625 | -> reset / clear Python pass holders, modules, registry |
| 1622 | -> Must not immediately py::finalize_interpreter unless confirmed TBE and other Python users have not initialized | 1626 | -> Must not immediately py::finalize_interpreter unless confirmed TBE and other Python users have not initialized |
| 1623 | GELib::Finalize() | 1627 | GELib::Finalize() |
| @@ -824,53 +824,57 @@ UnloadPassPlugins() | |||
| 824 | 824 | ||
| 825 | Unload 不触及 Python 解释器生命周期和 bridge so 句柄,确保下一轮 `Load()` 可以直接复用。 | 825 | Unload 不触及 Python 解释器生命周期和 bridge so 句柄,确保下一轮 `Load()` 可以直接复用。 |
| 826 | 826 | ||
| 827 | -##### ShutdownForProcess — 进程级关闭 | 827 | +##### 进程级关闭 |
| 828 | 828 | ||
| 829 | -进程退出时,GE 调用 `ShutdownPassPluginsForProcess()` 执行完整的资源释放。当前有 3 个入口可以触发: | 829 | +进程退出时,GE 通过 `UnloadPassPlugins()` 在引用计数归零时执行完整的资源释放。当前所有入口统一调用 `UnloadPassPlugins()`: |
| 830 | 830 | ||
| 831 | - `GEFinalizeV2()` — 在线模式进程结束时 | 831 | - `GEFinalizeV2()` — 在线模式进程结束时 |
| 832 | - `aclgrphBuildFinalize()` — 离线编译结束时 | 832 | - `aclgrphBuildFinalize()` — 离线编译结束时 |
| 833 | - `GeGenerator::Finalize()` — 生成器模式结束时 | 833 | - `GeGenerator::Finalize()` — 生成器模式结束时 |
| 834 | +- `atc main_impl` — ATC 结束时 | ||
| 834 | 835 | ||
| 835 | -当前实现链路: | 836 | +`UnloadPassPlugins()` 内部在引用计数归零时执行进程级清理: |
| 836 | 837 | ||
| 837 | ``` | 838 | ``` |
| 838 | -ShutdownPassPluginsForProcess() | 839 | +UnloadPassPlugins() |
| 839 | - → PassPluginLoader::ShutdownForProcess() [pass_plugin_loader.cc] | 840 | + → PassPluginLoader::Unload() [pass_plugin_loader.cc] |
| 840 | - ├─ 一次性守卫: if (shutdown_done_) return // 确保进程级 shutdown 只执行一次 | 841 | + ├─ active_users_-- |
| 841 | - ├─ shutdown_done_ = true | 842 | + ├─ if (active_users_ == 0): |
| 842 | - │ | 843 | + │ ├─ if (python_pass_loaded_): |
| 843 | - ├─ if (python_pass_loaded_): | 844 | + │ │ UnloadPythonFusionBasePasses() // 先清理注册态 |
| 844 | - │ UnloadPythonFusionBasePasses() // 先清理注册态(同 Unload) | 845 | + │ │ → BridgeLoader::Unload() |
| 845 | - │ → BridgeLoader::Unload() | 846 | + │ │ ├─ api_->reset_bridge_state() |
| 846 | - │ ├─ api_->reset_bridge_state() | 847 | + │ │ ├─ ClearPythonFusionBasePassRuntimeRegistry() |
| 847 | - │ ├─ ClearPythonFusionBasePassRuntimeRegistry() | 848 | + │ │ └─ PassRegistry::ClearPythonPasses() |
| 848 | - │ └─ PassRegistry::ClearPythonPasses() | 849 | + │ │ python_pass_loaded_ = false |
| 849 | - │ python_pass_loaded_ = false | 850 | + │ │ |
| 850 | - │ | 851 | + │ ├─ if (cpp_pass_loaded_): |
| 851 | - ├─ ShutdownPythonFusionBasePassesForProcess() // 无条件执行 | 852 | + │ │ cpp_pass_loaded_ = false |
| 852 | - │ → BridgeLoader::ShutdownForProcess() [bridge_loader.cc] | 853 | + │ │ CustomPassHelper::Unload() // 清理 C++ 自定义 pass |
| 853 | - │ ├─ if (api_ != nullptr): | 854 | + │ │ |
| 854 | - │ │ api_->shutdown_bridge() // 调用 bridge so 导出的 shutdown | 855 | + │ └─ if (!shutdown_done_): |
| 855 | - │ │ → PybindBridge::Shutdown() [pybind_bridge.cc] | 856 | + │ shutdown_done_ = true |
| 856 | - │ │ ├─ ResetBridgeStateUnlocked() // 清理 Python 侧状态、释放 bridge 模块引用并 gc.collect() | 857 | + │ ShutdownPythonFusionBasePassesForProcess() // 进程级 Python bridge 清理 |
| 857 | - │ │ └─ if (owns_interpreter_): // 仅当解释器由 bridge 自己拉起时 | 858 | + │ → BridgeLoader::ShutdownForProcess() [bridge_loader.cc] |
| 858 | - │ │ py::finalize_interpreter() // 终结 Python 解释器 | 859 | + │ ├─ if (api_ != nullptr): |
| 859 | - │ │ owns_interpreter_ = false | 860 | + │ │ api_->shutdown_bridge() |
| 860 | - │ ├─ api_ = nullptr // 置空,防止后续再调用 | 861 | + │ │ → PybindBridge::Shutdown() [pybind_bridge.cc] |
| 861 | - │ ├─ if (handle_ != nullptr): | 862 | + │ │ ├─ ResetBridgeStateUnlocked() |
| 862 | - │ │ dlclose(handle_) // 卸载 bridge so | 863 | + │ │ └─ if (owns_interpreter_): |
| 863 | - │ │ handle_ = nullptr // 置空,防止 dlclose 重复 | 864 | + │ │ py::finalize_interpreter() |
| 864 | - │ └─ loaded_path_.clear() | 865 | + │ │ owns_interpreter_ = false |
| 865 | - │ | 866 | + │ ├─ api_ = nullptr |
| 866 | - └─ CustomPassHelper::Unload() // 清理 C++ 自定义 pass | 867 | + │ ├─ if (handle_ != nullptr): |
| 868 | + │ │ dlclose(handle_) | ||
| 869 | + │ │ handle_ = nullptr | ||
| 870 | + │ └─ loaded_path_.clear() | ||
| 867 | ``` | 871 | ``` |
| 868 | 872 | ||
| 869 | ##### 幂等性保证 | 873 | ##### 幂等性保证 |
| 870 | 874 | ||
| 871 | -由于 `ShutdownPassPluginsForProcess()` 可能从多个入口被重复调用,整条链路通过以下守卫保证幂等: | 875 | +由于 `UnloadPassPlugins()` 可能从多个入口被重复调用,整条链路通过以下守卫保证幂等: |
| 872 | 876 | ||
| 873 | -1. **PassPluginLoader 层** — `shutdown_done_` 标志:首次执行后置为 `true`,后续调用直接返回 `SUCCESS` | 877 | +1. **PassPluginLoader 层** — `active_users_` 引用计数:每次调用递减,归零时才执行卸载;`shutdown_done_` 标志确保进程级清理只执行一次 |
| 874 | 2. **BridgeLoader 层** — `api_` / `handle_` 空指针守卫:首次执行后置为 `nullptr`,后续调用跳过 shutdown 和 dlclose | 878 | 2. **BridgeLoader 层** — `api_` / `handle_` 空指针守卫:首次执行后置为 `nullptr`,后续调用跳过 shutdown 和 dlclose |
| 875 | 3. **PybindBridge 层** — `Py_IsInitialized()` 守卫:解释器已终结后不再进入 Python 清理逻辑;`owns_interpreter_` 守卫确保只终结自己初始化的解释器 | 879 | 3. **PybindBridge 层** — `Py_IsInitialized()` 守卫:解释器已终结后不再进入 Python 清理逻辑;`owns_interpreter_` 守卫确保只终结自己初始化的解释器 |
| 876 | 880 | ||
| @@ -1610,7 +1614,7 @@ TBE 初始化 Python | |||
| 1610 | 1614 | ||
| 1611 | ```text | 1615 | ```text |
| 1612 | TBE 先初始化 Python: | 1616 | TBE 先初始化 Python: |
| 1613 | -ShutdownPassPluginsForProcess() | 1617 | +UnloadPassPlugins() (引用计数归零时) |
| 1614 | -> reset / clear Python pass holder、module、registry | 1618 | -> reset / clear Python pass holder、module、registry |
| 1615 | GELib::Finalize() | 1619 | GELib::Finalize() |
| 1616 | -> TBE / op store finalize | 1620 | -> TBE / op store finalize |
| @@ -1618,7 +1622,7 @@ GELib::Finalize() | |||
| 1618 | -> TBE 在自己拥有解释器时 Py_Finalize | 1622 | -> TBE 在自己拥有解释器时 Py_Finalize |
| 1619 | 1623 | ||
| 1620 | Python pass bridge 先初始化 Python: | 1624 | Python pass bridge 先初始化 Python: |
| 1621 | -ShutdownPassPluginsForProcess() | 1625 | +UnloadPassPlugins() (引用计数归零时) |
| 1622 | -> reset / clear Python pass holder、module、registry | 1626 | -> reset / clear Python pass holder、module、registry |
| 1623 | -> 不能立即 py::finalize_interpreter,除非确认 TBE 等其他 Python 用户尚未初始化 | 1627 | -> 不能立即 py::finalize_interpreter,除非确认 TBE 等其他 Python 用户尚未初始化 |
| 1624 | GELib::Finalize() | 1628 | GELib::Finalize() |
| @@ -270,10 +270,14 @@ CustomPassHelper &CustomPassHelper::Instance() { | |||
| 270 | } | 270 | } |
| 271 | 271 | ||
| 272 | void CustomPassHelper::Insert(const PassRegistrationData ®_data) { | 272 | void CustomPassHelper::Insert(const PassRegistrationData ®_data) { |
| 273 | + // Insert is only called from PassReceiver constructor during dlopen in Load(), | ||
| 274 | + // which already holds the unique_lock. Using a lock here would cause deadlock | ||
| 275 | + // since std::shared_mutex is not recursive. | ||
| 273 | (void)registration_datas_.emplace_back(reg_data); | 276 | (void)registration_datas_.emplace_back(reg_data); |
| 274 | } | 277 | } |
| 275 | 278 | ||
| 276 | Status CustomPassHelper::Load() { | 279 | Status CustomPassHelper::Load() { |
| 280 | + std::unique_lock<std::shared_mutex> lock(mutex_); | ||
| 277 | GELOGD("[Load][CustomPassLibs] Start to load custom pass libs"); | 281 | GELOGD("[Load][CustomPassLibs] Start to load custom pass libs"); |
| 278 | std::string opp_path; | 282 | std::string opp_path; |
| 279 | GE_ASSERT_SUCCESS(ge::PluginManager::GetOppPath(opp_path)); | 283 | GE_ASSERT_SUCCESS(ge::PluginManager::GetOppPath(opp_path)); |
| @@ -305,6 +309,7 @@ Status CustomPassHelper::Load() { | |||
| 305 | } | 309 | } |
| 306 | 310 | ||
| 307 | Status CustomPassHelper::Unload() { | 311 | Status CustomPassHelper::Unload() { |
| 312 | + std::unique_lock<std::shared_mutex> lock(mutex_); | ||
| 308 | registration_datas_.clear(); | 313 | registration_datas_.clear(); |
| 309 | for (auto &handle : handles_) { | 314 | for (auto &handle : handles_) { |
| 310 | if (handle != nullptr && dlclose(handle) != 0) { | 315 | if (handle != nullptr && dlclose(handle) != 0) { |
| @@ -323,6 +328,7 @@ Status CustomPassHelper::Run(GraphPtr &graph, CustomPassContext &custom_pass_con | |||
| 323 | 328 | ||
| 324 | Status CustomPassHelper::Run(GraphPtr &graph, CustomPassContext &custom_pass_context, | 329 | Status CustomPassHelper::Run(GraphPtr &graph, CustomPassContext &custom_pass_context, |
| 325 | const CustomPassStage stage) const { | 330 | const CustomPassStage stage) const { |
| 331 | + std::shared_lock<std::shared_mutex> lock(mutex_); | ||
| 326 | for (auto &item : registration_datas_) { | 332 | for (auto &item : registration_datas_) { |
| 327 | if (item.GetStage() != stage) { | 333 | if (item.GetStage() != stage) { |
| 328 | continue; | 334 | continue; |
| @@ -12,6 +12,7 @@ | |||
| 12 | 12 | ||
| 13 | 13 | ||
| 14 | 14 | ||
| 15 | + | ||
| 15 | 16 | ||
| 16 | 17 | ||
| 17 | 18 | ||
| @@ -35,6 +36,7 @@ class CustomPassHelper { | |||
| 35 | 36 | ||
| 36 | private: | 37 | private: |
| 37 | CustomPassHelper() = default; | 38 | CustomPassHelper() = default; |
| 39 | + mutable std::shared_mutex mutex_; | ||
| 38 | std::vector<PassRegistrationData> registration_datas_; | 40 | std::vector<PassRegistrationData> registration_datas_; |
| 39 | std::vector<void *> handles_; | 41 | std::vector<void *> handles_; |
| 40 | }; | 42 | }; |
| @@ -1914,6 +1914,25 @@ TEST_F(UtestFusionPassExecutor, PythonPassPluginLoader_RollbackOnDuplicatePassNa | |||
| 1914 | EXPECT_EQ(UnloadPassPlugins(), SUCCESS); | 1914 | EXPECT_EQ(UnloadPassPlugins(), SUCCESS); |
| 1915 | } | 1915 | } |
| 1916 | 1916 | ||
| 1917 | +TEST_F(UtestFusionPassExecutor, LoadPassPlugins_RefCounting) { | ||
| 1918 | + // 多引用:Load×2 后第一次 Unload 不释放,第二次才真正卸载 | ||
| 1919 | + EXPECT_EQ(LoadPassPlugins(), SUCCESS); | ||
| 1920 | + EXPECT_EQ(LoadPassPlugins(), SUCCESS); | ||
| 1921 | + EXPECT_EQ(UnloadPassPlugins(), SUCCESS); | ||
| 1922 | + EXPECT_EQ(UnloadPassPlugins(), SUCCESS); | ||
| 1923 | + | ||
| 1924 | + // 无引用时 Unload 不崩溃 | ||
| 1925 | + EXPECT_EQ(UnloadPassPlugins(), SUCCESS); | ||
| 1926 | +} | ||
| 1927 | + | ||
| 1928 | +TEST_F(UtestFusionPassExecutor, LoadPassPlugins_PythonPass) { | ||
| 1929 | + EnsureSharedPybindPassFile(); | ||
| 1930 | + ScopedEnvVar scoped_py_pass_path(kEnvPythonPassPath, GetSharedPybindPassFilePath()); | ||
| 1931 | + | ||
| 1932 | + EXPECT_EQ(LoadPassPlugins(), SUCCESS); | ||
| 1933 | + EXPECT_EQ(UnloadPassPlugins(), SUCCESS); | ||
| 1934 | +} | ||
| 1935 | + | ||
| 1917 | TEST_F(UtestFusionPassExecutor, PythonPatternFusionPass_CreateRunHooksAndDestroy) { | 1936 | TEST_F(UtestFusionPassExecutor, PythonPatternFusionPass_CreateRunHooksAndDestroy) { |
| 1918 | PythonPassDescriptor pass_desc; | 1937 | PythonPassDescriptor pass_desc; |
| 1919 | pass_desc.descriptor_key = "python.pattern.fusion.create"; | 1938 | pass_desc.descriptor_key = "python.pattern.fusion.create"; |
| @@ -26,6 +26,8 @@ | |||
| 26 | 26 | ||
| 27 | 27 | ||
| 28 | 28 | ||
| 29 | + | ||
| 30 | + | ||
| 29 | using namespace std; | 31 | using namespace std; |
| 30 | 32 | ||
| 31 | namespace ge { | 33 | namespace ge { |
| @@ -39,6 +41,12 @@ const string kOpMasterPath = "/op_impl/ai_core/tbe/op_tiling/lib/linux/x86_64/"; | |||
| 39 | graphStatus InferFunctionStub(Operator &op) { | 41 | graphStatus InferFunctionStub(Operator &op) { |
| 40 | return GRAPH_SUCCESS; | 42 | return GRAPH_SUCCESS; |
| 41 | } | 43 | } |
| 44 | + | ||
| 45 | +size_t g_custom_pass_run_count = 0U; | ||
| 46 | +Status CountCustomPassRun(GraphPtr &, CustomPassContext &) { | ||
| 47 | + g_custom_pass_run_count++; | ||
| 48 | + return SUCCESS; | ||
| 49 | +} | ||
| 42 | } // namespace | 50 | } // namespace |
| 43 | const char *const kKernelLibName = "DNN_VM_GE_LOCAL"; | 51 | const char *const kKernelLibName = "DNN_VM_GE_LOCAL"; |
| 44 | class UtestGeGenerator : public testing::Test { | 52 | class UtestGeGenerator : public testing::Test { |
| @@ -279,6 +287,24 @@ TEST_F(UtestGeGenerator, test_generate_online_model) { | |||
| 279 | EXPECT_NE(generator.GenerateOfflineModel(graph, name, inputs), SUCCESS); | 287 | EXPECT_NE(generator.GenerateOfflineModel(graph, name, inputs), SUCCESS); |
| 280 | } | 288 | } |
| 281 | 289 | ||
| 290 | +TEST_F(UtestGeGenerator, InitializeFailedReleasesPassPluginsImmediately) { | ||
| 291 | + ASSERT_EQ(fusion::LoadPassPlugins(), SUCCESS); | ||
| 292 | + PassRegistrationData pass_reg_data("CountCustomPassRun"); | ||
| 293 | + pass_reg_data.CustomPassFn(CountCustomPassRun); | ||
| 294 | + CustomPassHelper::Instance().Insert(pass_reg_data); | ||
| 295 | + | ||
| 296 | + GeGenerator generator; | ||
| 297 | + const std::map<std::string, std::string> invalid_options = {{STREAM_NUM, "0"}}; | ||
| 298 | + EXPECT_EQ(generator.Initialize(invalid_options), GE_GENERATOR_GRAPH_MANAGER_INIT_FAILED); | ||
| 299 | + EXPECT_EQ(fusion::UnloadPassPlugins(), SUCCESS); | ||
| 300 | + | ||
| 301 | + g_custom_pass_run_count = 0U; | ||
| 302 | + GraphPtr graph = std::make_shared<Graph>("graph"); | ||
| 303 | + CustomPassContext context; | ||
| 304 | + EXPECT_EQ(CustomPassHelper::Instance().Run(graph, context), SUCCESS); | ||
| 305 | + EXPECT_EQ(g_custom_pass_run_count, 0U); | ||
| 306 | +} | ||
| 307 | + | ||
| 282 | TEST_F(UtestGeGenerator, test_create_generalized_build_attrs) { | 308 | TEST_F(UtestGeGenerator, test_create_generalized_build_attrs) { |
| 283 | GeGenerator generator; | 309 | GeGenerator generator; |
| 284 | auto ret = generator.Initialize({}); | 310 | auto ret = generator.Initialize({}); |
ShutdownPassPluginsForProcess删除之后 设计文档需要同步更新一下