已合并
fix: 修复DFlowInitialize失败不回滚和DFlowFinalize中aclFinalize顺序错误 #4473
lining23666创建于 6 天前
fix: 修复DFlowInitialize失败不回滚和DFlowFinalize中aclFinalize顺序错误 #4473
已合并
共 1 个文件变更+21-23
| @@ -41,15 +41,33 @@ void ConvertAscendStringMap(const std::map<ge::AscendString, ge::AscendString> & | |||
| 41 | } | 41 | } |
| 42 | 42 | ||
| 43 | std::atomic_bool g_dflow_ge_initialized{false}; | 43 | std::atomic_bool g_dflow_ge_initialized{false}; |
| 44 | -std::mutex g_dflow_ge_release_mutex; // GEFinalize and ~DFlowSession use | 44 | +std::mutex g_dflow_ge_release_mutex; // DFlowInitialize, DFlowFinalize and ~DFlowSession use |
| 45 | std::shared_ptr<DFlowSessionManager> g_dflow_session_manager; | 45 | std::shared_ptr<DFlowSessionManager> g_dflow_session_manager; |
| 46 | + | ||
| 47 | +void DFlowFinalizeImpl() { | ||
| 48 | + GELOGT(TRACE_INIT, "DFlowFinalize start."); | ||
| 49 | + if (g_dflow_session_manager != nullptr) { | ||
| 50 | + g_dflow_session_manager->Finalize(); | ||
| 51 | + } | ||
| 52 | + (void)malloc_trim(0); | ||
| 53 | + g_dflow_ge_initialized = false; | ||
| 54 | + ge::DFlowFinalizeInner(); | ||
| 55 | + if (acl_owned_by_dflow) { | ||
| 56 | + aclFinalize(); | ||
| 57 | + acl_owned_by_dflow.store(false); | ||
| 58 | + } | ||
| 59 | + acl_initialized.store(false); | ||
| 60 | + GELOGT(TRACE_STOP, "DFlowFinalize finished"); | ||
| 61 | +} | ||
| 46 | } // namespace | 62 | } // namespace |
| 47 | 63 | ||
| 48 | Status DFlowInitialize(const std::map<AscendString, AscendString> &options) { | 64 | Status DFlowInitialize(const std::map<AscendString, AscendString> &options) { |
| 65 | + std::lock_guard<std::mutex> lock(g_dflow_ge_release_mutex); | ||
| 49 | if (g_dflow_ge_initialized) { | 66 | if (g_dflow_ge_initialized) { |
| 50 | GELOGW("DFlowInitialize is called more than once"); | 67 | GELOGW("DFlowInitialize is called more than once"); |
| 51 | return SUCCESS; | 68 | return SUCCESS; |
| 52 | } | 69 | } |
| 70 | + GE_DISMISSABLE_GUARD(rollback, ([]() { DFlowFinalizeImpl(); })); | ||
| 53 | if (!acl_initialized) { | 71 | if (!acl_initialized) { |
| 54 | aclError ret = aclInit(nullptr); | 72 | aclError ret = aclInit(nullptr); |
| 55 | if (ret != ACL_SUCCESS && ret != ACL_ERROR_REPEAT_INITIALIZE) { | 73 | if (ret != ACL_SUCCESS && ret != ACL_ERROR_REPEAT_INITIALIZE) { |
| @@ -62,7 +80,6 @@ Status DFlowInitialize(const std::map<AscendString, AscendString> &options) { | |||
| 62 | acl_owned_by_dflow.store(true); | 80 | acl_owned_by_dflow.store(true); |
| 63 | } | 81 | } |
| 64 | } | 82 | } |
| 65 | - // todo call GEInitialize in new so | ||
| 66 | GE_TIMESTAMP_START(DflowInitializeAll); | 83 | GE_TIMESTAMP_START(DflowInitializeAll); |
| 67 | GELOGI("sessionManager initial."); | 84 | GELOGI("sessionManager initial."); |
| 68 | GE_TIMESTAMP_START(DflowSessionManagerInitialize); | 85 | GE_TIMESTAMP_START(DflowSessionManagerInitialize); |
| @@ -76,6 +93,7 @@ Status DFlowInitialize(const std::map<AscendString, AscendString> &options) { | |||
| 76 | 93 | ||
| 77 | GE_CHK_STATUS_RET(ge::DFlowInitializeInner(options), "Failed to call dflow initialize inner"); | 94 | GE_CHK_STATUS_RET(ge::DFlowInitializeInner(options), "Failed to call dflow initialize inner"); |
| 78 | g_dflow_ge_initialized = true; | 95 | g_dflow_ge_initialized = true; |
| 96 | + GE_DISMISS_GUARD(rollback); | ||
| 79 | GELOGT(TRACE_STOP, "DFlowInitialize finished"); | 97 | GELOGT(TRACE_STOP, "DFlowInitialize finished"); |
| 80 | GE_TIMESTAMP_EVENT_END(DflowInitializeAll, "DflowInitialize::All"); | 98 | GE_TIMESTAMP_EVENT_END(DflowInitializeAll, "DflowInitialize::All"); |
| 81 | return SUCCESS; | 99 | return SUCCESS; |
| @@ -84,28 +102,8 @@ Status DFlowInitialize(const std::map<AscendString, AscendString> &options) { | |||
| 84 | // DFlow finalize, releasing all resources | 102 | // DFlow finalize, releasing all resources |
| 85 | Status DFlowFinalize() { | 103 | Status DFlowFinalize() { |
| 86 | GRAPH_PROFILING_REG(gert::GeProfInfoType::kGEFinalize); | 104 | GRAPH_PROFILING_REG(gert::GeProfInfoType::kGEFinalize); |
| 87 | - // check init status | ||
| 88 | - if (!g_dflow_ge_initialized) { | ||
| 89 | - GELOGW("[FINAL]DFlowFinalize is called before DFlowInitialize"); | ||
| 90 | - return SUCCESS; | ||
| 91 | - } | ||
| 92 | - if (acl_owned_by_dflow) { | ||
| 93 | - aclFinalize(); | ||
| 94 | - acl_owned_by_dflow.store(false); | ||
| 95 | - } | ||
| 96 | - acl_initialized.store(false); | ||
| 97 | std::lock_guard<std::mutex> lock(g_dflow_ge_release_mutex); | 105 | std::lock_guard<std::mutex> lock(g_dflow_ge_release_mutex); |
| 98 | - GELOGT(TRACE_INIT, "DFlowFinalize start."); | 106 | + DFlowFinalizeImpl(); |
| 99 | - | ||
| 100 | - GELOGI("DflowSessionManager finalization."); | ||
| 101 | - if (g_dflow_session_manager != nullptr) { | ||
| 102 | - g_dflow_session_manager->Finalize(); // always success. | ||
| 103 | - } | ||
| 104 | - (void)malloc_trim(0); | ||
| 105 | - g_dflow_ge_initialized = false; | ||
| 106 | - ge::DFlowFinalizeInner(); | ||
| 107 | - // todo GEFinalize | ||
| 108 | - GELOGT(TRACE_STOP, "DFlowFinalize finished"); | ||
| 109 | return SUCCESS; | 107 | return SUCCESS; |
| 110 | } | 108 | } |
| 111 | 109 | ||