Open
fix bug: The osd fails to recover after being restarted #129
AtomGit-BotAdd on Sep 12, 2024
fix bug: The osd fails to recover after being restarted #129
Open
Merged fromrefs/pull/129/headintomaster
Total: 5 files changed+121-79
| @@ -115,6 +115,11 @@ public: | |||
| 115 | recovery_op_ctx* ctx = new recovery_op_ctx{.recv = this, .obs = _obs, | 115 | recovery_op_ctx* ctx = new recovery_op_ctx{.recv = this, .obs = _obs, |
| 116 | .cb_fn = std::move(cb_fn), .arg = arg}; | 116 | .cb_fn = std::move(cb_fn), .arg = arg}; |
| 117 | iter_start(); | 117 | iter_start(); |
| 118 | + if(iter_is_end()){ | ||
| 119 | + cb_fn(arg, 0); | ||
| 120 | + delete ctx; | ||
| 121 | + return; | ||
| 122 | + } | ||
| 118 | auto next_name = iter_next_name(); | 123 | auto next_name = iter_next_name(); |
| 119 | // SPDK_DEBUGLOG(object_store, "object name:%s delete.\n", next_name.c_str()); | 124 | // SPDK_DEBUGLOG(object_store, "object name:%s delete.\n", next_name.c_str()); |
| 120 | _obs->recovery_delete(next_name, recovery_delete_continue, ctx); | 125 | _obs->recovery_delete(next_name, recovery_delete_continue, ctx); |
| @@ -169,7 +169,7 @@ std::vector<shard_manager::node_heartbeat> shard_manager::get_heartbeat_requests | |||
| 169 | raft_time_t now = utils::get_time(); | 169 | raft_time_t now = utils::get_time(); |
| 170 | for(auto& p : _pgs){ | 170 | for(auto& p : _pgs){ |
| 171 | auto raft = p.second; | 171 | auto raft = p.second; |
| 172 | - if(raft->raft_get_identity() != RAFT_STATE_LEADER){ | 172 | + if(!raft->raft_is_leader()){ |
| 173 | continue; | 173 | continue; |
| 174 | } | 174 | } |
| 175 | 175 | ||
| @@ -189,18 +189,29 @@ std::vector<shard_manager::node_heartbeat> shard_manager::get_heartbeat_requests | |||
| 189 | if(node->raft_get_append_time() + raft->raft_get_heartbeat_timeout() > now) | 189 | if(node->raft_get_append_time() + raft->raft_get_heartbeat_timeout() > now) |
| 190 | return; | 190 | return; |
| 191 | 191 | ||
| 192 | - SPDK_DEBUGLOG(pg_group, "------ heartbeat to node: %d pg: %lu.%lu\n", | ||
| 193 | - node->raft_node_get_id(), raft->raft_get_pool_id(), raft->raft_get_pg_id()); | ||
| 194 | // node->raft_node_set_heartbeating(true); | 192 | // node->raft_node_set_heartbeating(true); |
| 195 | node->raft_set_append_time(now); | 193 | node->raft_set_append_time(now); |
| 196 | 194 | ||
| 197 | raft->raft_set_election_timer(now); | 195 | raft->raft_set_election_timer(now); |
| 198 | 196 | ||
| 199 | raft_index_t next_idx = node->raft_node_get_next_idx(); | 197 | raft_index_t next_idx = node->raft_node_get_next_idx(); |
| 198 | + SPDK_DEBUGLOG(pg_group, "------ heartbeat to node: %d pg: %lu.%lu next_idx: %ld\n", | ||
| 199 | + node->raft_node_get_id(), raft->raft_get_pool_id(), raft->raft_get_pg_id(), next_idx); | ||
| 200 | raft_term_t term = 0; | 200 | raft_term_t term = 0; |
| 201 | auto got = raft->raft_get_entry_term(next_idx - 1, term); | 201 | auto got = raft->raft_get_entry_term(next_idx - 1, term); |
| 202 | if(!got && next_idx - 1 != 0){ | 202 | if(!got && next_idx - 1 != 0){ |
| 203 | - return; | 203 | + SPDK_DEBUGLOG(pg_group, "pg: %lu.%lu node: %d , not has %ld entry in raft_entry\n", |
| 204 | + raft->raft_get_pool_id(), raft->raft_get_pg_id(), node->raft_node_get_id(), next_idx - 1); | ||
| 205 | + if(next_idx - 1 < raft->raft_get_log()->log_get_base_index()){ | ||
| 206 | + next_idx = raft->raft_get_current_idx() + 1; | ||
| 207 | + got = raft->raft_get_entry_term(next_idx - 1, term); | ||
| 208 | + SPDK_DEBUGLOG(pg_group, "pg: %lu.%lu node: %d , next_idx %ld term %ld\n", | ||
| 209 | + raft->raft_get_pool_id(), raft->raft_get_pg_id(), node->raft_node_get_id(), next_idx, term); | ||
| 210 | + if(!got) | ||
| 211 | + return; | ||
| 212 | + }else { | ||
| 213 | + return; | ||
| 214 | + } | ||
| 204 | } | 215 | } |
| 205 | 216 | ||
| 206 | heartbeat_request* req = nullptr; | 217 | heartbeat_request* req = nullptr; |
| @@ -637,6 +637,7 @@ public: | |||
| 637 | msg_appendentries_t* create_appendentries(std::shared_ptr<raft_node> node, raft_index_t next_idx); | 637 | msg_appendentries_t* create_appendentries(std::shared_ptr<raft_node> node, raft_index_t next_idx); |
| 638 | void dispatch_recovery(std::shared_ptr<raft_node> node); | 638 | void dispatch_recovery(std::shared_ptr<raft_node> node); |
| 639 | void do_recovery(std::shared_ptr<raft_node> node); | 639 | void do_recovery(std::shared_ptr<raft_node> node); |
| 640 | + void stop_recovery(std::shared_ptr<raft_node> node, int result); | ||
| 640 | 641 | ||
| 641 | void raft_set_op_state(raft_op_state op_state){ | 642 | void raft_set_op_state(raft_op_state op_state){ |
| 642 | _op_state = op_state; | 643 | _op_state = op_state; |
| @@ -748,6 +749,8 @@ public: | |||
| 748 | _nodes_stat.update_with_node_configuration(cfg, {}); | 749 | _nodes_stat.update_with_node_configuration(cfg, {}); |
| 749 | } | 750 | } |
| 750 | 751 | ||
| 752 | + void cfg_change_check(std::shared_ptr<raft_node> node, int result, raft_index_t match_index); | ||
| 753 | + | ||
| 751 | raft_nodes& get_nodes_stat(){ | 754 | raft_nodes& get_nodes_stat(){ |
| 752 | return _nodes_stat; | 755 | return _nodes_stat; |
| 753 | } | 756 | } |
| @@ -791,11 +794,6 @@ public: | |||
| 791 | return _obr; | 794 | return _obr; |
| 792 | } | 795 | } |
| 793 | 796 | ||
| 794 | - void free_object_recovery(){ | ||
| 795 | - delete _obr; | ||
| 796 | - _obr = nullptr; | ||
| 797 | - } | ||
| 798 | - | ||
| 799 | /* | 797 | /* |
| 800 | * follower节点在接收完leader的快照后,才能调用 | 798 | * follower节点在接收完leader的快照后,才能调用 |
| 801 | */ | 799 | */ |
| @@ -395,7 +395,7 @@ int raft_server_t::raft_process_appendentries_reply( | |||
| 395 | } | 395 | } |
| 396 | 396 | ||
| 397 | if(node->raft_node_is_recovering() && result != 0){ | 397 | if(node->raft_node_is_recovering() && result != 0){ |
| 398 | - node->raft_node_set_recovering(false); | 398 | + stop_recovery(node, result); |
| 399 | return; | 399 | return; |
| 400 | } | 400 | } |
| 401 | 401 | ||
| @@ -460,6 +460,10 @@ int raft_server_t::raft_process_appendentries_reply( | |||
| 460 | } | 460 | } |
| 461 | 461 | ||
| 462 | if(r->success() == err::RAFT_ERR_LOG_NOT_MATCH){ | 462 | if(r->success() == err::RAFT_ERR_LOG_NOT_MATCH){ |
| 463 | + if(node->raft_node_is_recovering()){ | ||
| 464 | + if(is_heartbeat) | ||
| 465 | + return 1; | ||
| 466 | + } | ||
| 463 | node->raft_node_set_next_idx(r->current_idx() + 1); | 467 | node->raft_node_set_next_idx(r->current_idx() + 1); |
| 464 | dispatch_recovery(node); | 468 | dispatch_recovery(node); |
| 465 | return 1; | 469 | return 1; |
| @@ -477,8 +481,10 @@ int raft_server_t::raft_process_appendentries_reply( | |||
| 477 | 481 | ||
| 478 | raft_index_t next_idx = node->raft_node_get_next_idx(); | 482 | raft_index_t next_idx = node->raft_node_get_next_idx(); |
| 479 | if(node->raft_node_is_recovering()){ | 483 | if(node->raft_node_is_recovering()){ |
| 480 | - // SPDK_WARNLOG("node %d is recovering, match_idx: %ld next_idx: %ld r->current_idx: %ld\n", | 484 | + if(is_heartbeat) |
| 481 | - // node->raft_node_get_id(), match_idx, next_idx, r->current_idx()); | 485 | + return 1; |
| 486 | + SPDK_DEBUGLOG(pg_group, "node %d is recovering, match_idx: %ld next_idx: %ld r->current_idx: %ld\n", | ||
| 487 | + node->raft_node_get_id(), match_idx, next_idx, r->current_idx()); | ||
| 482 | 488 | ||
| 483 | if(next_idx > r->current_idx()){ | 489 | if(next_idx > r->current_idx()){ |
| 484 | node->raft_node_set_next_idx(r->current_idx() + 1); | 490 | node->raft_node_set_next_idx(r->current_idx() + 1); |
| @@ -554,8 +560,8 @@ struct follow_disk_append_complete : public utils::context{ | |||
| 554 | , rsp(_rsp) {} | 560 | , rsp(_rsp) {} |
| 555 | 561 | ||
| 556 | void finish(int r) override { | 562 | void finish(int r) override { |
| 557 | - SPDK_INFOLOG(pg_group, "follow_disk_append_complete finish, start_idx: %ld end_idx: %ld commit_idx %ld result: %d.\n", | 563 | + SPDK_INFOLOG(pg_group, "follow_disk_append_complete finish, pg: %s start_idx: %ld end_idx: %ld commit_idx %ld result: %d.\n", |
| 558 | - start_idx, end_idx, commit_idx, r); | 564 | + raft->raft_get_pg_name().c_str(), start_idx, end_idx, commit_idx, r); |
| 559 | if(r != 0){ | 565 | if(r != 0){ |
| 560 | SPDK_ERRLOG("follow_disk_append_complete, result: %d\n", r); | 566 | SPDK_ERRLOG("follow_disk_append_complete, result: %d\n", r); |
| 561 | rsp->set_success(r); | 567 | rsp->set_success(r); |
| @@ -1796,12 +1802,6 @@ int raft_server_t::raft_process_installsnapshot_reply(installsnapshot_response * | |||
| 1796 | return 1; | 1802 | return 1; |
| 1797 | } | 1803 | } |
| 1798 | 1804 | ||
| 1799 | - auto snap_end_err = [this, node](void *arg, int rerrno){ | ||
| 1800 | - delete _obr; | ||
| 1801 | - _obr = nullptr; | ||
| 1802 | - node->raft_node_set_recovering(false); | ||
| 1803 | - }; | ||
| 1804 | - | ||
| 1805 | auto snap_end = [this, node](void *arg, int rerrno){ | 1805 | auto snap_end = [this, node](void *arg, int rerrno){ |
| 1806 | delete _obr; | 1806 | delete _obr; |
| 1807 | _obr = nullptr; | 1807 | _obr = nullptr; |
| @@ -1809,26 +1809,32 @@ int raft_server_t::raft_process_installsnapshot_reply(installsnapshot_response * | |||
| 1809 | }; | 1809 | }; |
| 1810 | 1810 | ||
| 1811 | if (!raft_is_leader()){ | 1811 | if (!raft_is_leader()){ |
| 1812 | - _recovery_delete(snap_end_err, nullptr); | 1812 | + stop_recovery(node, err::RAFT_ERR_NOT_LEADER); |
| 1813 | return err::RAFT_ERR_NOT_LEADER; | 1813 | return err::RAFT_ERR_NOT_LEADER; |
| 1814 | } | 1814 | } |
| 1815 | 1815 | ||
| 1816 | + if((raft_get_op_state() == raft_op_state::RAFT_DOWN) | ||
| 1817 | + || (raft_get_op_state() == raft_op_state::RAFT_DELETE)){ | ||
| 1818 | + stop_recovery(node, err::RAFT_ERR_PG_DELETED); | ||
| 1819 | + return err::RAFT_ERR_PG_DELETED; | ||
| 1820 | + } | ||
| 1821 | + | ||
| 1816 | if (raft_get_current_term() < rsp->term()){ | 1822 | if (raft_get_current_term() < rsp->term()){ |
| 1817 | raft_set_current_term(rsp->term()); | 1823 | raft_set_current_term(rsp->term()); |
| 1818 | raft_become_follower(); | 1824 | raft_become_follower(); |
| 1819 | raft_set_current_leader(-1); | 1825 | raft_set_current_leader(-1); |
| 1820 | SPDK_ERRLOG("node %d change from leader to follow: node id %d term %ld\n", | 1826 | SPDK_ERRLOG("node %d change from leader to follow: node id %d term %ld\n", |
| 1821 | raft_get_nodeid(), node->raft_node_get_id(), rsp->term()); | 1827 | raft_get_nodeid(), node->raft_node_get_id(), rsp->term()); |
| 1822 | - _recovery_delete(snap_end_err, nullptr); | 1828 | + stop_recovery(node, err::RAFT_ERR_NOT_LEADER); |
| 1823 | return err::RAFT_ERR_NOT_LEADER; | 1829 | return err::RAFT_ERR_NOT_LEADER; |
| 1824 | } | 1830 | } |
| 1825 | else if (raft_get_current_term() != rsp->term()){ | 1831 | else if (raft_get_current_term() != rsp->term()){ |
| 1826 | - _recovery_delete(snap_end_err, nullptr); | 1832 | + stop_recovery(node, err::RAFT_ERR_SNAPSHOT_FAILED); |
| 1827 | return 1; | 1833 | return 1; |
| 1828 | } | 1834 | } |
| 1829 | 1835 | ||
| 1830 | if(rsp->success() != 1){ | 1836 | if(rsp->success() != 1){ |
| 1831 | - _recovery_delete(snap_end_err, nullptr); | 1837 | + stop_recovery(node, err::RAFT_ERR_SNAPSHOT_FAILED); |
| 1832 | return 1; | 1838 | return 1; |
| 1833 | } | 1839 | } |
| 1834 | 1840 | ||
| @@ -1841,7 +1847,7 @@ int raft_server_t::raft_process_installsnapshot_reply(installsnapshot_response * | |||
| 1841 | }else{ | 1847 | }else{ |
| 1842 | int res = raft_send_snapshot_check(node); | 1848 | int res = raft_send_snapshot_check(node); |
| 1843 | if(res != err::E_SUCCESS){ | 1849 | if(res != err::E_SUCCESS){ |
| 1844 | - _recovery_delete(snap_end_err, nullptr); | 1850 | + stop_recovery(node, res); |
| 1845 | SPDK_ERRLOG("pg: %lu.%lu, send snapshot_check_request failed, rerrno: %d\n", _pool_id, _pg_id, res); | 1851 | SPDK_ERRLOG("pg: %lu.%lu, send snapshot_check_request failed, rerrno: %d\n", _pool_id, _pg_id, res); |
| 1846 | return 1; | 1852 | return 1; |
| 1847 | } | 1853 | } |
| @@ -2059,11 +2065,9 @@ public: | |||
| 2059 | } | 2065 | } |
| 2060 | } | 2066 | } |
| 2061 | 2067 | ||
| 2062 | - void handle_end(){ | 2068 | + void handle_end(int result){ |
| 2063 | - _raft->free_object_recovery(); | ||
| 2064 | auto node = _raft->raft_get_cfg_node(_node_id); | 2069 | auto node = _raft->raft_get_cfg_node(_node_id); |
| 2065 | - if(node) | 2070 | + _raft->stop_recovery(node, result); |
| 2066 | - node->raft_node_set_recovering(false); | ||
| 2067 | delete this; | 2071 | delete this; |
| 2068 | }; | 2072 | }; |
| 2069 | 2073 | ||
| @@ -2107,12 +2111,12 @@ public: | |||
| 2107 | if(!node){ | 2111 | if(!node){ |
| 2108 | SPDK_ERRLOG("pg: %lu.%lu, not contain the node %d\n", | 2112 | SPDK_ERRLOG("pg: %lu.%lu, not contain the node %d\n", |
| 2109 | _raft->raft_get_pool_id(), _raft->raft_get_pg_id(), _node_id); | 2113 | _raft->raft_get_pool_id(), _raft->raft_get_pg_id(), _node_id); |
| 2110 | - handle_end(); | 2114 | + handle_end(err::RAFT_ERR_NO_FOUND_NODE); |
| 2111 | return; | 2115 | return; |
| 2112 | } | 2116 | } |
| 2113 | auto res = _raft->raft_send_snapshot_check(node); | 2117 | auto res = _raft->raft_send_snapshot_check(node); |
| 2114 | if(res != err::E_SUCCESS){ | 2118 | if(res != err::E_SUCCESS){ |
| 2115 | - handle_end(); | 2119 | + handle_end(res); |
| 2116 | SPDK_ERRLOG("pg: %lu.%lu, send installsnapshot_request failed, rerrno: %d\n", | 2120 | SPDK_ERRLOG("pg: %lu.%lu, send installsnapshot_request failed, rerrno: %d\n", |
| 2117 | _raft->raft_get_pool_id(), _raft->raft_get_pg_id(), res); | 2121 | _raft->raft_get_pool_id(), _raft->raft_get_pg_id(), res); |
| 2118 | return; | 2122 | return; |
| @@ -2122,7 +2126,7 @@ public: | |||
| 2122 | _raft->raft_get_pool_id(), _raft->raft_get_pg_id(), _node_id); | 2126 | _raft->raft_get_pool_id(), _raft->raft_get_pg_id(), _node_id); |
| 2123 | auto res = _raft->raft_send_installsnapshot(snapshot, _node_id); | 2127 | auto res = _raft->raft_send_installsnapshot(snapshot, _node_id); |
| 2124 | if(res != err::E_SUCCESS){ | 2128 | if(res != err::E_SUCCESS){ |
| 2125 | - handle_end(); | 2129 | + handle_end(res); |
| 2126 | SPDK_ERRLOG("pg: %lu.%lu, send installsnapshot_request failed, rerrno: %d\n", | 2130 | SPDK_ERRLOG("pg: %lu.%lu, send installsnapshot_request failed, rerrno: %d\n", |
| 2127 | _raft->raft_get_pool_id(), _raft->raft_get_pg_id(), res); | 2131 | _raft->raft_get_pool_id(), _raft->raft_get_pg_id(), res); |
| 2128 | return; | 2132 | return; |
| @@ -2136,17 +2140,22 @@ public: | |||
| 2136 | if(_rerrno == 0) | 2140 | if(_rerrno == 0) |
| 2137 | _rerrno = rerrno; | 2141 | _rerrno = rerrno; |
| 2138 | if(_num % SNAPSHOT_MAX_CONCURRENT == 0 || _num == _count){ | 2142 | if(_num % SNAPSHOT_MAX_CONCURRENT == 0 || _num == _count){ |
| 2139 | - handle_end(); | 2143 | + handle_end(rerrno); |
| 2140 | return; | 2144 | return; |
| 2141 | } | 2145 | } |
| 2142 | } | 2146 | } |
| 2143 | 2147 | ||
| 2144 | if(_num == _count){ | 2148 | if(_num == _count){ |
| 2145 | - SPDK_DEBUGLOG(pg_group, "pg: %lu.%lu, _num %d\n", _raft->raft_get_pool_id(), _raft->raft_get_pg_id(), _num); | 2149 | + SPDK_DEBUGLOG(pg_group, "pg: %lu.%lu, _num %d\n", |
| 2146 | - if(!_raft->raft_is_leader()){ | 2150 | + _raft->raft_get_pool_id(), _raft->raft_get_pg_id(), _num); |
| 2147 | - handle_end(); | 2151 | + if(!_raft->raft_is_leader() |
| 2152 | + || (_raft->raft_get_op_state() == raft_op_state::RAFT_DOWN) | ||
| 2153 | + || (_raft->raft_get_op_state() == raft_op_state::RAFT_DELETE)){ | ||
| 2154 | + SPDK_DEBUGLOG(pg_group, "stop recovery\n"); | ||
| 2155 | + handle_end(err::RAFT_ERR_NOT_LEADER); | ||
| 2148 | return; | 2156 | return; |
| 2149 | } | 2157 | } |
| 2158 | + | ||
| 2150 | send_snapshot(); | 2159 | send_snapshot(); |
| 2151 | delete this; | 2160 | delete this; |
| 2152 | }else if(_num % SNAPSHOT_MAX_CONCURRENT == 0){ | 2161 | }else if(_num % SNAPSHOT_MAX_CONCURRENT == 0){ |
| @@ -2211,29 +2220,29 @@ int raft_server_t::raft_process_snapshot_check_reply(snapshot_check_response *rs | |||
| 2211 | return 1; | 2220 | return 1; |
| 2212 | } | 2221 | } |
| 2213 | 2222 | ||
| 2214 | - auto handle_error = [this, node](){ | ||
| 2215 | - delete _obr; | ||
| 2216 | - _obr = nullptr; | ||
| 2217 | - node->raft_node_set_recovering(false); | ||
| 2218 | - }; | ||
| 2219 | - | ||
| 2220 | 2223 | ||
| 2221 | if (!raft_is_leader()){ | 2224 | if (!raft_is_leader()){ |
| 2222 | - handle_error(); | 2225 | + stop_recovery(node, err::RAFT_ERR_NOT_LEADER); |
| 2223 | return err::RAFT_ERR_NOT_LEADER; | 2226 | return err::RAFT_ERR_NOT_LEADER; |
| 2224 | } | 2227 | } |
| 2225 | 2228 | ||
| 2229 | + if((raft_get_op_state() == raft_op_state::RAFT_DOWN) | ||
| 2230 | + || (raft_get_op_state() == raft_op_state::RAFT_DELETE)){ | ||
| 2231 | + stop_recovery(node, err::RAFT_ERR_PG_DELETED); | ||
| 2232 | + return err::RAFT_ERR_PG_DELETED; | ||
| 2233 | + } | ||
| 2234 | + | ||
| 2226 | if (raft_get_current_term() < rsp->term()){ | 2235 | if (raft_get_current_term() < rsp->term()){ |
| 2227 | raft_set_current_term(rsp->term()); | 2236 | raft_set_current_term(rsp->term()); |
| 2228 | raft_become_follower(); | 2237 | raft_become_follower(); |
| 2229 | raft_set_current_leader(-1); | 2238 | raft_set_current_leader(-1); |
| 2230 | SPDK_ERRLOG("node %d change from leader to follow: node id %d term %ld\n", | 2239 | SPDK_ERRLOG("node %d change from leader to follow: node id %d term %ld\n", |
| 2231 | raft_get_nodeid(), node->raft_node_get_id(), rsp->term()); | 2240 | raft_get_nodeid(), node->raft_node_get_id(), rsp->term()); |
| 2232 | - handle_error(); | 2241 | + stop_recovery(node, err::RAFT_ERR_NOT_LEADER); |
| 2233 | return err::RAFT_ERR_NOT_LEADER; | 2242 | return err::RAFT_ERR_NOT_LEADER; |
| 2234 | } | 2243 | } |
| 2235 | else if (raft_get_current_term() != rsp->term()){ | 2244 | else if (raft_get_current_term() != rsp->term()){ |
| 2236 | - handle_error(); | 2245 | + stop_recovery(node, err::RAFT_ERR_SNAPSHOT_FAILED); |
| 2237 | return 1; | 2246 | return 1; |
| 2238 | } | 2247 | } |
| 2239 | 2248 | ||
| @@ -2243,7 +2252,7 @@ int raft_server_t::raft_process_snapshot_check_reply(snapshot_check_response *rs | |||
| 2243 | } | 2252 | } |
| 2244 | 2253 | ||
| 2245 | if(rsp->success() != 1){ | 2254 | if(rsp->success() != 1){ |
| 2246 | - handle_error(); | 2255 | + stop_recovery(node, err::RAFT_ERR_SNAPSHOT_FAILED); |
| 2247 | return 1; | 2256 | return 1; |
| 2248 | } | 2257 | } |
| 2249 | 2258 | ||
| @@ -2286,22 +2295,15 @@ int raft_server_t::raft_send_snapshot_check(std::shared_ptr<raft_node> node){ | |||
| 2286 | } | 2295 | } |
| 2287 | 2296 | ||
| 2288 | void raft_server_t::handle_snap_check_task(task_info& task){ | 2297 | void raft_server_t::handle_snap_check_task(task_info& task){ |
| 2289 | - auto handle_error = [this](std::shared_ptr<raft_node> node){ | ||
| 2290 | - delete _obr; | ||
| 2291 | - _obr = nullptr; | ||
| 2292 | - if(node) | ||
| 2293 | - node->raft_node_set_recovering(false); | ||
| 2294 | - }; | ||
| 2295 | - | ||
| 2296 | auto node = raft_get_cfg_node(task.node_id); | 2298 | auto node = raft_get_cfg_node(task.node_id); |
| 2297 | if(!node){ | 2299 | if(!node){ |
| 2298 | - handle_error(node); | 2300 | + stop_recovery(node, err::RAFT_ERR_NO_FOUND_NODE); |
| 2299 | return; | 2301 | return; |
| 2300 | } | 2302 | } |
| 2301 | 2303 | ||
| 2302 | int res = raft_send_snapshot_check(node); | 2304 | int res = raft_send_snapshot_check(node); |
| 2303 | if(res != err::E_SUCCESS){ | 2305 | if(res != err::E_SUCCESS){ |
| 2304 | - handle_error(node); | 2306 | + stop_recovery(node, res); |
| 2305 | return; | 2307 | return; |
| 2306 | } | 2308 | } |
| 2307 | } | 2309 | } |
| @@ -2317,28 +2319,22 @@ int raft_server_t::_recovery_by_snapshot(std::shared_ptr<raft_node> node){ | |||
| 2317 | auto obs = _machine->get_object_store(); | 2319 | auto obs = _machine->get_object_store(); |
| 2318 | _obr = new object_recovery(obs); | 2320 | _obr = new object_recovery(obs); |
| 2319 | 2321 | ||
| 2320 | - auto handle_error = [this, node](){ | ||
| 2321 | - delete _obr; | ||
| 2322 | - _obr = nullptr; | ||
| 2323 | - node->raft_node_set_recovering(false); | ||
| 2324 | - }; | ||
| 2325 | - | ||
| 2326 | uint32_t shard_id = core_sharded::get_core_sharded().this_shard_id(); | 2322 | uint32_t shard_id = core_sharded::get_core_sharded().this_shard_id(); |
| 2327 | - auto snap_fn = [this, node, shard_id, handle_error = std::move(handle_error)](void *arg, int rerrno){ | 2323 | + auto snap_fn = [this, node, shard_id](void *arg, int rerrno){ |
| 2328 | core_sharded::get_core_sharded().invoke_on( | 2324 | core_sharded::get_core_sharded().invoke_on( |
| 2329 | shard_id, | 2325 | shard_id, |
| 2330 | - [this, rerrno, node, handle_error = std::move(handle_error)](){ | 2326 | + [this, rerrno, node](){ |
| 2331 | //创建完对象的snapshot后就需要设置_snapshot_in_progress为false | 2327 | //创建完对象的snapshot后就需要设置_snapshot_in_progress为false |
| 2332 | raft_set_snapshot_in_progress(false); | 2328 | raft_set_snapshot_in_progress(false); |
| 2333 | if(rerrno != 0){ | 2329 | if(rerrno != 0){ |
| 2334 | - handle_error(); | 2330 | + stop_recovery(node, rerrno); |
| 2335 | SPDK_ERRLOG("pg: %lu.%lu, create raft snapshot failed, rerrno: %d\n", _pool_id, _pg_id, rerrno); | 2331 | SPDK_ERRLOG("pg: %lu.%lu, create raft snapshot failed, rerrno: %d\n", _pool_id, _pg_id, rerrno); |
| 2336 | return; | 2332 | return; |
| 2337 | } | 2333 | } |
| 2338 | _snapshot_index = _machine->get_last_applied_idx(); | 2334 | _snapshot_index = _machine->get_last_applied_idx(); |
| 2339 | raft_term_t term; | 2335 | raft_term_t term; |
| 2340 | if(!raft_get_entry_term(_snapshot_index, term)){ | 2336 | if(!raft_get_entry_term(_snapshot_index, term)){ |
| 2341 | - handle_error(); | 2337 | + stop_recovery(node, err::RAFT_ERR_SNAPSHOT_FAILED); |
| 2342 | return; | 2338 | return; |
| 2343 | } | 2339 | } |
| 2344 | _snapshot_term = term; | 2340 | _snapshot_term = term; |
| @@ -2347,7 +2343,7 @@ int raft_server_t::_recovery_by_snapshot(std::shared_ptr<raft_node> node){ | |||
| 2347 | _obr->iter_start(); | 2343 | _obr->iter_start(); |
| 2348 | int res = raft_send_snapshot_check(node); | 2344 | int res = raft_send_snapshot_check(node); |
| 2349 | if(res != err::E_SUCCESS){ | 2345 | if(res != err::E_SUCCESS){ |
| 2350 | - handle_error(); | 2346 | + stop_recovery(node, res); |
| 2351 | SPDK_ERRLOG("pg: %lu.%lu, send snapshot_check_request failed, rerrno: %d\n", _pool_id, _pg_id, res); | 2347 | SPDK_ERRLOG("pg: %lu.%lu, send snapshot_check_request failed, rerrno: %d\n", _pool_id, _pg_id, res); |
| 2352 | return; | 2348 | return; |
| 2353 | } | 2349 | } |
| @@ -2365,6 +2361,26 @@ int raft_server_t::_recovery_by_snapshot(std::shared_ptr<raft_node> node){ | |||
| 2365 | return 0; | 2361 | return 0; |
| 2366 | } | 2362 | } |
| 2367 | 2363 | ||
| 2364 | +//recovery失败时,调用此函数 | ||
| 2365 | +void raft_server_t::stop_recovery(std::shared_ptr<raft_node> node, int result){ | ||
| 2366 | + if(_obr){ | ||
| 2367 | + auto snap_end = [this, node, result](void *arg, int rerrno){ | ||
| 2368 | + delete _obr; | ||
| 2369 | + _obr = nullptr; | ||
| 2370 | + if(node){ | ||
| 2371 | + node->raft_node_set_recovering(false); | ||
| 2372 | + cfg_change_check(node, result, 0); | ||
| 2373 | + } | ||
| 2374 | + }; | ||
| 2375 | + _recovery_delete(snap_end, nullptr); | ||
| 2376 | + }else{ | ||
| 2377 | + if(node){ | ||
| 2378 | + node->raft_node_set_recovering(false); | ||
| 2379 | + cfg_change_check(node, result, 0); | ||
| 2380 | + } | ||
| 2381 | + } | ||
| 2382 | +} | ||
| 2383 | + | ||
| 2368 | void raft_server_t::do_recovery(std::shared_ptr<raft_node> node){ | 2384 | void raft_server_t::do_recovery(std::shared_ptr<raft_node> node){ |
| 2369 | raft_index_t next_idx = node->raft_node_get_next_idx(); | 2385 | raft_index_t next_idx = node->raft_node_get_next_idx(); |
| 2370 | SPDK_DEBUGLOG(pg_group, "pg: %lu.%lu, do_recovery for node %d, next_idx: %ld log base_index: %ld\n", | 2386 | SPDK_DEBUGLOG(pg_group, "pg: %lu.%lu, do_recovery for node %d, next_idx: %ld log base_index: %ld\n", |
| @@ -2379,6 +2395,18 @@ void raft_server_t::do_recovery(std::shared_ptr<raft_node> node){ | |||
| 2379 | if(!node){ | 2395 | if(!node){ |
| 2380 | return; | 2396 | return; |
| 2381 | } | 2397 | } |
| 2398 | + | ||
| 2399 | + if(!raft_is_leader()){ | ||
| 2400 | + stop_recovery(node, err::RAFT_ERR_NOT_LEADER); | ||
| 2401 | + return; | ||
| 2402 | + } | ||
| 2403 | + | ||
| 2404 | + if((raft_get_op_state() == raft_op_state::RAFT_DOWN) | ||
| 2405 | + || (raft_get_op_state() == raft_op_state::RAFT_DELETE)){ | ||
| 2406 | + stop_recovery(node, err::RAFT_ERR_PG_DELETED); | ||
| 2407 | + return; | ||
| 2408 | + } | ||
| 2409 | + | ||
| 2382 | msg_appendentries_t*ae = create_appendentries(node, node->raft_node_get_next_idx()); | 2410 | msg_appendentries_t*ae = create_appendentries(node, node->raft_node_get_next_idx()); |
| 2383 | for(auto &entry : entries){ | 2411 | for(auto &entry : entries){ |
| 2384 | auto entry_ptr = ae->add_entries(); | 2412 | auto entry_ptr = ae->add_entries(); |
| @@ -2422,27 +2450,26 @@ void raft_server_t::do_recovery(std::shared_ptr<raft_node> node){ | |||
| 2422 | }); | 2450 | }); |
| 2423 | } | 2451 | } |
| 2424 | 2452 | ||
| 2453 | +void raft_server_t::cfg_change_check(std::shared_ptr<raft_node> node, int result, raft_index_t match_index) { | ||
| 2454 | + if((get_configuration_state() == cfg_state::CFG_CATCHING_UP) | ||
| 2455 | + && node_is_cfg_change_process(node)){ | ||
| 2456 | + cfg_change_process(result, match_index, node); | ||
| 2457 | + }else if(get_configuration_state() == cfg_state::CFG_JOINT){ | ||
| 2458 | + cfg_change_process(result, match_index, node); | ||
| 2459 | + } | ||
| 2460 | +} | ||
| 2461 | + | ||
| 2425 | void raft_server_t::dispatch_recovery(std::shared_ptr<raft_node> node){ | 2462 | void raft_server_t::dispatch_recovery(std::shared_ptr<raft_node> node){ |
| 2426 | - auto cfg_change_check = [this, node](int result, raft_index_t match_index){ | ||
| 2427 | - if((get_configuration_state() == cfg_state::CFG_CATCHING_UP) | ||
| 2428 | - && node_is_cfg_change_process(node)){ | ||
| 2429 | - cfg_change_process(result, match_index, node); | ||
| 2430 | - }else if(get_configuration_state() == cfg_state::CFG_JOINT){ | ||
| 2431 | - cfg_change_process(result, match_index, node); | ||
| 2432 | - } | ||
| 2433 | - }; | ||
| 2434 | 2463 | ||
| 2435 | node->raft_node_set_recovering(true); | 2464 | node->raft_node_set_recovering(true); |
| 2436 | if(!raft_is_leader()){ | 2465 | if(!raft_is_leader()){ |
| 2437 | - node->raft_node_set_recovering(false); | 2466 | + stop_recovery(node, err::RAFT_ERR_NOT_LEADER); |
| 2438 | - cfg_change_check(err::RAFT_ERR_NOT_LEADER, 0); | ||
| 2439 | return; | 2467 | return; |
| 2440 | } | 2468 | } |
| 2441 | 2469 | ||
| 2442 | if((raft_get_op_state() == raft_op_state::RAFT_DOWN) | 2470 | if((raft_get_op_state() == raft_op_state::RAFT_DOWN) |
| 2443 | || (raft_get_op_state() == raft_op_state::RAFT_DELETE)){ | 2471 | || (raft_get_op_state() == raft_op_state::RAFT_DELETE)){ |
| 2444 | - node->raft_node_set_recovering(false); | 2472 | + stop_recovery(node, err::RAFT_ERR_PG_DELETED); |
| 2445 | - cfg_change_check(err::RAFT_ERR_PG_DELETED, 0); | ||
| 2446 | return; | 2473 | return; |
| 2447 | } | 2474 | } |
| 2448 | 2475 | ||
| @@ -2450,7 +2477,7 @@ void raft_server_t::dispatch_recovery(std::shared_ptr<raft_node> node){ | |||
| 2450 | SPDK_DEBUGLOG(pg_group, "end recovery. node %d match_idx: %ld raft current idx: %ld\n", | 2477 | SPDK_DEBUGLOG(pg_group, "end recovery. node %d match_idx: %ld raft current idx: %ld\n", |
| 2451 | node->raft_node_get_id(), node->raft_node_get_match_idx(), raft_get_current_idx()); | 2478 | node->raft_node_get_id(), node->raft_node_get_match_idx(), raft_get_current_idx()); |
| 2452 | node->raft_node_set_recovering(false); | 2479 | node->raft_node_set_recovering(false); |
| 2453 | - cfg_change_check(0, node->raft_node_get_match_idx()); | 2480 | + cfg_change_check(node, 0, node->raft_node_get_match_idx()); |
| 2454 | return; | 2481 | return; |
| 2455 | } | 2482 | } |
| 2456 | if(get_configuration_state() == cfg_state::CFG_CATCHING_UP){ | 2483 | if(get_configuration_state() == cfg_state::CFG_CATCHING_UP){ |
| @@ -52,6 +52,7 @@ enum { | |||
| 52 | OSD_ERR_ID_CONFLICT = -157, | 52 | OSD_ERR_ID_CONFLICT = -157, |
| 53 | OSD_ERR_ADDRESS_INVALID = -158, | 53 | OSD_ERR_ADDRESS_INVALID = -158, |
| 54 | OSD_ERR_UPDATE_STATE_FAILED = -159, | 54 | OSD_ERR_UPDATE_STATE_FAILED = -159, |
| 55 | + RAFT_ERR_SNAPSHOT_FAILED = -160, | ||
| 55 | 56 | ||
| 56 | RAFT_ERR_UNKNOWN = -199, | 57 | RAFT_ERR_UNKNOWN = -199, |
| 57 | RAFT_ERR_LAST = -200, | 58 | RAFT_ERR_LAST = -200, |