Open
fix bug: The osd fails to recover after being restarted #129
AtomGit-BotAdd on Sep 12, 2024
fix bug: The osd fails to recover after being restarted #129
Open
AtomGit-BotAdd on Sep 12, 2024
Merged fromrefs/pull/129/headintomaster
Total: 5 files changed+121-79
@@ -115,6 +115,11 @@ public:
115 recovery_op_ctx* ctx = new recovery_op_ctx{.recv = this, .obs = _obs,115 recovery_op_ctx* ctx = new recovery_op_ctx{.recv = this, .obs = _obs,
116 .cb_fn = std::move(cb_fn), .arg = arg};116 .cb_fn = std::move(cb_fn), .arg = arg};
117 iter_start();117 iter_start();
118+ if(iter_is_end()){
119+ cb_fn(arg, 0);
120+ delete ctx;
121+ return;
122+ }
118 auto next_name = iter_next_name();123 auto next_name = iter_next_name();
119 // SPDK_DEBUGLOG(object_store, "object name:%s delete.\n", next_name.c_str());124 // SPDK_DEBUGLOG(object_store, "object name:%s delete.\n", next_name.c_str());
120 _obs->recovery_delete(next_name, recovery_delete_continue, ctx);125 _obs->recovery_delete(next_name, recovery_delete_continue, ctx);
@@ -169,7 +169,7 @@ std::vector<shard_manager::node_heartbeat> shard_manager::get_heartbeat_requests
169 raft_time_t now = utils::get_time();169 raft_time_t now = utils::get_time();
170 for(auto& p : _pgs){170 for(auto& p : _pgs){
171 auto raft = p.second;171 auto raft = p.second;
172- if(raft->raft_get_identity() != RAFT_STATE_LEADER){172+ if(!raft->raft_is_leader()){
173 continue;173 continue;
174 }174 }
175 175 
@@ -189,18 +189,29 @@ std::vector<shard_manager::node_heartbeat> shard_manager::get_heartbeat_requests
189 if(node->raft_get_append_time() + raft->raft_get_heartbeat_timeout() > now)189 if(node->raft_get_append_time() + raft->raft_get_heartbeat_timeout() > now)
190 return;190 return;
191 191 
192- SPDK_DEBUGLOG(pg_group, "------ heartbeat to node: %d pg: %lu.%lu\n",
193- node->raft_node_get_id(), raft->raft_get_pool_id(), raft->raft_get_pg_id());
194 // node->raft_node_set_heartbeating(true);192 // node->raft_node_set_heartbeating(true);
195 node->raft_set_append_time(now);193 node->raft_set_append_time(now);
196 194 
197 raft->raft_set_election_timer(now);195 raft->raft_set_election_timer(now);
198 196 
199 raft_index_t next_idx = node->raft_node_get_next_idx();197 raft_index_t next_idx = node->raft_node_get_next_idx();
198+ SPDK_DEBUGLOG(pg_group, "------ heartbeat to node: %d pg: %lu.%lu next_idx: %ld\n",
199+ node->raft_node_get_id(), raft->raft_get_pool_id(), raft->raft_get_pg_id(), next_idx);
200 raft_term_t term = 0;200 raft_term_t term = 0;
201 auto got = raft->raft_get_entry_term(next_idx - 1, term);201 auto got = raft->raft_get_entry_term(next_idx - 1, term);
202 if(!got && next_idx - 1 != 0){202 if(!got && next_idx - 1 != 0){
203- return;203+ SPDK_DEBUGLOG(pg_group, "pg: %lu.%lu node: %d , not has %ld entry in raft_entry\n",
204+ raft->raft_get_pool_id(), raft->raft_get_pg_id(), node->raft_node_get_id(), next_idx - 1);
205+ if(next_idx - 1 < raft->raft_get_log()->log_get_base_index()){
206+ next_idx = raft->raft_get_current_idx() + 1;
207+ got = raft->raft_get_entry_term(next_idx - 1, term);
208+ SPDK_DEBUGLOG(pg_group, "pg: %lu.%lu node: %d , next_idx %ld term %ld\n",
209+ raft->raft_get_pool_id(), raft->raft_get_pg_id(), node->raft_node_get_id(), next_idx, term);
210+ if(!got)
211+ return;
212+ }else {
213+ return;
214+ }
204 }215 }
205 216 
206 heartbeat_request* req = nullptr;217 heartbeat_request* req = nullptr;
@@ -637,6 +637,7 @@ public:
637 msg_appendentries_t* create_appendentries(std::shared_ptr<raft_node> node, raft_index_t next_idx);637 msg_appendentries_t* create_appendentries(std::shared_ptr<raft_node> node, raft_index_t next_idx);
638 void dispatch_recovery(std::shared_ptr<raft_node> node);638 void dispatch_recovery(std::shared_ptr<raft_node> node);
639 void do_recovery(std::shared_ptr<raft_node> node);639 void do_recovery(std::shared_ptr<raft_node> node);
640+ void stop_recovery(std::shared_ptr<raft_node> node, int result);
640 641 
641 void raft_set_op_state(raft_op_state op_state){642 void raft_set_op_state(raft_op_state op_state){
642 _op_state = op_state;643 _op_state = op_state;
@@ -748,6 +749,8 @@ public:
748 _nodes_stat.update_with_node_configuration(cfg, {}); 749 _nodes_stat.update_with_node_configuration(cfg, {});
749 }750 }
750 751 
752+ void cfg_change_check(std::shared_ptr<raft_node> node, int result, raft_index_t match_index);
753+ 
751 raft_nodes& get_nodes_stat(){754 raft_nodes& get_nodes_stat(){
752 return _nodes_stat;755 return _nodes_stat;
753 }756 }
@@ -791,11 +794,6 @@ public:
791 return _obr;794 return _obr;
792 }795 }
793 796 
794- void free_object_recovery(){
795- delete _obr;
796- _obr = nullptr;
797- }
798- 
799 /*797 /*
800 * follower节点在接收完leader的快照后,才能调用798 * follower节点在接收完leader的快照后,才能调用
801 */799 */
@@ -395,7 +395,7 @@ int raft_server_t::raft_process_appendentries_reply(
395 }395 }
396 396 
397 if(node->raft_node_is_recovering() && result != 0){397 if(node->raft_node_is_recovering() && result != 0){
398- node->raft_node_set_recovering(false);398+ stop_recovery(node, result);
399 return;399 return;
400 }400 }
401 401 
@@ -460,6 +460,10 @@ int raft_server_t::raft_process_appendentries_reply(
460 }460 }
461 461 
462 if(r->success() == err::RAFT_ERR_LOG_NOT_MATCH){462 if(r->success() == err::RAFT_ERR_LOG_NOT_MATCH){
463+ if(node->raft_node_is_recovering()){
464+ if(is_heartbeat)
465+ return 1;
466+ }
463 node->raft_node_set_next_idx(r->current_idx() + 1);467 node->raft_node_set_next_idx(r->current_idx() + 1);
464 dispatch_recovery(node);468 dispatch_recovery(node);
465 return 1;469 return 1;
@@ -477,8 +481,10 @@ int raft_server_t::raft_process_appendentries_reply(
477 481 
478 raft_index_t next_idx = node->raft_node_get_next_idx();482 raft_index_t next_idx = node->raft_node_get_next_idx();
479 if(node->raft_node_is_recovering()){483 if(node->raft_node_is_recovering()){
480- // SPDK_WARNLOG("node %d is recovering, match_idx: %ld next_idx: %ld r->current_idx: %ld\n",484+ if(is_heartbeat)
481- // node->raft_node_get_id(), match_idx, next_idx, r->current_idx());485+ return 1;
486+ SPDK_DEBUGLOG(pg_group, "node %d is recovering, match_idx: %ld next_idx: %ld r->current_idx: %ld\n",
487+ node->raft_node_get_id(), match_idx, next_idx, r->current_idx());
482 488 
483 if(next_idx > r->current_idx()){489 if(next_idx > r->current_idx()){
484 node->raft_node_set_next_idx(r->current_idx() + 1);490 node->raft_node_set_next_idx(r->current_idx() + 1);
@@ -554,8 +560,8 @@ struct follow_disk_append_complete : public utils::context{
554 , rsp(_rsp) {}560 , rsp(_rsp) {}
555 561 
556 void finish(int r) override {562 void finish(int r) override {
557- SPDK_INFOLOG(pg_group, "follow_disk_append_complete finish, start_idx: %ld end_idx: %ld commit_idx %ld result: %d.\n",563+ SPDK_INFOLOG(pg_group, "follow_disk_append_complete finish, pg: %s start_idx: %ld end_idx: %ld commit_idx %ld result: %d.\n",
558- start_idx, end_idx, commit_idx, r);564+ raft->raft_get_pg_name().c_str(), start_idx, end_idx, commit_idx, r);
559 if(r != 0){565 if(r != 0){
560 SPDK_ERRLOG("follow_disk_append_complete, result: %d\n", r);566 SPDK_ERRLOG("follow_disk_append_complete, result: %d\n", r);
561 rsp->set_success(r);567 rsp->set_success(r);
@@ -1796,12 +1802,6 @@ int raft_server_t::raft_process_installsnapshot_reply(installsnapshot_response *
1796 return 1;1802 return 1;
1797 }1803 }
1798 1804 
1799- auto snap_end_err = [this, node](void *arg, int rerrno){
1800- delete _obr;
1801- _obr = nullptr;
1802- node->raft_node_set_recovering(false);
1803- };
1804- 
1805 auto snap_end = [this, node](void *arg, int rerrno){1805 auto snap_end = [this, node](void *arg, int rerrno){
1806 delete _obr;1806 delete _obr;
1807 _obr = nullptr;1807 _obr = nullptr;
@@ -1809,26 +1809,32 @@ int raft_server_t::raft_process_installsnapshot_reply(installsnapshot_response *
1809 };1809 };
1810 1810 
1811 if (!raft_is_leader()){1811 if (!raft_is_leader()){
1812- _recovery_delete(snap_end_err, nullptr);1812+ stop_recovery(node, err::RAFT_ERR_NOT_LEADER);
1813 return err::RAFT_ERR_NOT_LEADER;1813 return err::RAFT_ERR_NOT_LEADER;
1814 }1814 }
1815 1815 
1816+ if((raft_get_op_state() == raft_op_state::RAFT_DOWN)
1817+ || (raft_get_op_state() == raft_op_state::RAFT_DELETE)){
1818+ stop_recovery(node, err::RAFT_ERR_PG_DELETED);
1819+ return err::RAFT_ERR_PG_DELETED;
1820+ }
1821+ 
1816 if (raft_get_current_term() < rsp->term()){1822 if (raft_get_current_term() < rsp->term()){
1817 raft_set_current_term(rsp->term());1823 raft_set_current_term(rsp->term());
1818 raft_become_follower();1824 raft_become_follower();
1819 raft_set_current_leader(-1);1825 raft_set_current_leader(-1);
1820 SPDK_ERRLOG("node %d change from leader to follow: node id %d term %ld\n",1826 SPDK_ERRLOG("node %d change from leader to follow: node id %d term %ld\n",
1821 raft_get_nodeid(), node->raft_node_get_id(), rsp->term());1827 raft_get_nodeid(), node->raft_node_get_id(), rsp->term());
1822- _recovery_delete(snap_end_err, nullptr);1828+ stop_recovery(node, err::RAFT_ERR_NOT_LEADER);
1823 return err::RAFT_ERR_NOT_LEADER;1829 return err::RAFT_ERR_NOT_LEADER;
1824 }1830 }
1825 else if (raft_get_current_term() != rsp->term()){1831 else if (raft_get_current_term() != rsp->term()){
1826- _recovery_delete(snap_end_err, nullptr);1832+ stop_recovery(node, err::RAFT_ERR_SNAPSHOT_FAILED);
1827 return 1;1833 return 1;
1828 }1834 }
1829 1835 
1830 if(rsp->success() != 1){1836 if(rsp->success() != 1){
1831- _recovery_delete(snap_end_err, nullptr);1837+ stop_recovery(node, err::RAFT_ERR_SNAPSHOT_FAILED);
1832 return 1;1838 return 1;
1833 }1839 }
1834 1840 
@@ -1841,7 +1847,7 @@ int raft_server_t::raft_process_installsnapshot_reply(installsnapshot_response *
1841 }else{1847 }else{
1842 int res = raft_send_snapshot_check(node);1848 int res = raft_send_snapshot_check(node);
1843 if(res != err::E_SUCCESS){1849 if(res != err::E_SUCCESS){
1844- _recovery_delete(snap_end_err, nullptr);1850+ stop_recovery(node, res);
1845 SPDK_ERRLOG("pg: %lu.%lu, send snapshot_check_request failed, rerrno: %d\n", _pool_id, _pg_id, res);1851 SPDK_ERRLOG("pg: %lu.%lu, send snapshot_check_request failed, rerrno: %d\n", _pool_id, _pg_id, res);
1846 return 1;1852 return 1;
1847 }1853 }
@@ -2059,11 +2065,9 @@ public:
2059 }2065 }
2060 }2066 }
2061 2067 
2062- void handle_end(){2068+ void handle_end(int result){
2063- _raft->free_object_recovery();
2064 auto node = _raft->raft_get_cfg_node(_node_id);2069 auto node = _raft->raft_get_cfg_node(_node_id);
2065- if(node)2070+ _raft->stop_recovery(node, result);
2066- node->raft_node_set_recovering(false);
2067 delete this;2071 delete this;
2068 };2072 };
2069 2073 
@@ -2107,12 +2111,12 @@ public:
2107 if(!node){2111 if(!node){
2108 SPDK_ERRLOG("pg: %lu.%lu, not contain the node %d\n",2112 SPDK_ERRLOG("pg: %lu.%lu, not contain the node %d\n",
2109 _raft->raft_get_pool_id(), _raft->raft_get_pg_id(), _node_id);2113 _raft->raft_get_pool_id(), _raft->raft_get_pg_id(), _node_id);
2110- handle_end();2114+ handle_end(err::RAFT_ERR_NO_FOUND_NODE);
2111 return;2115 return;
2112 }2116 }
2113 auto res = _raft->raft_send_snapshot_check(node);2117 auto res = _raft->raft_send_snapshot_check(node);
2114 if(res != err::E_SUCCESS){2118 if(res != err::E_SUCCESS){
2115- handle_end();2119+ handle_end(res);
2116 SPDK_ERRLOG("pg: %lu.%lu, send installsnapshot_request failed, rerrno: %d\n",2120 SPDK_ERRLOG("pg: %lu.%lu, send installsnapshot_request failed, rerrno: %d\n",
2117 _raft->raft_get_pool_id(), _raft->raft_get_pg_id(), res);2121 _raft->raft_get_pool_id(), _raft->raft_get_pg_id(), res);
2118 return;2122 return;
@@ -2122,7 +2126,7 @@ public:
2122 _raft->raft_get_pool_id(), _raft->raft_get_pg_id(), _node_id);2126 _raft->raft_get_pool_id(), _raft->raft_get_pg_id(), _node_id);
2123 auto res = _raft->raft_send_installsnapshot(snapshot, _node_id);2127 auto res = _raft->raft_send_installsnapshot(snapshot, _node_id);
2124 if(res != err::E_SUCCESS){2128 if(res != err::E_SUCCESS){
2125- handle_end();2129+ handle_end(res);
2126 SPDK_ERRLOG("pg: %lu.%lu, send installsnapshot_request failed, rerrno: %d\n",2130 SPDK_ERRLOG("pg: %lu.%lu, send installsnapshot_request failed, rerrno: %d\n",
2127 _raft->raft_get_pool_id(), _raft->raft_get_pg_id(), res);2131 _raft->raft_get_pool_id(), _raft->raft_get_pg_id(), res);
2128 return;2132 return;
@@ -2136,17 +2140,22 @@ public:
2136 if(_rerrno == 0)2140 if(_rerrno == 0)
2137 _rerrno = rerrno;2141 _rerrno = rerrno;
2138 if(_num % SNAPSHOT_MAX_CONCURRENT == 0 || _num == _count){2142 if(_num % SNAPSHOT_MAX_CONCURRENT == 0 || _num == _count){
2139- handle_end();2143+ handle_end(rerrno);
2140 return;2144 return;
2141 }2145 }
2142 }2146 }
2143 2147 
2144 if(_num == _count){2148 if(_num == _count){
2145- SPDK_DEBUGLOG(pg_group, "pg: %lu.%lu, _num %d\n", _raft->raft_get_pool_id(), _raft->raft_get_pg_id(), _num);2149+ SPDK_DEBUGLOG(pg_group, "pg: %lu.%lu, _num %d\n",
2146- if(!_raft->raft_is_leader()){2150+ _raft->raft_get_pool_id(), _raft->raft_get_pg_id(), _num);
2147- handle_end();2151+ if(!_raft->raft_is_leader()
2152+ || (_raft->raft_get_op_state() == raft_op_state::RAFT_DOWN)
2153+ || (_raft->raft_get_op_state() == raft_op_state::RAFT_DELETE)){
2154+ SPDK_DEBUGLOG(pg_group, "stop recovery\n");
2155+ handle_end(err::RAFT_ERR_NOT_LEADER);
2148 return;2156 return;
2149 }2157 }
2158+ 
2150 send_snapshot();2159 send_snapshot();
2151 delete this;2160 delete this;
2152 }else if(_num % SNAPSHOT_MAX_CONCURRENT == 0){2161 }else if(_num % SNAPSHOT_MAX_CONCURRENT == 0){
@@ -2211,29 +2220,29 @@ int raft_server_t::raft_process_snapshot_check_reply(snapshot_check_response *rs
2211 return 1;2220 return 1;
2212 }2221 }
2213 2222 
2214- auto handle_error = [this, node](){
2215- delete _obr;
2216- _obr = nullptr;
2217- node->raft_node_set_recovering(false);
2218- };
2219- 
2220 2223 
2221 if (!raft_is_leader()){2224 if (!raft_is_leader()){
2222- handle_error();2225+ stop_recovery(node, err::RAFT_ERR_NOT_LEADER);
2223 return err::RAFT_ERR_NOT_LEADER;2226 return err::RAFT_ERR_NOT_LEADER;
2224 }2227 }
2225 2228 
2229+ if((raft_get_op_state() == raft_op_state::RAFT_DOWN)
2230+ || (raft_get_op_state() == raft_op_state::RAFT_DELETE)){
2231+ stop_recovery(node, err::RAFT_ERR_PG_DELETED);
2232+ return err::RAFT_ERR_PG_DELETED;
2233+ }
2234+ 
2226 if (raft_get_current_term() < rsp->term()){2235 if (raft_get_current_term() < rsp->term()){
2227 raft_set_current_term(rsp->term());2236 raft_set_current_term(rsp->term());
2228 raft_become_follower();2237 raft_become_follower();
2229 raft_set_current_leader(-1);2238 raft_set_current_leader(-1);
2230 SPDK_ERRLOG("node %d change from leader to follow: node id %d term %ld\n",2239 SPDK_ERRLOG("node %d change from leader to follow: node id %d term %ld\n",
2231 raft_get_nodeid(), node->raft_node_get_id(), rsp->term());2240 raft_get_nodeid(), node->raft_node_get_id(), rsp->term());
2232- handle_error();2241+ stop_recovery(node, err::RAFT_ERR_NOT_LEADER);
2233 return err::RAFT_ERR_NOT_LEADER;2242 return err::RAFT_ERR_NOT_LEADER;
2234 }2243 }
2235 else if (raft_get_current_term() != rsp->term()){2244 else if (raft_get_current_term() != rsp->term()){
2236- handle_error();2245+ stop_recovery(node, err::RAFT_ERR_SNAPSHOT_FAILED);
2237 return 1;2246 return 1;
2238 }2247 }
2239 2248 
@@ -2243,7 +2252,7 @@ int raft_server_t::raft_process_snapshot_check_reply(snapshot_check_response *rs
2243 }2252 }
2244 2253 
2245 if(rsp->success() != 1){2254 if(rsp->success() != 1){
2246- handle_error();2255+ stop_recovery(node, err::RAFT_ERR_SNAPSHOT_FAILED);
2247 return 1;2256 return 1;
2248 }2257 }
2249 2258 
@@ -2286,22 +2295,15 @@ int raft_server_t::raft_send_snapshot_check(std::shared_ptr<raft_node> node){
2286}2295}
2287 2296 
2288void raft_server_t::handle_snap_check_task(task_info& task){2297void raft_server_t::handle_snap_check_task(task_info& task){
2289- auto handle_error = [this](std::shared_ptr<raft_node> node){
2290- delete _obr;
2291- _obr = nullptr;
2292- if(node)
2293- node->raft_node_set_recovering(false);
2294- };
2295- 
2296 auto node = raft_get_cfg_node(task.node_id);2298 auto node = raft_get_cfg_node(task.node_id);
2297 if(!node){2299 if(!node){
2298- handle_error(node);2300+ stop_recovery(node, err::RAFT_ERR_NO_FOUND_NODE);
2299 return;2301 return;
2300 }2302 }
2301 2303 
2302 int res = raft_send_snapshot_check(node);2304 int res = raft_send_snapshot_check(node);
2303 if(res != err::E_SUCCESS){2305 if(res != err::E_SUCCESS){
2304- handle_error(node);2306+ stop_recovery(node, res);
2305 return;2307 return;
2306 }2308 }
2307}2309}
@@ -2317,28 +2319,22 @@ int raft_server_t::_recovery_by_snapshot(std::shared_ptr<raft_node> node){
2317 auto obs = _machine->get_object_store();2319 auto obs = _machine->get_object_store();
2318 _obr = new object_recovery(obs);2320 _obr = new object_recovery(obs);
2319 2321 
2320- auto handle_error = [this, node](){
2321- delete _obr;
2322- _obr = nullptr;
2323- node->raft_node_set_recovering(false);
2324- };
2325- 
2326 uint32_t shard_id = core_sharded::get_core_sharded().this_shard_id();2322 uint32_t shard_id = core_sharded::get_core_sharded().this_shard_id();
2327- auto snap_fn = [this, node, shard_id, handle_error = std::move(handle_error)](void *arg, int rerrno){2323+ auto snap_fn = [this, node, shard_id](void *arg, int rerrno){
2328 core_sharded::get_core_sharded().invoke_on(2324 core_sharded::get_core_sharded().invoke_on(
2329 shard_id,2325 shard_id,
2330- [this, rerrno, node, handle_error = std::move(handle_error)](){2326+ [this, rerrno, node](){
2331 //创建完对象的snapshot后就需要设置_snapshot_in_progress为false2327 //创建完对象的snapshot后就需要设置_snapshot_in_progress为false
2332 raft_set_snapshot_in_progress(false);2328 raft_set_snapshot_in_progress(false);
2333 if(rerrno != 0){2329 if(rerrno != 0){
2334- handle_error();2330+ stop_recovery(node, rerrno);
2335 SPDK_ERRLOG("pg: %lu.%lu, create raft snapshot failed, rerrno: %d\n", _pool_id, _pg_id, rerrno);2331 SPDK_ERRLOG("pg: %lu.%lu, create raft snapshot failed, rerrno: %d\n", _pool_id, _pg_id, rerrno);
2336 return;2332 return;
2337 }2333 }
2338 _snapshot_index = _machine->get_last_applied_idx();2334 _snapshot_index = _machine->get_last_applied_idx();
2339 raft_term_t term;2335 raft_term_t term;
2340 if(!raft_get_entry_term(_snapshot_index, term)){2336 if(!raft_get_entry_term(_snapshot_index, term)){
2341- handle_error();2337+ stop_recovery(node, err::RAFT_ERR_SNAPSHOT_FAILED);
2342 return;2338 return;
2343 }2339 }
2344 _snapshot_term = term;2340 _snapshot_term = term;
@@ -2347,7 +2343,7 @@ int raft_server_t::_recovery_by_snapshot(std::shared_ptr<raft_node> node){
2347 _obr->iter_start();2343 _obr->iter_start();
2348 int res = raft_send_snapshot_check(node);2344 int res = raft_send_snapshot_check(node);
2349 if(res != err::E_SUCCESS){2345 if(res != err::E_SUCCESS){
2350- handle_error();2346+ stop_recovery(node, res);
2351 SPDK_ERRLOG("pg: %lu.%lu, send snapshot_check_request failed, rerrno: %d\n", _pool_id, _pg_id, res);2347 SPDK_ERRLOG("pg: %lu.%lu, send snapshot_check_request failed, rerrno: %d\n", _pool_id, _pg_id, res);
2352 return;2348 return;
2353 }2349 }
@@ -2365,6 +2361,26 @@ int raft_server_t::_recovery_by_snapshot(std::shared_ptr<raft_node> node){
2365 return 0;2361 return 0;
2366}2362}
2367 2363 
2364+//recovery失败时,调用此函数
2365+void raft_server_t::stop_recovery(std::shared_ptr<raft_node> node, int result){
2366+ if(_obr){
2367+ auto snap_end = [this, node, result](void *arg, int rerrno){
2368+ delete _obr;
2369+ _obr = nullptr;
2370+ if(node){
2371+ node->raft_node_set_recovering(false);
2372+ cfg_change_check(node, result, 0);
2373+ }
2374+ };
2375+ _recovery_delete(snap_end, nullptr);
2376+ }else{
2377+ if(node){
2378+ node->raft_node_set_recovering(false);
2379+ cfg_change_check(node, result, 0);
2380+ }
2381+ }
2382+}
2383+ 
2368void raft_server_t::do_recovery(std::shared_ptr<raft_node> node){2384void raft_server_t::do_recovery(std::shared_ptr<raft_node> node){
2369 raft_index_t next_idx = node->raft_node_get_next_idx();2385 raft_index_t next_idx = node->raft_node_get_next_idx();
2370 SPDK_DEBUGLOG(pg_group, "pg: %lu.%lu, do_recovery for node %d, next_idx: %ld log base_index: %ld\n",2386 SPDK_DEBUGLOG(pg_group, "pg: %lu.%lu, do_recovery for node %d, next_idx: %ld log base_index: %ld\n",
@@ -2379,6 +2395,18 @@ void raft_server_t::do_recovery(std::shared_ptr<raft_node> node){
2379 if(!node){2395 if(!node){
2380 return;2396 return;
2381 }2397 }
2398+ 
2399+ if(!raft_is_leader()){
2400+ stop_recovery(node, err::RAFT_ERR_NOT_LEADER);
2401+ return;
2402+ }
2403+
2404+ if((raft_get_op_state() == raft_op_state::RAFT_DOWN)
2405+ || (raft_get_op_state() == raft_op_state::RAFT_DELETE)){
2406+ stop_recovery(node, err::RAFT_ERR_PG_DELETED);
2407+ return;
2408+ }
2409+ 
2382 msg_appendentries_t*ae = create_appendentries(node, node->raft_node_get_next_idx());2410 msg_appendentries_t*ae = create_appendentries(node, node->raft_node_get_next_idx());
2383 for(auto &entry : entries){2411 for(auto &entry : entries){
2384 auto entry_ptr = ae->add_entries();2412 auto entry_ptr = ae->add_entries();
@@ -2422,27 +2450,26 @@ void raft_server_t::do_recovery(std::shared_ptr<raft_node> node){
2422 });2450 });
2423}2451}
2424 2452 
2453+void raft_server_t::cfg_change_check(std::shared_ptr<raft_node> node, int result, raft_index_t match_index) {
2454+ if((get_configuration_state() == cfg_state::CFG_CATCHING_UP)
2455+ && node_is_cfg_change_process(node)){
2456+ cfg_change_process(result, match_index, node);
2457+ }else if(get_configuration_state() == cfg_state::CFG_JOINT){
2458+ cfg_change_process(result, match_index, node);
2459+ }
2460+}
2461+ 
2425void raft_server_t::dispatch_recovery(std::shared_ptr<raft_node> node){2462void raft_server_t::dispatch_recovery(std::shared_ptr<raft_node> node){
2426- auto cfg_change_check = [this, node](int result, raft_index_t match_index){
2427- if((get_configuration_state() == cfg_state::CFG_CATCHING_UP)
2428- && node_is_cfg_change_process(node)){
2429- cfg_change_process(result, match_index, node);
2430- }else if(get_configuration_state() == cfg_state::CFG_JOINT){
2431- cfg_change_process(result, match_index, node);
2432- }
2433- };
2434 2463 
2435 node->raft_node_set_recovering(true);2464 node->raft_node_set_recovering(true);
2436 if(!raft_is_leader()){2465 if(!raft_is_leader()){
2437- node->raft_node_set_recovering(false);2466+ stop_recovery(node, err::RAFT_ERR_NOT_LEADER);
2438- cfg_change_check(err::RAFT_ERR_NOT_LEADER, 0);
2439 return;2467 return;
2440 }2468 }
2441 2469 
2442 if((raft_get_op_state() == raft_op_state::RAFT_DOWN)2470 if((raft_get_op_state() == raft_op_state::RAFT_DOWN)
2443 || (raft_get_op_state() == raft_op_state::RAFT_DELETE)){2471 || (raft_get_op_state() == raft_op_state::RAFT_DELETE)){
2444- node->raft_node_set_recovering(false);2472+ stop_recovery(node, err::RAFT_ERR_PG_DELETED);
2445- cfg_change_check(err::RAFT_ERR_PG_DELETED, 0);
2446 return;2473 return;
2447 }2474 }
2448 2475 
@@ -2450,7 +2477,7 @@ void raft_server_t::dispatch_recovery(std::shared_ptr<raft_node> node){
2450 SPDK_DEBUGLOG(pg_group, "end recovery. node %d match_idx: %ld raft current idx: %ld\n",2477 SPDK_DEBUGLOG(pg_group, "end recovery. node %d match_idx: %ld raft current idx: %ld\n",
2451 node->raft_node_get_id(), node->raft_node_get_match_idx(), raft_get_current_idx());2478 node->raft_node_get_id(), node->raft_node_get_match_idx(), raft_get_current_idx());
2452 node->raft_node_set_recovering(false);2479 node->raft_node_set_recovering(false);
2453- cfg_change_check(0, node->raft_node_get_match_idx());2480+ cfg_change_check(node, 0, node->raft_node_get_match_idx());
2454 return;2481 return;
2455 }2482 }
2456 if(get_configuration_state() == cfg_state::CFG_CATCHING_UP){2483 if(get_configuration_state() == cfg_state::CFG_CATCHING_UP){
@@ -52,6 +52,7 @@ enum {
52 OSD_ERR_ID_CONFLICT = -157,52 OSD_ERR_ID_CONFLICT = -157,
53 OSD_ERR_ADDRESS_INVALID = -158,53 OSD_ERR_ADDRESS_INVALID = -158,
54 OSD_ERR_UPDATE_STATE_FAILED = -159,54 OSD_ERR_UPDATE_STATE_FAILED = -159,
55+ RAFT_ERR_SNAPSHOT_FAILED = -160,
55 56 
56 RAFT_ERR_UNKNOWN = -199,57 RAFT_ERR_UNKNOWN = -199,
57 RAFT_ERR_LAST = -200,58 RAFT_ERR_LAST = -200,