已合并
[UBSE] #feat UBSE 链路亚健康自动化 #1469
quanletian创建于 19 天前
[UBSE] #feat UBSE 链路亚健康自动化 #1469
已合并
共 1 个文件变更+267-62
| @@ -14,14 +14,22 @@ | |||
| 14 | 14 | ||
| 15 | 15 | ||
| 16 | 16 | ||
| 17 | + | ||
| 18 | + | ||
| 19 | + | ||
| 17 | 20 | ||
| 18 | 21 | ||
| 22 | + | ||
| 19 | 23 | ||
| 24 | + | ||
| 25 | + | ||
| 20 | 26 | ||
| 21 | 27 | ||
| 22 | 28 | ||
| 23 | 29 | ||
| 24 | 30 | ||
| 31 | + | ||
| 32 | + | ||
| 25 | 33 | ||
| 26 | 34 | ||
| 27 | 35 | ||
| @@ -46,6 +54,8 @@ | |||
| 46 | 54 | ||
| 47 | 55 | ||
| 48 | 56 | ||
| 57 | +extern char** environ; | ||
| 58 | + | ||
| 49 | namespace ubse::nodeController { | 59 | namespace ubse::nodeController { |
| 50 | using namespace ubse::context; | 60 | using namespace ubse::context; |
| 51 | using namespace ubse::election; | 61 | using namespace ubse::election; |
| @@ -70,7 +80,15 @@ const std::string SUB_HEALTH_CONF_SECTION = "ubse.memory"; | |||
| 70 | const std::string SUB_HEALTH_ENABLED_KEY = "subHealthPenaltyEnabled"; | 80 | const std::string SUB_HEALTH_ENABLED_KEY = "subHealthPenaltyEnabled"; |
| 71 | const std::string SUB_HEALTH_REFRESH_INTERVAL_KEY = "subHealthRefreshInterval"; | 81 | const std::string SUB_HEALTH_REFRESH_INTERVAL_KEY = "subHealthRefreshInterval"; |
| 72 | constexpr uint32_t SUB_HEALTH_DEFAULT_REFRESH_INTERVAL = 60; | 82 | constexpr uint32_t SUB_HEALTH_DEFAULT_REFRESH_INTERVAL = 60; |
| 73 | -const std::string SUB_HEALTH_DETECTION_FILE = "/var/log/ubse/detection.json"; | 83 | + |
| 84 | +const std::string SUB_HEALTH_HIKPTOOL_BIN = "/usr/bin/hikptool/build/hikptool"; | ||
| 85 | +const std::string SUB_HEALTH_HIKPTOOL_LIB_DIR = "/usr/bin/hikptool/build/libhikptdev/src/rciep"; | ||
| 86 | +const std::string SUB_HEALTH_WORK_DIR = "/var/log/ubse"; | ||
| 87 | +const std::string SUB_HEALTH_DETECTION_FILE = "/var/log/ubse/detection.json"; | ||
| 88 | +constexpr uint32_t SUB_HEALTH_HIKPTOOL_TIMEOUT_SECONDS = 30; | ||
| 89 | +constexpr useconds_t SUB_HEALTH_HIKPTOOL_WAIT_INTERVAL_US = 100000; | ||
| 90 | +constexpr uint32_t SUB_HEALTH_HIKPTOOL_WAIT_COUNT = | ||
| 91 | + (SUB_HEALTH_HIKPTOOL_TIMEOUT_SECONDS + 1) * 1000000 / SUB_HEALTH_HIKPTOOL_WAIT_INTERVAL_US; | ||
| 74 | 92 | ||
| 75 | struct SubHealthPortLocation { | 93 | struct SubHealthPortLocation { |
| 76 | std::string nodeId; | 94 | std::string nodeId; |
| @@ -91,6 +109,208 @@ bool IsSubHealthNodeAvailable(const UbseNodeInfo& nodeInfo) | |||
| 91 | nodeInfo.clusterState != UbseNodeClusterState::UBSE_NODE_FAULT; | 109 | nodeInfo.clusterState != UbseNodeClusterState::UBSE_NODE_FAULT; |
| 92 | } | 110 | } |
| 93 | 111 | ||
| 112 | +uint32_t PrepareSubHealthWorkDir() | ||
| 113 | +{ | ||
| 114 | + if (mkdir(SUB_HEALTH_WORK_DIR.c_str(), 0755) != 0 && errno != EEXIST) { | ||
| 115 | + UBSE_LOG_ERROR << "[SUB_HEALTH] create work directory failed" | ||
| 116 | + << ", dir=" << SUB_HEALTH_WORK_DIR << ", errno=" << errno; | ||
| 117 | + return UBSE_ERROR; | ||
| 118 | + } | ||
| 119 | + | ||
| 120 | + if (access(SUB_HEALTH_WORK_DIR.c_str(), W_OK) != 0) { | ||
| 121 | + UBSE_LOG_ERROR << "[SUB_HEALTH] work directory is not writable" | ||
| 122 | + << ", dir=" << SUB_HEALTH_WORK_DIR << ", errno=" << errno; | ||
| 123 | + return UBSE_ERROR; | ||
| 124 | + } | ||
| 125 | + | ||
| 126 | + return UBSE_OK; | ||
| 127 | +} | ||
| 128 | + | ||
| 129 | +void ReapSubHealthProcessAsync(pid_t pid) | ||
| 130 | +{ | ||
| 131 | + std::thread([pid]() { | ||
| 132 | + int status = 0; | ||
| 133 | + pid_t waitRet = 0; | ||
| 134 | + do { | ||
| 135 | + waitRet = waitpid(pid, &status, 0); | ||
| 136 | + } while (waitRet < 0 && errno == EINTR); | ||
| 137 | + }).detach(); | ||
| 138 | +} | ||
| 139 | + | ||
| 140 | +uint32_t RunSubHealthDetection(const std::atomic<bool>& subHealthEnabled) | ||
| 141 | +{ | ||
| 142 | + struct stat toolStat {}; | ||
| 143 | + if (stat(SUB_HEALTH_HIKPTOOL_BIN.c_str(), &toolStat) != 0 || | ||
| 144 | + !S_ISREG(toolStat.st_mode) || | ||
| 145 | + access(SUB_HEALTH_HIKPTOOL_BIN.c_str(), X_OK) != 0) { | ||
| 146 | + UBSE_LOG_ERROR << "[SUB_HEALTH] hikptool is invalid or not executable" | ||
| 147 | + << ", path=" << SUB_HEALTH_HIKPTOOL_BIN | ||
| 148 | + << ", errno=" << errno; | ||
| 149 | + return UBSE_ERROR; | ||
| 150 | + } | ||
| 151 | + | ||
| 152 | + std::string hikptoolLib = SUB_HEALTH_HIKPTOOL_LIB_DIR + "/libhikptdev.so.1"; | ||
| 153 | + if (access(hikptoolLib.c_str(), R_OK) != 0) { | ||
| 154 | + UBSE_LOG_ERROR << "[SUB_HEALTH] hikptool library is not readable" | ||
| 155 | + << ", path=" << hikptoolLib | ||
| 156 | + << ", errno=" << errno; | ||
| 157 | + return UBSE_ERROR; | ||
| 158 | + } | ||
| 159 | + | ||
| 160 | + auto ret = PrepareSubHealthWorkDir(); | ||
| 161 | + if (ret != UBSE_OK) { | ||
| 162 | + return ret; | ||
| 163 | + } | ||
| 164 | + | ||
| 165 | + // 删除上一轮结果，保证本轮读取的一定是新生成的detection.json | ||
| 166 | + if (unlink(SUB_HEALTH_DETECTION_FILE.c_str()) != 0 && errno != ENOENT) { | ||
| 167 | + UBSE_LOG_ERROR << "[SUB_HEALTH] remove old detection file failed" | ||
| 168 | + << ", file=" << SUB_HEALTH_DETECTION_FILE | ||
| 169 | + << ", errno=" << errno; | ||
| 170 | + return UBSE_ERROR; | ||
| 171 | + } | ||
| 172 | + | ||
| 173 | + std::string ldLibraryPath = "LD_LIBRARY_PATH=" + SUB_HEALTH_HIKPTOOL_LIB_DIR; | ||
| 174 | + const char* oldLdLibraryPath = std::getenv("LD_LIBRARY_PATH"); | ||
| 175 | + if (oldLdLibraryPath != nullptr && oldLdLibraryPath[0] != '\0') { | ||
| 176 | + ldLibraryPath += ":"; | ||
| 177 | + ldLibraryPath += oldLdLibraryPath; | ||
| 178 | + } | ||
| 179 | + | ||
| 180 | + // 在父进程中构造子进程环境，避免fork后调用setenv | ||
| 181 | + std::vector<std::string> envStrings; | ||
| 182 | + for (char** env = environ; env != nullptr && *env != nullptr; ++env) { | ||
| 183 | + std::string item = *env; | ||
| 184 | + if (item.rfind("LD_LIBRARY_PATH=", 0) == 0) { | ||
| 185 | + continue; | ||
| 186 | + } | ||
| 187 | + envStrings.emplace_back(std::move(item)); | ||
| 188 | + } | ||
| 189 | + envStrings.emplace_back(ldLibraryPath); | ||
| 190 | + | ||
| 191 | + std::vector<char*> envp; | ||
| 192 | + envp.reserve(envStrings.size() + 1); | ||
| 193 | + for (auto& item : envStrings) { | ||
| 194 | + envp.push_back(const_cast<char*>(item.c_str())); | ||
| 195 | + } | ||
| 196 | + envp.push_back(nullptr); | ||
| 197 | + | ||
| 198 | + // fork前获取最大文件描述符，子进程仅执行close等安全操作 | ||
| 199 | + long maxFd = sysconf(_SC_OPEN_MAX); | ||
| 200 | + if (maxFd < 0) { | ||
| 201 | + maxFd = 1024; | ||
| 202 | + } | ||
| 203 | + | ||
| 204 | + UBSE_LOG_INFO << "[SUB_HEALTH] start hikptool detection" | ||
| 205 | + << ", hikptool=" << SUB_HEALTH_HIKPTOOL_BIN | ||
| 206 | + << ", libraryDir=" << SUB_HEALTH_HIKPTOOL_LIB_DIR | ||
| 207 | + << ", workDir=" << SUB_HEALTH_WORK_DIR; | ||
| 208 | + | ||
| 209 | + pid_t pid = fork(); | ||
| 210 | + if (pid < 0) { | ||
| 211 | + UBSE_LOG_ERROR << "[SUB_HEALTH] fork hikptool failed" | ||
| 212 | + << ", errno=" << errno; | ||
| 213 | + return UBSE_ERROR; | ||
| 214 | + } | ||
| 215 | + | ||
| 216 | + if (pid == 0) { | ||
 | |||
| 217 | + if (chdir(SUB_HEALTH_WORK_DIR.c_str()) != 0) { | ||
| 218 | + _exit(126); | ||
| 219 | + } | ||
| 220 | + | ||
| 221 | + // 避免hikptool继承UBSE的socket、UDS、epoll及内部IPC文件描述符 | ||
| 222 | + for (int fd = STDERR_FILENO + 1; fd < maxFd; ++fd) { | ||
| 223 | + (void)close(fd); | ||
| 224 | + } | ||
| 225 | + | ||
| 226 | + // hikptool最多执行30秒，避免工具异常卡死 | ||
| 227 | + alarm(SUB_HEALTH_HIKPTOOL_TIMEOUT_SECONDS); | ||
| 228 | + | ||
| 229 | + char* const argv[] = { | ||
| 230 | + const_cast<char*>(SUB_HEALTH_HIKPTOOL_BIN.c_str()), | ||
| 231 | + const_cast<char*>("sub_health"), | ||
| 232 | + nullptr, | ||
| 233 | + }; | ||
| 234 | + | ||
| 235 | + execve(SUB_HEALTH_HIKPTOOL_BIN.c_str(), argv, envp.data()); | ||
| 236 | + _exit(127); | ||
| 237 | + } | ||
| 238 | + | ||
| 239 | + int status = 0; | ||
| 240 | + bool processExited = false; | ||
| 241 | + | ||
| 242 | + for (uint32_t count = 0; count < SUB_HEALTH_HIKPTOOL_WAIT_COUNT; ++count) { | ||
| 243 | + pid_t waitRet = waitpid(pid, &status, WNOHANG); | ||
| 244 | + if (waitRet == pid) { | ||
| 245 | + processExited = true; | ||
| 246 | + break; | ||
| 247 | + } | ||
| 248 | + | ||
| 249 | + if (waitRet < 0) { | ||
| 250 | + if (errno == EINTR) { | ||
| 251 | + continue; | ||
| 252 | + } | ||
| 253 | + | ||
| 254 | + UBSE_LOG_ERROR << "[SUB_HEALTH] wait hikptool failed" | ||
| 255 | + << ", pid=" << pid | ||
| 256 | + << ", errno=" << errno; | ||
| 257 | + | ||
| 258 | + if (errno != ECHILD) { | ||
| 259 | + ReapSubHealthProcessAsync(pid); | ||
| 260 | + } | ||
| 261 | + return UBSE_ERROR; | ||
| 262 | + } | ||
| 263 | + | ||
| 264 | + // UBSE停止时不继续阻塞executor，由独立线程负责回收子进程 | ||
| 265 | + if (!subHealthEnabled.load()) { | ||
| 266 | + UBSE_LOG_INFO << "[SUB_HEALTH] sub health is stopping" | ||
| 267 | + << ", pid=" << pid; | ||
| 268 | + ReapSubHealthProcessAsync(pid); | ||
| 269 | + return UBSE_ERROR; | ||
| 270 | + } | ||
| 271 | + | ||
| 272 | + usleep(SUB_HEALTH_HIKPTOOL_WAIT_INTERVAL_US); | ||
| 273 | + } | ||
| 274 | + | ||
| 275 | + if (!processExited) { | ||
异常路径不终止/回收子进程，存在僵尸进程泄漏与关闭阻塞风险 。超时路径（L246-248）和 waitpid 错误路径（L231-236）直接 return UBSE_ERROR ，未 kill(pid, SIGKILL) 也未阻塞回收。若 hikptool 卡在 D 态（硬件诊断工具的典型故障模式）或 exec 后自装 SIGALRM 处理器导致 alarm 失效，子进程将永久滞留或变成僵尸（全库无 SIGCHLD 处理，无人回收，默认 60s 刷新周期每次泄漏一个）。同时停止路径（L238-241）发现 subHealthEnabled=false 后仅打日志继续等待——不退出循环、不 kill，而 UbseTaskExecutor::Stop() 会 join 工作线程（已核实 ubse_thread_pool.cpp ），守护进程退出最多阻塞 31s，且该日志每 100ms 打一条、单次停止最多刷 ~310 行   | |||
| 276 | + UBSE_LOG_ERROR << "[SUB_HEALTH] wait hikptool timeout" | ||
| 277 | + << ", pid=" << pid | ||
| 278 | + << ", timeout=" << SUB_HEALTH_HIKPTOOL_TIMEOUT_SECONDS << "s"; | ||
| 279 | + ReapSubHealthProcessAsync(pid); | ||
| 280 | + return UBSE_ERROR; | ||
| 281 | + } | ||
| 282 | + | ||
| 283 | + if (!WIFEXITED(status)) { | ||
| 284 | + UBSE_LOG_ERROR << "[SUB_HEALTH] hikptool exited abnormally" | ||
| 285 | + << ", pid=" << pid; | ||
| 286 | + return UBSE_ERROR; | ||
| 287 | + } | ||
| 288 | + | ||
| 289 | + int exitCode = WEXITSTATUS(status); | ||
| 290 | + if (exitCode != 0) { | ||
| 291 | + UBSE_LOG_ERROR << "[SUB_HEALTH] hikptool detection failed" | ||
| 292 | + << ", pid=" << pid | ||
| 293 | + << ", exitCode=" << exitCode; | ||
| 294 | + return UBSE_ERROR; | ||
| 295 | + } | ||
| 296 | + | ||
| 297 | + if (!subHealthEnabled.load()) { | ||
| 298 | + return UBSE_ERROR; | ||
| 299 | + } | ||
| 300 | + | ||
| 301 | + if (access(SUB_HEALTH_DETECTION_FILE.c_str(), R_OK) != 0) { | ||
| 302 | + UBSE_LOG_ERROR << "[SUB_HEALTH] detection.json was not generated" | ||
| 303 | + << ", file=" << SUB_HEALTH_DETECTION_FILE | ||
| 304 | + << ", errno=" << errno; | ||
| 305 | + return UBSE_ERROR; | ||
| 306 | + } | ||
| 307 | + | ||
| 308 | + UBSE_LOG_INFO << "[SUB_HEALTH] hikptool detection success" | ||
| 309 | + << ", file=" << SUB_HEALTH_DETECTION_FILE; | ||
| 310 | + | ||
| 311 | + return UBSE_OK; | ||
| 312 | +} | ||
| 313 | + | ||
| 94 | size_t UbseNodeController::SubHealthSocketPairHash::operator()(const SubHealthSocketPair& key) const | 314 | size_t UbseNodeController::SubHealthSocketPairHash::operator()(const SubHealthSocketPair& key) const |
| 95 | { | 315 | { |
| 96 | size_t hash = std::hash<std::string>{}(key.importNodeId); | 316 | size_t hash = std::hash<std::string>{}(key.importNodeId); |
| @@ -527,16 +747,16 @@ uint32_t UbseNodeController::RegGlobalStateNotifyHandler(const UbseGlobalStateNo | |||
| 527 | return UBSE_OK; | 747 | return UBSE_OK; |
| 528 | } | 748 | } |
| 529 | 749 | ||
| 530 | -uint32_t ParseSubHealthLinkArray( | 750 | +uint32_t ParseSubHealthLinkArray(const rapidjson::Value& result, const std::string& srcEid, |
| 531 | - const rapidjson::Value& result, const std::string& srcEid, const SubHealthPortLocation& srcLocation, | 751 | + const SubHealthPortLocation& srcLocation, |
| 532 | - const std::unordered_map<std::string, SubHealthPortLocation>& portLocations, | 752 | + const std::unordered_map<std::string, SubHealthPortLocation>& portLocations, |
| 533 | - UbseSubHealthFlagMap& subHealthFlags, size_t& linkCount) | 753 | + UbseSubHealthFlagMap& subHealthFlags, size_t& linkCount) |
| 534 | { | 754 | { |
| 535 | const char* dstEidsKey = "sub_health_dst_eids"; | 755 | const char* dstEidsKey = "sub_health_dst_eids"; |
| 536 | const char* latenciesKey = "sub_health_latencies"; | 756 | const char* latenciesKey = "sub_health_latencies"; |
| 537 | 757 | ||
| 538 | - if (!result.HasMember(dstEidsKey) || !result[dstEidsKey].IsArray() || | 758 | + if (!result.HasMember(dstEidsKey) || !result[dstEidsKey].IsArray() || !result.HasMember(latenciesKey) || |
| 539 | - !result.HasMember(latenciesKey) || !result[latenciesKey].IsArray()) { | 759 | + !result[latenciesKey].IsArray()) { |
| 540 | UBSE_LOG_ERROR << "[SUB_HEALTH] invalid detection result" | 760 | UBSE_LOG_ERROR << "[SUB_HEALTH] invalid detection result" |
| 541 | << ", srcEid=" << srcEid; | 761 | << ", srcEid=" << srcEid; |
| 542 | return UBSE_ERROR_INVAL; | 762 | return UBSE_ERROR_INVAL; |
| @@ -547,8 +767,7 @@ uint32_t ParseSubHealthLinkArray( | |||
| 547 | 767 | ||
| 548 | if (dstEids.Size() != latencies.Size()) { | 768 | if (dstEids.Size() != latencies.Size()) { |
| 549 | UBSE_LOG_ERROR << "[SUB_HEALTH] dst eid and latency size mismatch" | 769 | UBSE_LOG_ERROR << "[SUB_HEALTH] dst eid and latency size mismatch" |
| 550 | - << ", srcEid=" << srcEid | 770 | + << ", srcEid=" << srcEid << ", dstCount=" << dstEids.Size() |
| 551 | - << ", dstCount=" << dstEids.Size() | ||
| 552 | << ", latencyCount=" << latencies.Size(); | 771 | << ", latencyCount=" << latencies.Size(); |
| 553 | return UBSE_ERROR_INVAL; | 772 | return UBSE_ERROR_INVAL; |
| 554 | } | 773 | } |
| @@ -556,8 +775,7 @@ uint32_t ParseSubHealthLinkArray( | |||
| 556 | for (rapidjson::SizeType i = 0; i < dstEids.Size(); ++i) { | 775 | for (rapidjson::SizeType i = 0; i < dstEids.Size(); ++i) { |
| 557 | if (!dstEids[i].IsString() || !latencies[i].IsNumber()) { | 776 | if (!dstEids[i].IsString() || !latencies[i].IsNumber()) { |
| 558 | UBSE_LOG_ERROR << "[SUB_HEALTH] invalid link item" | 777 | UBSE_LOG_ERROR << "[SUB_HEALTH] invalid link item" |
| 559 | - << ", srcEid=" << srcEid | 778 | + << ", srcEid=" << srcEid << ", index=" << i; |
| 560 | - << ", index=" << i; | ||
| 561 | return UBSE_ERROR_INVAL; | 779 | return UBSE_ERROR_INVAL; |
| 562 | } | 780 | } |
| 563 | 781 | ||
| @@ -565,9 +783,7 @@ uint32_t ParseSubHealthLinkArray( | |||
| 565 | auto dstIter = portLocations.find(NormalizeSubHealthEid(dstEid)); | 783 | auto dstIter = portLocations.find(NormalizeSubHealthEid(dstEid)); |
| 566 | if (dstIter == portLocations.end()) { | 784 | if (dstIter == portLocations.end()) { |
| 567 | UBSE_LOG_WARN << "[SUB_HEALTH] destination primary eid not found in topology" | 785 | UBSE_LOG_WARN << "[SUB_HEALTH] destination primary eid not found in topology" |
| 568 | - << ", srcEid=" << srcEid | 786 | + << ", srcEid=" << srcEid << ", dstEid=" << dstEid << ", keep old cache"; |
| 569 | - << ", dstEid=" << dstEid | ||
| 570 | - << ", keep old cache"; | ||
| 571 | return UBSE_ERROR; | 787 | return UBSE_ERROR; |
| 572 | } | 788 | } |
| 573 | 789 | ||
| @@ -608,13 +824,9 @@ uint32_t ParseSubHealthLinkArray( | |||
| 608 | 824 | ||
| 609 | if (isSubHealthy) { | 825 | if (isSubHealthy) { |
| 610 | UBSE_LOG_INFO << "[SUB_HEALTH] sub healthy link detected" | 826 | UBSE_LOG_INFO << "[SUB_HEALTH] sub healthy link detected" |
| 611 | - << ", srcEid=" << srcEid | 827 | + << ", srcEid=" << srcEid << ", dstEid=" << dstEid << ", latency=" << latency << "ms" |
| 612 | - << ", dstEid=" << dstEid | 828 | + << ", nodeId=" << srcLocation.nodeId << ", socketId=" << srcLocation.socketId |
| 613 | - << ", latency=" << latency << "ms" | 829 | + << ", peerNodeId=" << dstLocation.nodeId << ", peerSocketId=" << dstLocation.socketId |
| 614 | - << ", nodeId=" << srcLocation.nodeId | ||
| 615 | - << ", socketId=" << srcLocation.socketId | ||
| 616 | - << ", peerNodeId=" << dstLocation.nodeId | ||
| 617 | - << ", peerSocketId=" << dstLocation.socketId | ||
| 618 | << ", cacheDirection=bidirectional"; | 830 | << ", cacheDirection=bidirectional"; |
| 619 | } | 831 | } |
| 620 | } | 832 | } |
| @@ -627,8 +839,7 @@ uint32_t UbseNodeController::LoadSubHealthDetection(UbseSubHealthFlagMap& subHea | |||
| 627 | std::ifstream file(SUB_HEALTH_DETECTION_FILE); | 839 | std::ifstream file(SUB_HEALTH_DETECTION_FILE); |
| 628 | if (!file.is_open()) { | 840 | if (!file.is_open()) { |
| 629 | UBSE_LOG_WARN << "[SUB_HEALTH] open detection file failed" | 841 | UBSE_LOG_WARN << "[SUB_HEALTH] open detection file failed" |
| 630 | - << ", file=" << SUB_HEALTH_DETECTION_FILE | 842 | + << ", file=" << SUB_HEALTH_DETECTION_FILE << ", keep old cache"; |
| 631 | - << ", keep old cache"; | ||
| 632 | return UBSE_ERROR; | 843 | return UBSE_ERROR; |
| 633 | } | 844 | } |
| 634 | 845 | ||
| @@ -638,17 +849,14 @@ uint32_t UbseNodeController::LoadSubHealthDetection(UbseSubHealthFlagMap& subHea | |||
| 638 | 849 | ||
| 639 | if (document.HasParseError()) { | 850 | if (document.HasParseError()) { |
| 640 | UBSE_LOG_ERROR << "[SUB_HEALTH] parse detection file failed" | 851 | UBSE_LOG_ERROR << "[SUB_HEALTH] parse detection file failed" |
| 641 | - << ", file=" << SUB_HEALTH_DETECTION_FILE | 852 | + << ", file=" << SUB_HEALTH_DETECTION_FILE << ", offset=" << document.GetErrorOffset() |
| 642 | - << ", offset=" << document.GetErrorOffset() | 853 | + << ", error=" << rapidjson::GetParseError_En(document.GetParseError()) << ", keep old cache"; |
| 643 | - << ", error=" << rapidjson::GetParseError_En(document.GetParseError()) | ||
| 644 | - << ", keep old cache"; | ||
| 645 | return UBSE_ERROR_INVAL; | 854 | return UBSE_ERROR_INVAL; |
| 646 | } | 855 | } |
| 647 | 856 | ||
| 648 | if (!document.IsObject()) { | 857 | if (!document.IsObject()) { |
| 649 | UBSE_LOG_ERROR << "[SUB_HEALTH] detection root is not object" | 858 | UBSE_LOG_ERROR << "[SUB_HEALTH] detection root is not object" |
| 650 | - << ", file=" << SUB_HEALTH_DETECTION_FILE | 859 | + << ", file=" << SUB_HEALTH_DETECTION_FILE << ", keep old cache"; |
| 651 | - << ", keep old cache"; | ||
| 652 | return UBSE_ERROR_INVAL; | 860 | return UBSE_ERROR_INVAL; |
| 653 | } | 861 | } |
| 654 | 862 | ||
| @@ -682,12 +890,9 @@ uint32_t UbseNodeController::LoadSubHealthDetection(UbseSubHealthFlagMap& subHea | |||
| 682 | if (iter != portLocations.end()) { | 890 | if (iter != portLocations.end()) { |
| 683 | if (iter->second.nodeId != nodeId || iter->second.socketId != cpuInfo.socketId) { | 891 | if (iter->second.nodeId != nodeId || iter->second.socketId != cpuInfo.socketId) { |
| 684 | UBSE_LOG_ERROR << "[SUB_HEALTH] duplicated primary eid" | 892 | UBSE_LOG_ERROR << "[SUB_HEALTH] duplicated primary eid" |
| 685 | - << ", eid=" << cpuInfo.primaryEid | 893 | + << ", eid=" << cpuInfo.primaryEid << ", oldNodeId=" << iter->second.nodeId |
| 686 | - << ", oldNodeId=" << iter->second.nodeId | 894 | + << ", oldSocketId=" << iter->second.socketId << ", newNodeId=" << nodeId |
| 687 | - << ", oldSocketId=" << iter->second.socketId | 895 | + << ", newSocketId=" << cpuInfo.socketId << ", keep old cache"; |
| 688 | - << ", newNodeId=" << nodeId | ||
| 689 | - << ", newSocketId=" << cpuInfo.socketId | ||
| 690 | - << ", keep old cache"; | ||
| 691 | return UBSE_ERROR; | 896 | return UBSE_ERROR; |
| 692 | } | 897 | } |
| 693 | continue; | 898 | continue; |
| @@ -700,8 +905,7 @@ uint32_t UbseNodeController::LoadSubHealthDetection(UbseSubHealthFlagMap& subHea | |||
| 700 | }; | 905 | }; |
| 701 | 906 | ||
| 702 | UBSE_LOG_INFO << "[SUB_HEALTH] primary eid mapping" | 907 | UBSE_LOG_INFO << "[SUB_HEALTH] primary eid mapping" |
| 703 | - << ", nodeId=" << nodeId | 908 | + << ", nodeId=" << nodeId << ", socketId=" << cpuInfo.socketId |
| 704 | - << ", socketId=" << cpuInfo.socketId | ||
| 705 | << ", primaryEid=" << cpuInfo.primaryEid; | 909 | << ", primaryEid=" << cpuInfo.primaryEid; |
| 706 | } | 910 | } |
| 707 | } | 911 | } |
| @@ -712,8 +916,7 @@ uint32_t UbseNodeController::LoadSubHealthDetection(UbseSubHealthFlagMap& subHea | |||
| 712 | } | 916 | } |
| 713 | 917 | ||
| 714 | UBSE_LOG_INFO << "[SUB_HEALTH] build primary eid mapping success" | 918 | UBSE_LOG_INFO << "[SUB_HEALTH] build primary eid mapping success" |
| 715 | - << ", nodeCount=" << allNodes.size() | 919 | + << ", nodeCount=" << allNodes.size() << ", eidCount=" << portLocations.size(); |
| 716 | - << ", eidCount=" << portLocations.size(); | ||
| 717 | 920 | ||
| 718 | size_t sourceCount = 0; | 921 | size_t sourceCount = 0; |
| 719 | size_t linkCount = 0; | 922 | size_t linkCount = 0; |
| @@ -723,8 +926,7 @@ uint32_t UbseNodeController::LoadSubHealthDetection(UbseSubHealthFlagMap& subHea | |||
| 723 | 926 | ||
| 724 | if (!hostIter->value.IsObject()) { | 927 | if (!hostIter->value.IsObject()) { |
| 725 | UBSE_LOG_ERROR << "[SUB_HEALTH] host detection result is not object" | 928 | UBSE_LOG_ERROR << "[SUB_HEALTH] host detection result is not object" |
| 726 | - << ", host=" << host | 929 | + << ", host=" << host << ", keep old cache"; |
| 727 | - << ", keep old cache"; | ||
| 728 | return UBSE_ERROR_INVAL; | 930 | return UBSE_ERROR_INVAL; |
| 729 | } | 931 | } |
| 730 | 932 | ||
| @@ -733,8 +935,7 @@ uint32_t UbseNodeController::LoadSubHealthDetection(UbseSubHealthFlagMap& subHea | |||
| 733 | for (auto socketIter = socketResults.MemberBegin(); socketIter != socketResults.MemberEnd(); ++socketIter) { | 935 | for (auto socketIter = socketResults.MemberBegin(); socketIter != socketResults.MemberEnd(); ++socketIter) { |
| 734 | if (!socketIter->value.IsObject()) { | 936 | if (!socketIter->value.IsObject()) { |
| 735 | UBSE_LOG_ERROR << "[SUB_HEALTH] socket detection result is not object" | 937 | UBSE_LOG_ERROR << "[SUB_HEALTH] socket detection result is not object" |
| 736 | - << ", host=" << host | 938 | + << ", host=" << host << ", socket=" << socketIter->name.GetString() |
| 737 | - << ", socket=" << socketIter->name.GetString() | ||
| 738 | << ", keep old cache"; | 939 | << ", keep old cache"; |
| 739 | return UBSE_ERROR_INVAL; | 940 | return UBSE_ERROR_INVAL; |
| 740 | } | 941 | } |
| @@ -743,8 +944,7 @@ uint32_t UbseNodeController::LoadSubHealthDetection(UbseSubHealthFlagMap& subHea | |||
| 743 | 944 | ||
| 744 | if (!result.HasMember("src_eid") || !result["src_eid"].IsString()) { | 945 | if (!result.HasMember("src_eid") || !result["src_eid"].IsString()) { |
| 745 | UBSE_LOG_ERROR << "[SUB_HEALTH] src_eid is invalid" | 946 | UBSE_LOG_ERROR << "[SUB_HEALTH] src_eid is invalid" |
| 746 | - << ", host=" << host | 947 | + << ", host=" << host << ", socket=" << socketIter->name.GetString() |
| 747 | - << ", socket=" << socketIter->name.GetString() | ||
| 748 | << ", keep old cache"; | 948 | << ", keep old cache"; |
| 749 | return UBSE_ERROR_INVAL; | 949 | return UBSE_ERROR_INVAL; |
| 750 | } | 950 | } |
| @@ -754,17 +954,15 @@ uint32_t UbseNodeController::LoadSubHealthDetection(UbseSubHealthFlagMap& subHea | |||
| 754 | auto srcIter = portLocations.find(NormalizeSubHealthEid(srcEid)); | 954 | auto srcIter = portLocations.find(NormalizeSubHealthEid(srcEid)); |
| 755 | if (srcIter == portLocations.end()) { | 955 | if (srcIter == portLocations.end()) { |
| 756 | UBSE_LOG_WARN << "[SUB_HEALTH] source primary eid not found in topology" | 956 | UBSE_LOG_WARN << "[SUB_HEALTH] source primary eid not found in topology" |
| 757 | - << ", host=" << host | 957 | + << ", host=" << host << ", socket=" << socketIter->name.GetString() |
| 758 | - << ", socket=" << socketIter->name.GetString() | 958 | + << ", srcEid=" << srcEid << ", keep old cache"; |
| 759 | - << ", srcEid=" << srcEid | ||
| 760 | - << ", keep old cache"; | ||
| 761 | return UBSE_ERROR; | 959 | return UBSE_ERROR; |
| 762 | } | 960 | } |
| 763 | 961 | ||
| 764 | ++sourceCount; | 962 | ++sourceCount; |
| 765 | 963 | ||
| 766 | - auto ret = ParseSubHealthLinkArray(result, srcEid, srcIter->second, portLocations, subHealthFlags, | 964 | + auto ret = |
| 767 | - linkCount); | 965 | + ParseSubHealthLinkArray(result, srcEid, srcIter->second, portLocations, subHealthFlags, linkCount); |
| 768 | if (ret != UBSE_OK) { | 966 | if (ret != UBSE_OK) { |
| 769 | return ret; | 967 | return ret; |
| 770 | } | 968 | } |
| @@ -779,10 +977,8 @@ uint32_t UbseNodeController::LoadSubHealthDetection(UbseSubHealthFlagMap& subHea | |||
| 779 | } | 977 | } |
| 780 | 978 | ||
| 781 | UBSE_LOG_INFO << "[SUB_HEALTH] load detection success" | 979 | UBSE_LOG_INFO << "[SUB_HEALTH] load detection success" |
| 782 | - << ", file=" << SUB_HEALTH_DETECTION_FILE | 980 | + << ", file=" << SUB_HEALTH_DETECTION_FILE << ", sourceCount=" << sourceCount |
| 783 | - << ", sourceCount=" << sourceCount | 981 | + << ", linkCount=" << linkCount << ", socketPairCount=" << subHealthFlags.size() |
| 784 | - << ", linkCount=" << linkCount | ||
| 785 | - << ", socketPairCount=" << subHealthFlags.size() | ||
| 786 | << ", subHealthyPairCount=" << subHealthyCount; | 982 | << ", subHealthyPairCount=" << subHealthyCount; |
| 787 | 983 | ||
| 788 | return UBSE_OK; | 984 | return UBSE_OK; |
| @@ -858,21 +1054,30 @@ uint32_t UbseNodeController::RefreshSubHealthCache() | |||
| 858 | return UBSE_OK; | 1054 | return UBSE_OK; |
| 859 | } | 1055 | } |
| 860 | 1056 | ||
| 861 | - UBSE_LOG_INFO << "[SUB_HEALTH] start refresh" | 1057 | + UBSE_LOG_INFO << "[SUB_HEALTH] start refresh"; |
| 862 | - << ", file=" << SUB_HEALTH_DETECTION_FILE; | 1058 | + |
| 1059 | + // 刷新缓存前先执行hikptool生成最新检测结果 | ||
| 1060 | + auto ret = RunSubHealthDetection(subHealthEnabled_); | ||
| 1061 | + if (ret != UBSE_OK) { | ||
| 1062 | + if (!subHealthEnabled_.load()) { | ||
| 1063 | + UBSE_LOG_INFO << "[SUB_HEALTH] detection stopped because sub health is stopping"; | ||
| 1064 | + return UBSE_OK; | ||
| 1065 | + } | ||
| 1066 | + | ||
| 1067 | + UBSE_LOG_WARN << "[SUB_HEALTH] run detection failed, keep old cache, " << FormatRetCode(ret); | ||
| 1068 | + return ret; | ||
| 1069 | + } | ||
| 863 | 1070 | ||
| 864 | UbseSubHealthFlagMap subHealthFlags; | 1071 | UbseSubHealthFlagMap subHealthFlags; |
| 865 | - auto ret = LoadSubHealthDetection(subHealthFlags); | 1072 | + ret = LoadSubHealthDetection(subHealthFlags); |
| 866 | if (ret != UBSE_OK) { | 1073 | if (ret != UBSE_OK) { |
| 867 | - UBSE_LOG_WARN << "[SUB_HEALTH] load detection failed, keep old cache, " | 1074 | + UBSE_LOG_WARN << "[SUB_HEALTH] load detection failed, keep old cache, " << FormatRetCode(ret); |
| 868 | - << FormatRetCode(ret); | ||
| 869 | return ret; | 1075 | return ret; |
| 870 | } | 1076 | } |
| 871 | 1077 | ||
| 872 | ret = UpdateSubHealthCache(subHealthFlags); | 1078 | ret = UpdateSubHealthCache(subHealthFlags); |
| 873 | if (ret != UBSE_OK) { | 1079 | if (ret != UBSE_OK) { |
| 874 | - UBSE_LOG_WARN << "[SUB_HEALTH] update cache failed, keep old cache, " | 1080 | + UBSE_LOG_WARN << "[SUB_HEALTH] update cache failed, keep old cache, " << FormatRetCode(ret); |
| 875 | - << FormatRetCode(ret); | ||
| 876 | return ret; | 1081 | return ret; |
| 877 | } | 1082 | } |
| 878 | 1083 | ||
子进程继承守护进程全部文件描述符 。全 src 目录 grep 无任何 O_CLOEXEC / SOCK_CLOEXEC / FD_CLOEXEC / posix_spawn 使用，fork 后 exec 前也未关闭继承的 fd。hikptool 将持有守护进程的监听 socket、UDS、epoll、日志 fd 长达 30s：若此窗口内守护进程重启，新进程可能因 hikptool 占用 socket/UDS 而绑定失败；同时内部 IPC fd 泄漏给外部工具