已合并
[Ubse] #feat# IT框架支持 1.测试场景初始启动部分节点 2.测试用例 支持停止节点,修改配置后启动节点 #1506
朱程成创建于 8月26日
[Ubse] #feat# IT框架支持 1.测试场景初始启动部分节点 2.测试用例 支持停止节点,修改配置后启动节点 #1506
已合并
共 14 个文件变更+446-4
| @@ -203,6 +203,7 @@ set(UBSE_IT_SCENARIOS | |||
| 203 | tongsuan_1d_four_nodes_two_group | 203 | tongsuan_1d_four_nodes_two_group |
| 204 | tongsuan_1d_four_nodes_boundary_hostname | 204 | tongsuan_1d_four_nodes_boundary_hostname |
| 205 | tongsuan_1d_four_nodes_candidate_nodes | 205 | tongsuan_1d_four_nodes_candidate_nodes |
| 206 | + tongsuan_1d_four_nodes_deferred_start | ||
| 206 | zhisuan_npu_single_node | 207 | zhisuan_npu_single_node |
| 207 | ) | 208 | ) |
| 208 | 209 | ||
| @@ -53,6 +53,14 @@ ItNode::ClusterContext ItCluster::BuildClusterContext() const | |||
| 53 | 53 | ||
| 54 | UbseResult ItCluster::StartCluster(bool waitForElection, uint32_t electionTimeoutMs) | 54 | UbseResult ItCluster::StartCluster(bool waitForElection, uint32_t electionTimeoutMs) |
| 55 | { | 55 | { |
| 56 | + // 校验 deferred 节点 ID 必须存在于拓扑中, 尽早暴露 DeferNodes 配置错误 | ||
| 57 | + for (const auto& nodeId : clusterSpec_.deferredNodeIds) { | ||
| 58 | + if (clusterSpec_.FindNode(nodeId) == nullptr) { | ||
| 59 | + IT_LOG_ERROR << "Deferred node " << nodeId << " not in topology, check DeferNodes()"; | ||
| 60 | + return UBSE_ERROR_DEF(1); | ||
| 61 | + } | ||
| 62 | + } | ||
| 63 | + | ||
| 56 | // Phase 0: Generate per-node config files | 64 | // Phase 0: Generate per-node config files |
| 57 | ItConfigBuilder configBuilder(clusterSpec_.nodes, clusterSpec_.baseWorkDir); | 65 | ItConfigBuilder configBuilder(clusterSpec_.nodes, clusterSpec_.baseWorkDir); |
| 58 | // Apply per-node config overrides (e.g. election.candidate per node) | 66 | // Apply per-node config overrides (e.g. election.candidate per node) |
| @@ -74,8 +82,14 @@ UbseResult ItCluster::StartCluster(bool waitForElection, uint32_t electionTimeou | |||
| 74 | } | 82 | } |
| 75 | 83 | ||
| 76 | // Phase 1: Create ItNode instances and start all nodes simultaneously | 84 | // Phase 1: Create ItNode instances and start all nodes simultaneously |
| 85 | + // Deferred nodes (DeferNodes) are skipped here: their config was generated in | ||
| 86 | + // Phase 0, but the daemon is only started later via StartNode(). | ||
| 77 | auto ctx = BuildClusterContext(); | 87 | auto ctx = BuildClusterContext(); |
| 78 | for (const auto& cfg : clusterSpec_.nodes) { | 88 | for (const auto& cfg : clusterSpec_.nodes) { |
| 89 | + if (clusterSpec_.deferredNodeIds.count(cfg.nodeId) != 0) { | ||
| 90 | + IT_LOG_INFO << "Node " << cfg.nodeId << " is deferred, skip auto-start (use StartNode to start it)"; | ||
| 91 | + continue; | ||
| 92 | + } | ||
| 79 | auto node = std::make_unique<ItNode>(cfg, ctx); | 93 | auto node = std::make_unique<ItNode>(cfg, ctx); |
| 80 | ret = node->Start(); | 94 | ret = node->Start(); |
| 81 | if (ret != UBSE_OK) { | 95 | if (ret != UBSE_OK) { |
| @@ -361,6 +375,96 @@ UbseResult ItCluster::RestartNode(const std::string& nodeId, bool waitForElectio | |||
| 361 | return UBSE_OK; | 375 | return UBSE_OK; |
| 362 | } | 376 | } |
| 363 | 377 | ||
| 378 | +UbseResult ItCluster::StartNode(const std::string& nodeId, | ||
| 379 | + const std::map<std::string, std::map<std::string, std::string>>& configOverrides, | ||
| 380 | + bool waitForElection, uint32_t electionTimeoutMs) | ||
| 381 | +{ | ||
| 382 | + // 节点已启动则直接返回成功 (幂等) | ||
| 383 | + if (nodes_.find(nodeId) != nodes_.end()) { | ||
| 384 | + IT_LOG_WARN << "StartNode: node " << nodeId << " already running"; | ||
| 385 | + return UBSE_OK; | ||
| 386 | + } | ||
| 387 | + // 节点必须在拓扑中 | ||
| 388 | + const auto* cfg = clusterSpec_.FindNode(nodeId); | ||
| 389 | + if (cfg == nullptr) { | ||
| 390 | + IT_LOG_ERROR << "StartNode: node " << nodeId << " not in topology"; | ||
| 391 | + return UBSE_ERROR_DEF(1); | ||
| 392 | + } | ||
| 393 | + // 节点通常应是 deferred 的 (StartCluster 跳过启动); 这里不强制报错, 只 warn, | ||
| 394 | + // 允许 case 在 StopNode 后用 StartNode 重新启动 (修改配置后重启节点) | ||
| 395 | + if (clusterSpec_.deferredNodeIds.count(nodeId) == 0) { | ||
| 396 | + IT_LOG_WARN << "StartNode: node " << nodeId << " is not in deferredNodeIds"; | ||
| 397 | + } | ||
| 398 | + | ||
| 399 | + // 若提供了 configOverrides, 在启动前重新生成该节点配置 (覆盖 StartCluster 阶段生成的默认配置) | ||
| 400 | + if (!configOverrides.empty()) { | ||
| 401 | + ItConfigBuilder configBuilder(clusterSpec_.nodes, clusterSpec_.baseWorkDir); | ||
| 402 | + configBuilder.WithClusterIps(clusterSpec_.ClusterIps()) | ||
| 403 | + .WithCertUse(false) | ||
| 404 | + .WithMockPlugin(clusterSpec_.mockPluginEnabled); | ||
| 405 | + // 先还原 StartCluster 阶段为该节点预置的 override | ||
| 406 | + for (const auto& [section, keys] : clusterSpec_.nodeConfigOverrides[nodeId]) { | ||
| 407 | + for (const auto& [key, value] : keys) { | ||
| 408 | + configBuilder.WithNodeOverride(nodeId, section, key, value); | ||
| 409 | + } | ||
| 410 | + } | ||
| 411 | + // 再应用 case 提供的 override (后应用者覆盖前者, case 优先级最高) | ||
| 412 | + for (const auto& [section, keys] : configOverrides) { | ||
| 413 | + for (const auto& [key, value] : keys) { | ||
| 414 | + configBuilder.WithNodeOverride(nodeId, section, key, value); | ||
| 415 | + } | ||
| 416 | + } | ||
| 417 | + UbseResult ret = configBuilder.GenerateNodeConfig(nodeId); | ||
| 418 | + if (ret != UBSE_OK) { | ||
| 419 | + IT_LOG_ERROR << "StartNode: failed to regenerate config for node " << nodeId; | ||
| 420 | + return ret; | ||
| 421 | + } | ||
| 422 | + IT_LOG_INFO << "StartNode: regenerated config for node " << nodeId << " with " << configOverrides.size() | ||
| 423 | + << " sections of overrides"; | ||
| 424 | + } | ||
| 425 | + | ||
| 426 | + auto ctx = BuildClusterContext(); | ||
| 427 | + auto node = std::make_unique<ItNode>(*cfg, ctx); | ||
| 428 | + UbseResult ret = node->Start(); | ||
| 429 | + if (ret != UBSE_OK) { | ||
| 430 | + IT_LOG_ERROR << "Failed to start node " << nodeId; | ||
| 431 | + return ret; | ||
| 432 | + } | ||
| 433 | + ret = node->WaitForStartup(clusterSpec_.startupTimeoutMs); | ||
| 434 | + if (ret != UBSE_OK) { | ||
| 435 | + IT_LOG_ERROR << "Node " << nodeId << " did not start up in time"; | ||
| 436 | + return ret; | ||
| 437 | + } | ||
| 438 | + ret = node->InitializeSdkClient(); | ||
| 439 | + if (ret != UBSE_OK) { | ||
| 440 | + IT_LOG_WARN << "Failed to initialize SDK client for node " << nodeId; | ||
| 441 | + } | ||
| 442 | + nodes_[nodeId] = std::move(node); | ||
| 443 | + IT_LOG_INFO << "Node " << nodeId << " started via StartNode"; | ||
| 444 | + | ||
| 445 | + if (waitForElection) { | ||
| 446 | + ret = WaitForElectionConvergence(electionTimeoutMs); | ||
| 447 | + if (ret != UBSE_OK) { | ||
| 448 | + IT_LOG_ERROR << "Election did not converge after starting node " << nodeId; | ||
| 449 | + return ret; | ||
| 450 | + } | ||
| 451 | + } | ||
| 452 | + return UBSE_OK; | ||
| 453 | +} | ||
| 454 | + | ||
| 455 | +UbseResult ItCluster::StopNode(const std::string& nodeId) | ||
| 456 | +{ | ||
| 457 | + auto it = nodes_.find(nodeId); | ||
| 458 | + if (it == nodes_.end()) { | ||
| 459 | + IT_LOG_WARN << "StopNode: node " << nodeId << " is not running, no-op"; | ||
| 460 | + return UBSE_OK; | ||
| 461 | + } | ||
| 462 | + it->second->Stop(); | ||
| 463 | + nodes_.erase(it); | ||
| 464 | + IT_LOG_INFO << "Node " << nodeId << " stopped via StopNode"; | ||
| 465 | + return UBSE_OK; | ||
| 466 | +} | ||
| 467 | + | ||
| 364 | bool ItCluster::IsNodeRunning(const std::string& nodeId) const | 468 | bool ItCluster::IsNodeRunning(const std::string& nodeId) const |
| 365 | { | 469 | { |
| 366 | auto it = nodes_.find(nodeId); | 470 | auto it = nodes_.find(nodeId); |
| @@ -389,13 +493,15 @@ UbseResult ItCluster::WaitForElectionConvergence(uint32_t timeoutMs) | |||
| 389 | bool hasMaster = false; | 493 | bool hasMaster = false; |
| 390 | bool hasStandby = false; | 494 | bool hasStandby = false; |
| 391 | uint32_t queriedNodeCount = 0; | 495 | uint32_t queriedNodeCount = 0; |
| 496 | + uint32_t runningNodeCount = 0; | ||
| 392 | 497 | ||
| 393 | for (const auto& nodeId : nodeIds_) { | 498 | for (const auto& nodeId : nodeIds_) { |
| 394 | auto it = nodes_.find(nodeId); | 499 | auto it = nodes_.find(nodeId); |
| 395 | if (it == nodes_.end()) { | 500 | if (it == nodes_.end()) { |
| 396 | - stableConvergenceCount = 0; | 501 | + // deferred/未启动节点 (DeferNodes + StartNode 场景): 不参与本次收敛判断 |
| 397 | - return false; | 502 | + continue; |
| 398 | } | 503 | } |
| 504 | + ++runningNodeCount; | ||
| 399 | 505 | ||
| 400 | std::string role; | 506 | std::string role; |
| 401 | int32_t ret = it->second->GetCliInvoker().GetRole(role); | 507 | int32_t ret = it->second->GetCliInvoker().GetRole(role); |
| @@ -412,12 +518,12 @@ UbseResult ItCluster::WaitForElectionConvergence(uint32_t timeoutMs) | |||
| 412 | } | 518 | } |
| 413 | } | 519 | } |
| 414 | 520 | ||
| 415 | - if (queriedNodeCount != nodeIds_.size()) { | 521 | + if (queriedNodeCount != runningNodeCount) { |
| 416 | stableConvergenceCount = 0; | 522 | stableConvergenceCount = 0; |
| 417 | return false; | 523 | return false; |
| 418 | } | 524 | } |
| 419 | 525 | ||
| 420 | - bool converged = nodeIds_.size() == 1 ? hasMaster : (hasMaster && hasStandby); | 526 | + bool converged = runningNodeCount == 1 ? hasMaster : (hasMaster && hasStandby); |
| 421 | if (!converged) { | 527 | if (!converged) { |
| 422 | stableConvergenceCount = 0; | 528 | stableConvergenceCount = 0; |
| 423 | return false; | 529 | return false; |
| @@ -148,6 +148,34 @@ public: | |||
| 148 | */ | 148 | */ |
| 149 | UbseResult RestartNode(const std::string& nodeId, bool waitForElection = true, uint32_t electionTimeoutMs = 30000); | 149 | UbseResult RestartNode(const std::string& nodeId, bool waitForElection = true, uint32_t electionTimeoutMs = 30000); |
| 150 | 150 | ||
| 151 | + /** | ||
| 152 | + * @brief 启动一个延迟节点 (deferred node). | ||
| 153 | + * | ||
| 154 | + * 配合 ItClusterBuilder::DeferNodes 使用: StartCluster 跳过延迟节点, case 在需要时调用本接口启动. | ||
| 155 | + * 节点必须存在于拓扑中且尚未启动. 重复启动返回错误. | ||
| 156 | + * | ||
| 157 | + * @param nodeId 节点 ID | ||
| 158 | + * @param configOverrides 启动前重新生成该节点配置时应用的 override (section -> key -> value). | ||
| 159 | + * 非空时会在启动前重新生成 workDir/<nodeId>/ubse.conf; 空时沿用 StartCluster 阶段生成的配置. | ||
| 160 | + * @param waitForElection 是否等待选举收敛 | ||
| 161 | + * @param electionTimeoutMs 选举收敛超时 | ||
| 162 | + * @return UBSE_OK on success | ||
| 163 | + */ | ||
| 164 | + UbseResult StartNode(const std::string& nodeId, | ||
| 165 | + const std::map<std::string, std::map<std::string, std::string>>& configOverrides = {}, | ||
| 166 | + bool waitForElection = false, uint32_t electionTimeoutMs = 30000); | ||
| 167 | + | ||
| 168 | + /** | ||
| 169 | + * @brief 优雅停止一个节点 (正常关闭流程, 区别于 KillNode 的 SIGKILL). | ||
| 170 | + * | ||
| 171 | + * 停止后可通过 StartNode(nodeId) 重新启动, 实现"停止 -> 修改配置 -> 重启"的故障注入/恢复流程. | ||
| 172 | + * 节点未启动时返回 UBSE_OK (幂等). | ||
| 173 | + * | ||
| 174 | + * @param nodeId 节点 ID | ||
| 175 | + * @return UBSE_OK on success | ||
| 176 | + */ | ||
| 177 | + UbseResult StopNode(const std::string& nodeId); | ||
| 178 | + | ||
| 151 | /** | 179 | /** |
| 152 | * @brief Check if a specific node is running. | 180 | * @brief Check if a specific node is running. |
| 153 | */ | 181 | */ |
| @@ -122,6 +122,12 @@ ItClusterBuilder& ItClusterBuilder::WithNodeHostname(const std::string& nodeId, | |||
| 122 | return *this; | 122 | return *this; |
| 123 | } | 123 | } |
| 124 | 124 | ||
| 125 | +ItClusterBuilder& ItClusterBuilder::DeferNodes(std::vector<std::string> ids) | ||
| 126 | +{ | ||
| 127 | + deferredNodeIds_.insert(ids.begin(), ids.end()); | ||
| 128 | + return *this; | ||
| 129 | +} | ||
| 130 | + | ||
| 125 | UbseResult ItClusterBuilder::Start(std::unique_ptr<ItCluster>& cluster) const | 131 | UbseResult ItClusterBuilder::Start(std::unique_ptr<ItCluster>& cluster) const |
| 126 | { | 132 | { |
| 127 | if (nodes_.empty()) { | 133 | if (nodes_.empty()) { |
| @@ -142,6 +148,7 @@ ClusterSpec ItClusterBuilder::BuildSpec() const | |||
| 142 | spec.mockPluginEnabled = mockPluginEnabled_; | 148 | spec.mockPluginEnabled = mockPluginEnabled_; |
| 143 | spec.meshType = meshType_; | 149 | spec.meshType = meshType_; |
| 144 | spec.nodeConfigOverrides = nodeConfigOverrides_; | 150 | spec.nodeConfigOverrides = nodeConfigOverrides_; |
| 151 | + spec.deferredNodeIds = deferredNodeIds_; | ||
| 145 | // 应用每节点自定义主机名 | 152 | // 应用每节点自定义主机名 |
| 146 | for (auto& node : spec.nodes) { | 153 | for (auto& node : spec.nodes) { |
| 147 | auto it = nodeHostnameOverrides_.find(node.nodeId); | 154 | auto it = nodeHostnameOverrides_.find(node.nodeId); |
| @@ -16,6 +16,7 @@ | |||
| 16 | 16 | ||
| 17 | 17 | ||
| 18 | 18 | ||
| 19 | + | ||
| 19 | 20 | ||
| 20 | 21 | ||
| 21 | 22 | ||
| @@ -73,6 +74,17 @@ public: | |||
| 73 | */ | 74 | */ |
| 74 | ItClusterBuilder& WithNodeHostname(const std::string& nodeId, const std::string& hostname); | 75 | ItClusterBuilder& WithNodeHostname(const std::string& nodeId, const std::string& hostname); |
| 75 | 76 | ||
| 77 | + /** | ||
| 78 | + * @brief 标记指定节点为延迟启动. | ||
| 79 | + * | ||
| 80 | + * 被标记的节点在 StartCluster 阶段只生成配置文件, 不启动 daemon; | ||
| 81 | + * 测试用例在需要时通过 ItCluster::StartNode(nodeId) 手动启动. | ||
| 82 | + * 适用于"先启动部分节点验证选举/组网, 再按需拉起剩余节点"的场景. | ||
| 83 | + * | ||
| 84 | + * @param ids 延迟启动的节点 ID 列表 | ||
| 85 | + */ | ||
| 86 | + ItClusterBuilder& DeferNodes(std::vector<std::string> ids); | ||
| 87 | + | ||
| 76 | /** | 88 | /** |
| 77 | * @brief Build and start the cluster. | 89 | * @brief Build and start the cluster. |
| 78 | * | 90 | * |
| @@ -99,6 +111,7 @@ private: | |||
| 99 | bool waitForElection_ = true; | 111 | bool waitForElection_ = true; |
| 100 | std::map<std::string, std::map<std::string, std::map<std::string, std::string>>> nodeConfigOverrides_; | 112 | std::map<std::string, std::map<std::string, std::map<std::string, std::string>>> nodeConfigOverrides_; |
| 101 | std::map<std::string, std::string> nodeHostnameOverrides_; | 113 | std::map<std::string, std::string> nodeHostnameOverrides_; |
| 114 | + std::set<std::string> deferredNodeIds_; | ||
| 102 | }; | 115 | }; |
| 103 | 116 | ||
| 104 | } // namespace ubse::it::infra | 117 | } // namespace ubse::it::infra |
| @@ -74,6 +74,24 @@ UbseResult ItConfigBuilder::GenerateAllConfigs(const std::string& templatePath) | |||
| 74 | return UBSE_OK; | 74 | return UBSE_OK; |
| 75 | } | 75 | } |
| 76 | 76 | ||
| 77 | +UbseResult ItConfigBuilder::GenerateNodeConfig(const std::string& nodeId, const std::string& templatePath) | ||
| 78 | +{ | ||
| 79 | + for (const auto& spec : nodeSpecs_) { | ||
| 80 | + if (spec.nodeId != nodeId) { | ||
| 81 | + continue; | ||
| 82 | + } | ||
| 83 | + std::string outputDir = spec.workDir.empty() ? baseWorkDir_ + "/" + spec.nodeId : spec.workDir; | ||
| 84 | + UbseResult ret = GenerateConfig(spec, outputDir, templatePath); | ||
| 85 | + if (ret != UBSE_OK) { | ||
| 86 | + IT_LOG_ERROR << "Failed to generate config for node " << nodeId; | ||
| 87 | + return ret; | ||
| 88 | + } | ||
| 89 | + return UBSE_OK; | ||
| 90 | + } | ||
| 91 | + IT_LOG_ERROR << "GenerateNodeConfig: node " << nodeId << " not in nodeSpecs"; | ||
| 92 | + return UBSE_ERROR_DEF(1); | ||
| 93 | +} | ||
| 94 | + | ||
| 77 | std::string ItConfigBuilder::FindTemplatePath(const std::string& templatePath) | 95 | std::string ItConfigBuilder::FindTemplatePath(const std::string& templatePath) |
| 78 | { | 96 | { |
| 79 | if (!templatePath.empty() && std::filesystem::exists(templatePath)) { | 97 | if (!templatePath.empty() && std::filesystem::exists(templatePath)) { |
| @@ -68,6 +68,18 @@ public: | |||
| 68 | /** @brief Generate config files for all nodes. Returns UBSE_OK on success. */ | 68 | /** @brief Generate config files for all nodes. Returns UBSE_OK on success. */ |
| 69 | UbseResult GenerateAllConfigs(const std::string& templatePath = ""); | 69 | UbseResult GenerateAllConfigs(const std::string& templatePath = ""); |
| 70 | 70 | ||
| 71 | + /** | ||
| 72 | + * @brief 为单个节点重新生成配置文件 (StartNode 场景使用). | ||
| 73 | + * | ||
| 74 | + * 用已有的 nodeSpecs/overrides 重新生成指定节点的 ubse.conf, 覆盖 workDir/<nodeId>/ubse.conf. | ||
| 75 | + * 用于 case 在 StartNode 前修改配置 (通过 WithNodeOverride 预置后调用本方法). | ||
| 76 | + * | ||
| 77 | + * @param nodeId 目标节点 ID (必须存在于构造时的 nodeSpecs) | ||
| 78 | + * @param templatePath 配置模板路径 (空则自动查找) | ||
| 79 | + * @return UBSE_OK on success | ||
| 80 | + */ | ||
| 81 | + UbseResult GenerateNodeConfig(const std::string& nodeId, const std::string& templatePath = ""); | ||
| 82 | + | ||
| 71 | private: | 83 | private: |
| 72 | UbseResult GenerateConfig(const NodeSpec& nodeSpec, const std::string& outputDir, const std::string& templatePath); | 84 | UbseResult GenerateConfig(const NodeSpec& nodeSpec, const std::string& outputDir, const std::string& templatePath); |
| 73 | 85 | ||
| @@ -15,6 +15,7 @@ | |||
| 15 | 15 | ||
| 16 | 16 | ||
| 17 | 17 | ||
| 18 | + | ||
| 18 | 19 | ||
| 19 | 20 | ||
| 20 | 21 | ||
| @@ -53,6 +54,11 @@ struct ClusterSpec { | |||
| 53 | // Applied on top of global overrides during config generation. | 54 | // Applied on top of global overrides during config generation. |
| 54 | std::map<std::string, std::map<std::string, std::map<std::string, std::string>>> nodeConfigOverrides; | 55 | std::map<std::string, std::map<std::string, std::map<std::string, std::string>>> nodeConfigOverrides; |
| 55 | 56 | ||
| 57 | + // Node IDs that StartCluster should NOT auto-start (deferred start via StartNode). | ||
| 58 | + // Config files for these nodes are still generated in Phase 0, but their ItNode | ||
| 59 | + // instances are only created when the test case calls StartNode(). | ||
| 60 | + std::set<std::string> deferredNodeIds; | ||
| 61 | + | ||
| 56 | static ClusterSpec FromRuntimePaths(const std::string& binaryPath, const std::string& baseWorkDir, | 62 | static ClusterSpec FromRuntimePaths(const std::string& binaryPath, const std::string& baseWorkDir, |
| 57 | const std::vector<NodeSpec>& nodes, const std::string& stubLibDir = ""); | 63 | const std::vector<NodeSpec>& nodes, const std::string& stubLibDir = ""); |
| 58 | 64 | ||
| @@ -0,0 +1,3 @@ | |||
| 1 | +add_it_scene(four_nodes_deferred_start | ||
| 2 | + LINK_LIBS it_election_cases | ||
| 3 | +) | ||
| @@ -0,0 +1,41 @@ | |||
| 1 | +/* | ||
| 2 | + * Copyright (c) Huawei Technologies Co., Ltd. 2026-2026. All rights reserved. | ||
| 3 | + * ubs-engine is licensed under Mulan PSL v2. | ||
| 4 | + * You can use this software according to the terms and conditions of the Mulan PSL v2. | ||
| 5 | + * You may obtain a copy of Mulan PSL v2 at: | ||
| 6 | + * http://license.coscl.org.cn/MulanPSL2 | ||
| 7 | + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, | ||
| 8 | + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, | ||
| 9 | + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. | ||
| 10 | + * See the Mulan PSL v2 for more details. | ||
| 11 | + */ | ||
| 12 | + | ||
| 13 | + | ||
| 14 | + | ||
| 15 | + | ||
| 16 | +using ubse::it::infra::Tongsuan1dFourNodesDeferredStartScenario; | ||
| 17 | + | ||
| 18 | +// P0-DeferredStart-NotRunning: 节点 4 延迟启动, StartCluster 后应未运行; 其余节点正常运行并收敛 | ||
| 19 | +TEST_F(Tongsuan1dFourNodesDeferredStartScenario, P0DeferredNodeNotRunning) | ||
| 20 | +{ | ||
| 21 | + ubse::it::tests::election::RunDeferredStartNodeNotRunningTest(Cluster()); | ||
| 22 | +} | ||
| 23 | + | ||
| 24 | +// P0-DeferredStart-StartWithConfigOverride: 带 configOverrides 启动节点 4, | ||
| 25 | +// 启动前应重新生成配置 (写入 log.level=DEBUG), 集群收敛为 1 主 + 1 备 + 2 代理 | ||
| 26 | +TEST_F(Tongsuan1dFourNodesDeferredStartScenario, P0StartNodeWithConfigOverride) | ||
| 27 | +{ | ||
| 28 | + ubse::it::tests::election::RunDeferredStartNodeWithConfigOverrideTest(Cluster()); | ||
| 29 | +} | ||
| 30 | + | ||
| 31 | +// P0-DeferredStart-StopNode: 优雅停止节点 4 (正常关闭, 区别于 KillNode 的 SIGKILL) | ||
| 32 | +TEST_F(Tongsuan1dFourNodesDeferredStartScenario, P0StopNode) | ||
| 33 | +{ | ||
| 34 | + ubse::it::tests::election::RunDeferredStopNodeTest(Cluster()); | ||
| 35 | +} | ||
| 36 | + | ||
| 37 | +// P0-DeferredStart-RestartWithNewConfig: 停止后带新配置重启节点 4, 验证"修改配置 -> 重启"流程 | ||
| 38 | +TEST_F(Tongsuan1dFourNodesDeferredStartScenario, P0RestartNodeWithNewConfig) | ||
| 39 | +{ | ||
| 40 | + ubse::it::tests::election::RunDeferredRestartNodeWithNewConfigTest(Cluster()); | ||
| 41 | +} | ||
| @@ -0,0 +1,34 @@ | |||
| 1 | +/* | ||
| 2 | + * Copyright (c) Huawei Technologies Co., Ltd. 2026-2026. All rights reserved. | ||
| 3 | + * ubs-engine is licensed under Mulan PSL v2. | ||
| 4 | + * You can use this software according to the terms and conditions of the Mulan PSL v2. | ||
| 5 | + * You may obtain a copy of Mulan PSL v2 at: | ||
| 6 | + * http://license.coscl.org.cn/MulanPSL2 | ||
| 7 | + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, | ||
| 8 | + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, | ||
| 9 | + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. | ||
| 10 | + * See the Mulan PSL v2 for more details. | ||
| 11 | + */ | ||
| 12 | + | ||
| 13 | + | ||
| 14 | + | ||
| 15 | + | ||
| 16 | + | ||
| 17 | + | ||
| 18 | +/** | ||
| 19 | + * @brief 四节点延迟启动场景。 | ||
| 20 | + * | ||
| 21 | + * 使用 DeferNodes({"4"}) 延迟启动节点 4: | ||
| 22 | + * - StartCluster 阶段只启动节点 1/2/3 并等待选举收敛 (1 主 + 1 备 + 1 代理); | ||
| 23 | + * - 节点 4 的配置文件在 Phase 0 已生成, 但 daemon 未启动. | ||
| 24 | + * | ||
| 25 | + * 测试用例验证: | ||
| 26 | + * - P0: deferred 节点初始未运行, 其余节点正常运行 | ||
| 27 | + * - P0: StartNode 带 configOverrides 启动节点 4 (启动前重新生成配置), 集群收敛为 1 主 + 1 备 + 2 代理 | ||
| 28 | + * - P0: StopNode 优雅停止节点 4 | ||
| 29 | + * - P0: 停止后带新配置重启节点 4 | ||
| 30 | + */ | ||
| 31 | +IT_DEFINE_SCENARIO(Tongsuan1dFourNodesDeferredStartScenario, | ||
| 32 | + MakeBuilder().Tongsuan().FourNode().DeferNodes({"4"}).Start(cluster_)) | ||
| 33 | + | ||
| 34 | + | ||
| @@ -1,5 +1,6 @@ | |||
| 1 | add_library(it_election_cases STATIC | 1 | add_library(it_election_cases STATIC |
| 2 | election_cases.cpp | 2 | election_cases.cpp |
| 3 | + election_deferred_start_cases.cpp | ||
| 3 | ) | 4 | ) |
| 4 | target_include_directories(it_election_cases PUBLIC | 5 | target_include_directories(it_election_cases PUBLIC |
| 5 | ${CMAKE_SOURCE_DIR}/test/IT | 6 | ${CMAKE_SOURCE_DIR}/test/IT |
| @@ -0,0 +1,138 @@ | |||
| 1 | +/* | ||
| 2 | + * Copyright (c) Huawei Technologies Co., Ltd. 2026-2026. All rights reserved. | ||
| 3 | + * ubs-engine is licensed under Mulan PSL v2. | ||
| 4 | + * You can use this software according to the terms and conditions of the Mulan PSL v2. | ||
| 5 | + * You may obtain a copy of Mulan PSL v2 at: | ||
| 6 | + * http://license.coscl.org.cn/MulanPSL2 | ||
| 7 | + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, | ||
| 8 | + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, | ||
| 9 | + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. | ||
| 10 | + * See the Mulan PSL v2 for more details. | ||
| 11 | + */ | ||
| 12 | + | ||
| 13 | + | ||
| 14 | + | ||
| 15 | + | ||
| 16 | + | ||
| 17 | + | ||
| 18 | + | ||
| 19 | + | ||
| 20 | + | ||
| 21 | + | ||
| 22 | + | ||
| 23 | + | ||
| 24 | +namespace ubse::it::tests::election { | ||
| 25 | + | ||
| 26 | +namespace { | ||
| 27 | + | ||
| 28 | +// 延迟启动的节点 ID (DeferNodes({"4"}) 场景) | ||
| 29 | +constexpr const char* DEFERRED_NODE_ID = "4"; | ||
| 30 | + | ||
| 31 | +// 选举角色统计 (仅统计运行中的节点; deferred/停止的节点跳过) | ||
| 32 | +struct RunningRoles { | ||
| 33 | + uint32_t masterCount{0}; | ||
| 34 | + uint32_t standbyCount{0}; | ||
| 35 | + uint32_t agentCount{0}; | ||
| 36 | +}; | ||
| 37 | + | ||
| 38 | +RunningRoles CollectRunningRoles(ubse::it::infra::ItCluster& cluster) | ||
| 39 | +{ | ||
| 40 | + RunningRoles roles; | ||
| 41 | + for (const auto& nodeId : cluster.GetNodeIds()) { | ||
| 42 | + if (!cluster.IsNodeRunning(nodeId)) { | ||
| 43 | + continue; | ||
| 44 | + } | ||
| 45 | + std::string role; | ||
| 46 | + int32_t cliRet = cluster.GetCliInvoker(nodeId).GetRole(role); | ||
| 47 | + EXPECT_EQ(cliRet, UBS_SUCCESS) << "GetRole failed for node " << nodeId; | ||
| 48 | + if (role == ubse::election::ELECTION_ROLE_MASTER) { | ||
| 49 | + ++roles.masterCount; | ||
| 50 | + } else if (role == ubse::election::ELECTION_ROLE_STANDBY) { | ||
| 51 | + ++roles.standbyCount; | ||
| 52 | + } else if (role == ubse::election::ELECTION_ROLE_AGENT) { | ||
| 53 | + ++roles.agentCount; | ||
| 54 | + } | ||
| 55 | + } | ||
| 56 | + return roles; | ||
| 57 | +} | ||
| 58 | + | ||
| 59 | +// 读取节点配置文件内容, 用于断言 configOverrides 是否真正写入了 ubse.conf | ||
| 60 | +std::string ReadNodeConfig(const ubse::it::infra::ItCluster& cluster, const std::string& nodeId) | ||
| 61 | +{ | ||
| 62 | + std::ifstream ifs(cluster.GetBaseWorkDir() + "/" + nodeId + "/ubse.conf"); | ||
| 63 | + std::stringstream ss; | ||
| 64 | + ss << ifs.rdbuf(); | ||
| 65 | + return ss.str(); | ||
| 66 | +} | ||
| 67 | + | ||
| 68 | +} // namespace | ||
| 69 | + | ||
| 70 | +void RunDeferredStartNodeNotRunningTest(ubse::it::infra::ItCluster& cluster) | ||
| 71 | +{ | ||
| 72 | + EXPECT_FALSE(cluster.IsNodeRunning(DEFERRED_NODE_ID)) | ||
| 73 | + << "deferred node " << DEFERRED_NODE_ID << " should not be running after StartCluster"; | ||
| 74 | + EXPECT_TRUE(cluster.IsNodeRunning("1")); | ||
| 75 | + EXPECT_TRUE(cluster.IsNodeRunning("2")); | ||
| 76 | + EXPECT_TRUE(cluster.IsNodeRunning("3")); | ||
| 77 | + | ||
| 78 | + // 仅启动 3 个节点时应收敛为 1 主 + 1 备 + 1 代理 | ||
| 79 | + auto roles = CollectRunningRoles(cluster); | ||
| 80 | + EXPECT_EQ(roles.masterCount, 1U); | ||
| 81 | + EXPECT_EQ(roles.standbyCount, 1U); | ||
| 82 | + EXPECT_EQ(roles.agentCount, 1U); | ||
| 83 | +} | ||
| 84 | + | ||
| 85 | +void RunDeferredStartNodeWithConfigOverrideTest(ubse::it::infra::ItCluster& cluster) | ||
| 86 | +{ | ||
| 87 | + if (cluster.IsNodeRunning(DEFERRED_NODE_ID)) { | ||
| 88 | + GTEST_SKIP() << "node " << DEFERRED_NODE_ID << " already running (case order dependency)"; | ||
| 89 | + } | ||
| 90 | + | ||
| 91 | + ASSERT_IT_OK(cluster.StartNode(DEFERRED_NODE_ID, {{"ubse.log", {{"log.level", "DEBUG"}}}}, true)); | ||
| 92 | + EXPECT_TRUE(cluster.IsNodeRunning(DEFERRED_NODE_ID)); | ||
| 93 | + | ||
| 94 | + // 校验 configOverrides 已写入节点配置文件 | ||
| 95 | + auto config = ReadNodeConfig(cluster, DEFERRED_NODE_ID); | ||
| 96 | + EXPECT_NE(config.find("log.level=DEBUG"), std::string::npos) | ||
| 97 | + << "configOverride log.level=DEBUG not written to node " << DEFERRED_NODE_ID << " config"; | ||
| 98 | + | ||
| 99 | + auto roles = CollectRunningRoles(cluster); | ||
| 100 | + EXPECT_EQ(roles.masterCount, 1U); | ||
| 101 | + EXPECT_EQ(roles.standbyCount, 1U); | ||
| 102 | + EXPECT_EQ(roles.agentCount, 2U); | ||
| 103 | +} | ||
| 104 | + | ||
| 105 | +void RunDeferredStopNodeTest(ubse::it::infra::ItCluster& cluster) | ||
| 106 | +{ | ||
| 107 | + if (!cluster.IsNodeRunning(DEFERRED_NODE_ID)) { | ||
| 108 | + GTEST_SKIP() << "node " << DEFERRED_NODE_ID << " not running (case order dependency)"; | ||
| 109 | + } | ||
| 110 | + | ||
| 111 | + ASSERT_IT_OK(cluster.StopNode(DEFERRED_NODE_ID)); | ||
| 112 | + EXPECT_FALSE(cluster.IsNodeRunning(DEFERRED_NODE_ID)); | ||
| 113 | + | ||
| 114 | + // 停止未运行节点应幂等返回 OK | ||
| 115 | + ASSERT_IT_OK(cluster.StopNode(DEFERRED_NODE_ID)); | ||
| 116 | +} | ||
| 117 | + | ||
| 118 | +void RunDeferredRestartNodeWithNewConfigTest(ubse::it::infra::ItCluster& cluster) | ||
| 119 | +{ | ||
| 120 | + if (cluster.IsNodeRunning(DEFERRED_NODE_ID)) { | ||
| 121 | + GTEST_SKIP() << "node " << DEFERRED_NODE_ID << " already running (case order dependency)"; | ||
| 122 | + } | ||
| 123 | + | ||
| 124 | + ASSERT_IT_OK(cluster.StartNode(DEFERRED_NODE_ID, {{"ubse.log", {{"log.level", "ERROR"}}}}, true)); | ||
| 125 | + EXPECT_TRUE(cluster.IsNodeRunning(DEFERRED_NODE_ID)); | ||
| 126 | + | ||
| 127 | + // 新配置应覆盖旧配置 | ||
| 128 | + auto config = ReadNodeConfig(cluster, DEFERRED_NODE_ID); | ||
| 129 | + EXPECT_NE(config.find("log.level=ERROR"), std::string::npos) | ||
| 130 | + << "new configOverride log.level=ERROR not written to node " << DEFERRED_NODE_ID << " config"; | ||
| 131 | + | ||
| 132 | + auto roles = CollectRunningRoles(cluster); | ||
| 133 | + EXPECT_EQ(roles.masterCount, 1U); | ||
| 134 | + EXPECT_EQ(roles.standbyCount, 1U); | ||
| 135 | + EXPECT_EQ(roles.agentCount, 2U); | ||
| 136 | +} | ||
| 137 | + | ||
| 138 | +} // namespace ubse::it::tests::election | ||
| @@ -0,0 +1,34 @@ | |||
| 1 | +/* | ||
| 2 | + * Copyright (c) Huawei Technologies Co., Ltd. 2026-2026. All rights reserved. | ||
| 3 | + * ubs-engine is licensed under Mulan PSL v2. | ||
| 4 | + * You can use this software according to the terms and conditions of the Mulan PSL v2. | ||
| 5 | + * You may obtain a copy of Mulan PSL v2 at: | ||
| 6 | + * http://license.coscl.org.cn/MulanPSL2 | ||
| 7 | + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, | ||
| 8 | + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, | ||
| 9 | + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. | ||
| 10 | + * See the Mulan PSL v2 for more details. | ||
| 11 | + */ | ||
| 12 | + | ||
| 13 | + | ||
| 14 | + | ||
| 15 | + | ||
| 16 | + | ||
| 17 | + | ||
| 18 | +namespace ubse::it::tests::election { | ||
| 19 | + | ||
| 20 | +// P0: deferred 节点初始未运行, 其余节点正常运行并收敛 | ||
| 21 | +void RunDeferredStartNodeNotRunningTest(ubse::it::infra::ItCluster& cluster); | ||
| 22 | + | ||
| 23 | +// P0: StartNode 带 configOverrides 启动 deferred 节点, 校验配置写入 + 集群收敛 | ||
| 24 | +void RunDeferredStartNodeWithConfigOverrideTest(ubse::it::infra::ItCluster& cluster); | ||
| 25 | + | ||
| 26 | +// P0: StopNode 优雅停止 deferred 节点 + 幂等 | ||
| 27 | +void RunDeferredStopNodeTest(ubse::it::infra::ItCluster& cluster); | ||
| 28 | + | ||
| 29 | +// P0: 停止后带新配置重启 deferred 节点, 验证"修改配置 -> 重启"流程 | ||
| 30 | +void RunDeferredRestartNodeWithNewConfigTest(ubse::it::infra::ItCluster& cluster); | ||
| 31 | + | ||
| 32 | +} // namespace ubse::it::tests::election | ||
| 33 | + | ||
| 34 | + | ||