* Copyright (c) Huawei Technologies Co., Ltd. 2025-2025. All rights reserved.
* ubs-engine is licensed under Mulan PSL v2.
* You can use this software according to the terms and conditions of the Mulan PSL v2.
* You may obtain a copy of Mulan PSL v2 at:
* http://license.coscl.org.cn/MulanPSL2
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PSL v2 for more details.
*/
#include "election_cases.h"
#include <chrono>
#include <fstream>
#include <thread>
#include <gtest/gtest.h>
#include "ubse_common_def.h"
#include "it_assertion.h"
#include "it_wait_helper.h"
namespace ubse::it::tests::election {
namespace {
std::string ReadConfigValue(const std::string& path, const std::string& section, const std::string& key)
{
std::ifstream ifs(path);
if (!ifs.is_open()) {
return "";
}
std::string line;
std::string curSection;
while (std::getline(ifs, line)) {
auto hashPos = line.find('#');
if (hashPos != std::string::npos) {
line = line.substr(0, hashPos);
}
auto start = line.find_first_not_of(" \t\r\n");
if (start == std::string::npos) {
continue;
}
auto end = line.find_last_not_of(" \t\r\n");
line = line.substr(start, end - start + 1);
if (line.empty()) {
continue;
}
if (line[0] == '[') {
auto close = line.find(']');
curSection = (close == std::string::npos) ? line.substr(1) : line.substr(1, close - 1);
continue;
}
auto eq = line.find('=');
if (eq == std::string::npos) {
continue;
}
std::string k = line.substr(0, eq);
std::string v = line.substr(eq + 1);
auto kStart = k.find_first_not_of(" \t");
auto kEnd = k.find_last_not_of(" \t");
k = (kStart == std::string::npos) ? "" : k.substr(kStart, kEnd - kStart + 1);
auto vStart = v.find_first_not_of(" \t");
auto vEnd = v.find_last_not_of(" \t");
v = (vStart == std::string::npos) ? "" : v.substr(vStart, vEnd - vStart + 1);
if (curSection == section && k == key) {
return v;
}
}
return "";
}
size_t CountLogLines(const std::string& path, const std::string& substring)
{
std::ifstream ifs(path);
if (!ifs.is_open()) {
return 0;
}
size_t count = 0;
std::string line;
while (std::getline(ifs, line)) {
if (line.find(substring) != std::string::npos) {
++count;
}
}
return count;
}
void CheckMasterPeriodicHeartbeat(ubse::it::infra::ItCluster& cluster, const std::string& masterNodeId,
const std::string& peerNodeId)
{
uint32_t hbIntervalMs = 2000;
auto intervalStr =
ReadConfigValue(cluster.GetNode(masterNodeId).GetConfigFilePath(), "ubse.election", "heartbeat.timeInterval");
if (!intervalStr.empty()) {
try {
hbIntervalMs = static_cast<uint32_t>(std::stoul(intervalStr));
} catch (const std::exception&) {
hbIntervalMs = 2000;
}
}
if (hbIntervalMs == 0) {
hbIntervalMs = 2000;
}
auto waitMs = std::chrono::milliseconds(3 * static_cast<int64_t>(hbIntervalMs) + 1000);
std::this_thread::sleep_for(waitMs);
const std::string hbLog = cluster.GetNode(masterNodeId).GetLogFilePath();
const std::string sendMark = "[ELECTION] ProcTimer MASTER send pkt id=";
size_t expectedMinSends = static_cast<size_t>(waitMs.count()) / hbIntervalMs;
size_t totalSends = CountLogLines(hbLog, sendMark);
EXPECT_GE(totalSends, expectedMinSends)
<< "master should periodically send heartbeats (>= " << expectedMinSends << " observed in " << waitMs.count()
<< "ms @ interval " << hbIntervalMs << "ms), log=" << hbLog;
std::string sendToPeer = sendMark + peerNodeId;
size_t sendsToPeer = CountLogLines(hbLog, sendToPeer);
EXPECT_GE(sendsToPeer, 1U) << "master should send heartbeat to peer node " << peerNodeId;
}
}
ElectionRoles CollectElectionRoles(ubse::it::infra::ItCluster& cluster)
{
ElectionRoles roles;
for (const auto& nodeId : cluster.GetNodeIds()) {
auto& cliInvoker = cluster.GetCliInvoker(nodeId);
std::string role;
int32_t cliRet = cliInvoker.GetRole(role);
EXPECT_EQ(cliRet, UBS_SUCCESS);
if (role == ubse::election::ELECTION_ROLE_MASTER) {
++roles.masterCount;
roles.masterNodeId = nodeId;
} else if (role == ubse::election::ELECTION_ROLE_STANDBY) {
++roles.standbyCount;
roles.standbyNodeId = nodeId;
} else if (role == ubse::election::ELECTION_ROLE_AGENT) {
++roles.agentCount;
roles.agentNodeIds.push_back(nodeId);
} else {
ADD_FAILURE() << "Unexpected election role for " << nodeId << ": " << role;
}
}
return roles;
}
void RunSingleNodeElectionTest(ubse::it::infra::ItCluster& cluster)
{
std::string masterNodeId;
auto ret = cluster.GetMasterNodeId(masterNodeId);
EXPECT_IT_OK(ret);
EXPECT_EQ(masterNodeId, "1");
auto& cliInvoker = cluster.GetCliInvoker("1");
std::string role;
int32_t cliRet = cliInvoker.GetRole(role);
EXPECT_EQ(cliRet, UBS_SUCCESS);
EXPECT_EQ(role, ubse::election::ELECTION_ROLE_MASTER);
}
void RunTwoNodeElectionTest(ubse::it::infra::ItCluster& cluster)
{
std::string masterNodeId;
auto ret = cluster.GetMasterNodeId(masterNodeId);
EXPECT_IT_OK(ret);
auto roles = CollectElectionRoles(cluster);
EXPECT_EQ(roles.masterCount, 1U);
EXPECT_EQ(roles.standbyCount, 1U);
EXPECT_EQ(roles.masterNodeId, "1");
EXPECT_EQ(roles.standbyNodeId, "2");
EXPECT_EQ(masterNodeId, roles.masterNodeId);
std::string peerNodeId;
for (const auto& id : cluster.GetNodeIds()) {
if (id != roles.masterNodeId) {
peerNodeId = id;
break;
}
}
CheckMasterPeriodicHeartbeat(cluster, roles.masterNodeId, peerNodeId);
}
void RunTwoNodeElectionCandidateFalseTest(ubse::it::infra::ItCluster& cluster)
{
const auto& nodeIds = cluster.GetNodeIds();
ASSERT_GE(nodeIds.size(), 2U);
std::string minNodeId = nodeIds.front();
for (const auto& id : nodeIds) {
if (id < minNodeId) {
minNodeId = id;
}
}
std::string otherNodeId;
for (const auto& id : nodeIds) {
if (id != minNodeId) {
otherNodeId = id;
break;
}
}
std::string minCandidate =
ReadConfigValue(cluster.GetNode(minNodeId).GetConfigFilePath(), "ubse.election", "election.candidate");
EXPECT_EQ(minCandidate, "false") << "min node " << minNodeId << " should have election.candidate=false";
std::string otherCandidate =
ReadConfigValue(cluster.GetNode(otherNodeId).GetConfigFilePath(), "ubse.election", "election.candidate");
EXPECT_EQ(otherCandidate, "true") << "other node " << otherNodeId << " should have election.candidate=true";
std::string masterNodeId;
auto ret = cluster.GetMasterNodeId(masterNodeId);
EXPECT_IT_OK(ret);
auto roles = CollectElectionRoles(cluster);
EXPECT_EQ(roles.masterCount, 1U);
EXPECT_EQ(roles.standbyCount, 1U);
EXPECT_NE(roles.masterNodeId, minNodeId);
EXPECT_EQ(roles.masterNodeId, otherNodeId);
EXPECT_EQ(roles.standbyNodeId, minNodeId);
EXPECT_EQ(masterNodeId, roles.masterNodeId);
std::string peerNodeId;
for (const auto& id : nodeIds) {
if (id != roles.masterNodeId) {
peerNodeId = id;
break;
}
}
CheckMasterPeriodicHeartbeat(cluster, roles.masterNodeId, peerNodeId);
}
void RunFourNodeElectionTest(ubse::it::infra::ItCluster& cluster)
{
std::string masterNodeId;
auto ret = cluster.GetMasterNodeId(masterNodeId);
EXPECT_IT_OK(ret);
auto roles = CollectElectionRoles(cluster);
EXPECT_EQ(roles.masterCount, 1U);
EXPECT_EQ(roles.standbyCount, 1U);
EXPECT_EQ(roles.agentCount, 2U);
EXPECT_EQ(roles.agentNodeIds.size(), 2U);
EXPECT_EQ(masterNodeId, roles.masterNodeId);
}
void RunFourNodeMasterRestartTest(ubse::it::infra::ItCluster& cluster)
{
RunFourNodeElectionTest(cluster);
auto before = CollectElectionRoles(cluster);
ASSERT_EQ(before.masterCount, 1U);
ASSERT_EQ(before.standbyCount, 1U);
std::string oldMaster = before.masterNodeId;
std::string oldStandby = before.standbyNodeId;
ASSERT_IT_OK(cluster.KillNode(oldMaster));
std::string newMaster;
auto ret = ubse::it::infra::ItWaitHelper::WaitForCondition(
[&]() -> bool {
std::string m;
if (cluster.GetMasterNodeId(m) == UBSE_OK && m != oldMaster) {
newMaster = m;
return true;
}
return false;
},
30000);
EXPECT_IT_OK(ret) << "standby did not take over as master after master was killed";
EXPECT_EQ(newMaster, oldStandby) << "the previous standby should become the new master";
ASSERT_IT_OK(cluster.RestartNode(oldMaster, true, 30000));
EXPECT_TRUE(cluster.IsNodeRunning(oldMaster));
auto after = CollectElectionRoles(cluster);
EXPECT_EQ(after.masterCount, 1U);
EXPECT_EQ(after.standbyCount, 1U);
EXPECT_EQ(after.agentCount, 2U);
}
}