Star137
89
代码
代码
Issues36
Pull Requests30
项目讨论
Wiki
分析
Star137
89
已合并
[feature] 配置文件解析增加 ock.mmc 前缀过滤及加载清单打印 #389
dajiang创建于 7月21日
[feature] 配置文件解析增加 ock.mmc 前缀过滤及加载清单打印 #389
已合并
dajiang创建于 7月21日
从gcw_qYJeyWK4/memcachefeature/config-prefix-filter合入到Ascend/memcachedevelop
共 11 个文件变更+212-18
M3rdparty/memfabric_hybrid+1-1
@@ -1 +1 @@
1-Subproject commit 522d25de16bcc517dae41e9d88e6ea28777249441+Subproject commit c3b826c8fb27911aea784c9106efcdd6de3ec744
Mcmake/config_ubsio.cmake+0-5
@@ -63,9 +63,4 @@ if (BUILD_UBSIO)
63 file(COPY ${UBSIO_SO_FILES} DESTINATION ${UBSIO_OUTPUT_DIR}/lib)63 file(COPY ${UBSIO_SO_FILES} DESTINATION ${UBSIO_OUTPUT_DIR}/lib)
64 message(STATUS "ubs-io lib installed to ${UBSIO_OUTPUT_DIR}/lib")64 message(STATUS "ubs-io lib installed to ${UBSIO_OUTPUT_DIR}/lib")
65 65 
66- # install ubsio.conf
67- file(MAKE_DIRECTORY ${UBSIO_OUTPUT_DIR}/conf)
68- file(COPY_FILE ${ubs-io_SOURCE_DIR}/ubsio-boostio/configs/ubsio.conf
69- ${UBSIO_OUTPUT_DIR}/conf/ubsio.conf)
70- message(STATUS "ubs-io config installed to ${UBSIO_OUTPUT_DIR}/conf")
71endif ()66endif ()
Mscript/build.sh+0-3
@@ -117,9 +117,6 @@ if [ "${BUILD_UBSIO:-OFF}" == "ON" ]; then
117fi117fi
118mkdir -p "${PROJ_DIR}/src/memcache/python/memcache_hybrid/config"118mkdir -p "${PROJ_DIR}/src/memcache/python/memcache_hybrid/config"
119\cp -v "${PROJ_DIR}"/config/* "${PROJ_DIR}/src/memcache/python/memcache_hybrid/config"119\cp -v "${PROJ_DIR}"/config/* "${PROJ_DIR}/src/memcache/python/memcache_hybrid/config"
120-if [ "${BUILD_UBSIO:-OFF}" == "ON" ]; then
121- \cp -v "${PROJ_DIR}/output/3rdparty/ubsio/conf/"* "${PROJ_DIR}/src/memcache/python/memcache_hybrid/config" 2>/dev/null || true
122-fi
123 120 
124cd "${PROJ_DIR}"121cd "${PROJ_DIR}"
125rm -f "${PROJ_DIR}"/src/memcache/python/memcache_hybrid/_pymmc.cpython*.so122rm -f "${PROJ_DIR}"/src/memcache/python/memcache_hybrid/_pymmc.cpython*.so
Mscript/run_pkg_maker/install.sh+1-1
@@ -309,7 +309,7 @@ function generate_set_env()
309 cat > "${default_install_dir}/set_env.sh" <<EOF309 cat > "${default_install_dir}/set_env.sh" <<EOF
310PYTHON_LIB_DIR=\$(python3 -c 'import sys,os;print(os.path.join(sys.prefix,"lib"))')310PYTHON_LIB_DIR=\$(python3 -c 'import sys,os;print(os.path.join(sys.prefix,"lib"))')
311export MEMCACHE_HYBRID_HOME_PATH=${default_install_dir}/latest311export MEMCACHE_HYBRID_HOME_PATH=${default_install_dir}/latest
312-export UBSIO_CONFIG_PATH=\${MEMCACHE_HYBRID_HOME_PATH}/config/ubsio.conf312+export UBSIO_CONFIG_PATH=\${MEMCACHE_HYBRID_HOME_PATH}/config/mmc-local.conf
313export LD_LIBRARY_PATH=${default_install_dir}/latest/${pkg_arch}-${os1}/lib64:\${PYTHON_LIB_DIR}:\$LD_LIBRARY_PATH313export LD_LIBRARY_PATH=${default_install_dir}/latest/${pkg_arch}-${os1}/lib64:\${PYTHON_LIB_DIR}:\$LD_LIBRARY_PATH
314export PATH=${default_install_dir}/latest/${pkg_arch}-${os1}/bin:\$PATH314export PATH=${default_install_dir}/latest/${pkg_arch}-${os1}/bin:\$PATH
315EOF315EOF
Mscript/run_pkg_maker/make_run.sh+1-1
@@ -65,7 +65,7 @@ cp "${PROJECT_DIR}"/config/* ${PKG_DIR}/config
65 65 
66if compgen -G "${OUTPUT_DIR}/3rdparty/ubsio/lib/*.so*" > /dev/null; then66if compgen -G "${OUTPUT_DIR}/3rdparty/ubsio/lib/*.so*" > /dev/null; then
67 \cp -d "${OUTPUT_DIR}"/3rdparty/ubsio/lib/*.so* ${PKG_DIR}/"${ARCH_OS}"/lib64/67 \cp -d "${OUTPUT_DIR}"/3rdparty/ubsio/lib/*.so* ${PKG_DIR}/"${ARCH_OS}"/lib64/
68- \cp -r "${OUTPUT_DIR}"/3rdparty/ubsio/conf/* ${PKG_DIR}/config/68+ 
69fi69fi
70 70 
71if [ "$BUILD_TEST" = "ON" ]; then71if [ "$BUILD_TEST" = "ON" ]; then
Msrc/memcache/csrc/config/mmc_configuration.cpp+9-0
@@ -152,6 +152,15 @@ bool Configuration::LoadFromFile(const std::string &filePath)
152 SAFE_DELETE(kvParser);152 SAFE_DELETE(kvParser);
153 return false;153 return false;
154 }154 }
155+ 
156+ MMC_LOG_INFO("Loaded " << size << " config items:");
157+ for (uint32_t i = 0; i < size; i++) {
158+ std::string key;
159+ std::string value;
160+ kvParser->GetI(i, key, value);
161+ MMC_LOG_INFO(" " << key << " = " << value);
162+ }
163+ 
155 SAFE_DELETE(kvParser);164 SAFE_DELETE(kvParser);
156 165 
157 ResolveAllUrlDomains();166 ResolveAllUrlDomains();
Msrc/memcache/csrc/config/mmc_functions.h+5-0
@@ -76,6 +76,11 @@ inline bool OckStof(const std::string &str, float &value)
76 76 
77const std::unordered_map<std::string, bool> Str2Bool{{"0", false}, {"1", true}, {"false", false}, {"true", true}};77const std::unordered_map<std::string, bool> Str2Bool{{"0", false}, {"1", true}, {"false", false}, {"true", true}};
78 78 
79+inline bool StartsWith(const std::string &str, const std::string &prefix)
80+{
81+ return str.size() >= prefix.size() && str.compare(0, prefix.size(), prefix) == 0;
82+}
83+ 
79inline bool IsBool(const std::string &str, bool &value)84inline bool IsBool(const std::string &str, bool &value)
80{85{
81 std::string tmp = str;86 std::string tmp = str;
Msrc/memcache/csrc/config/mmc_kv_parser.cpp+3-0
@@ -194,6 +194,9 @@ Result KVParser::ParseLine(std::string &strLine)
194 MMC_LOG_ERROR("Configuration item has empty key");194 MMC_LOG_ERROR("Configuration item has empty key");
195 return MMC_ERROR;195 return MMC_ERROR;
196 }196 }
197+ if (!StartsWith(strKey, "ock.mmc.")) {
198+ return MMC_OK;
199+ }
197 if (SetItem(strKey, strValue) != MMC_OK) {200 if (SetItem(strKey, strValue) != MMC_OK) {
198 MMC_LOG_ERROR("Failed to set key <" << strKey << "> with value <" << strValue << ">");201 MMC_LOG_ERROR("Failed to set key <" << strKey << "> with value <" << strValue << ">");
199 return MMC_ERROR;202 return MMC_ERROR;
Msrc/memcache/csrc/meta_service/mmc_meta_manager.cpp+7-7
@@ -899,7 +899,7 @@ Result MmcMetaManager::BlobDeleteRpc(const std::string &key, const MmcMemBlobDes
899 << ", resp: " << resp.ret_);899 << ", resp: " << resp.ret_);
900 return MMC_ERROR;900 return MMC_ERROR;
901 }901 }
902- MMC_LOG_INFO("Deleted blob via RPC successfully, key=" << key << ", rank=" << blob.rank_);902+ MMC_LOG_DEBUG("Deleted blob via RPC successfully, key=" << key << ", rank=" << blob.rank_);
903 return MMC_OK;903 return MMC_OK;
904}904}
905 905 
@@ -921,7 +921,7 @@ Result MmcMetaManager::RemoveSsdBlob(const std::string &key, uint32_t rank)
921 if (objMeta->NumBlobs() == 0) {921 if (objMeta->NumBlobs() == 0) {
922 guard.unlock();922 guard.unlock();
923 metaContainer_->Erase(key);923 metaContainer_->Erase(key);
924- MMC_LOG_INFO("RemoveSsdBlob: key=" << key << " fully removed (no remaining blobs)");924+ MMC_LOG_DEBUG("RemoveSsdBlob: key=" << key << " fully removed (no remaining blobs)");
925 }925 }
926 return MMC_OK;926 return MMC_OK;
927}927}
@@ -1613,7 +1613,7 @@ bool MmcMetaManager::HandleEvictSsdBranch(const std::string &key, const MmcMemOb
1613 }1613 }
1614 uint16_t dramWatermark = GetRewarmWatermark(MEDIA_DRAM);1614 uint16_t dramWatermark = GetRewarmWatermark(MEDIA_DRAM);
1615 if (globalAllocator_->IsAboveUsageRatio(MEDIA_DRAM, dramWatermark)) {1615 if (globalAllocator_->IsAboveUsageRatio(MEDIA_DRAM, dramWatermark)) {
1616- MMC_LOG_WARN("Evict REMOVE key=" << key << " from " << srcMediaType << " reason=dram_full_skip_ssd");1616+ MMC_LOG_DEBUG("Evict REMOVE key=" << key << " from " << srcMediaType << " reason=dram_full_skip_ssd");
1617 if (evictRank != UINT32_MAX) {1617 if (evictRank != UINT32_MAX) {
1618 MmcBlobFilterPtr ssdFilter = MmcMakeRef<MmcBlobFilter>(evictRank, MEDIA_SSD, NONE);1618 MmcBlobFilterPtr ssdFilter = MmcMakeRef<MmcBlobFilter>(evictRank, MEDIA_SSD, NONE);
1619 if (ssdFilter != nullptr) {1619 if (ssdFilter != nullptr) {
@@ -1678,7 +1678,7 @@ EvictResult MmcMetaManager::EvictCallBackFunction(const std::string &key, const
1678 std::vector<MmcMemBlobDesc> evictBlobs;1678 std::vector<MmcMemBlobDesc> evictBlobs;
1679 objMeta->GetBlobsDesc(evictBlobs, srcFilter);1679 objMeta->GetBlobsDesc(evictBlobs, srcFilter);
1680 if (evictBlobs.empty()) {1680 if (evictBlobs.empty()) {
1681- MMC_LOG_DEBUG("Evict skip key=" << key << " from " << srcMediaType << ", no READABLE blobs");1681+ MMC_LOG_WARN("Evict skip key=" << key << " from " << srcMediaType << ", no READABLE blobs");
1682 TP_TRACE_END(TP_MMC_META_EVICT, MMC_OK);1682 TP_TRACE_END(TP_MMC_META_EVICT, MMC_OK);
1683 return EvictResult::FAIL;1683 return EvictResult::FAIL;
1684 }1684 }
@@ -1686,7 +1686,7 @@ EvictResult MmcMetaManager::EvictCallBackFunction(const std::string &key, const
1686 MmcMetaMetricManager::GetInstance().IncrementEvictCounter(evictRank);1686 MmcMetaMetricManager::GetInstance().IncrementEvictCounter(evictRank);
1687 1687 
1688 if (dstMedium == MEDIA_NONE) {1688 if (dstMedium == MEDIA_NONE) {
1689- MMC_LOG_WARN("Evict REMOVE key=" << key << " from " << srcMediaType << " reason=no_lower_tier");1689+ MMC_LOG_DEBUG("Evict REMOVE key=" << key << " from " << srcMediaType << " reason=no_lower_tier");
1690 TP_TRACE_END(TP_MMC_META_EVICT, MMC_OK);1690 TP_TRACE_END(TP_MMC_META_EVICT, MMC_OK);
1691 return EvictRemoveSrc(key, objMeta, srcFilter, evictRank, srcMediaType, dstMedium, true);1691 return EvictRemoveSrc(key, objMeta, srcFilter, evictRank, srcMediaType, dstMedium, true);
1692 }1692 }
@@ -1701,8 +1701,8 @@ EvictResult MmcMetaManager::EvictCallBackFunction(const std::string &key, const
1701 1701 
1702 uint64_t freeSize = globalAllocator_->GetFreeSpace(dstMedium);1702 uint64_t freeSize = globalAllocator_->GetFreeSpace(dstMedium);
1703 if (dstMedium != MEDIA_SSD && freeSize < objMeta->Size()) {1703 if (dstMedium != MEDIA_SSD && freeSize < objMeta->Size()) {
1704- MMC_LOG_WARN("Evict REMOVE key=" << key << " from " << srcMediaType << " reason=no_space, freeSize=" << freeSize1704+ MMC_LOG_DEBUG("Evict REMOVE key=" << key << " from " << srcMediaType
1705- << ", need=" << objMeta->Size());1705+ << " reason=no_space, freeSize=" << freeSize << ", need=" << objMeta->Size());
1706 TP_TRACE_END(TP_MMC_META_EVICT, MMC_OK);1706 TP_TRACE_END(TP_MMC_META_EVICT, MMC_OK);
1707 return EvictRemoveSrc(key, objMeta, srcFilter, evictRank, srcMediaType, dstMedium, false);1707 return EvictRemoveSrc(key, objMeta, srcFilter, evictRank, srcMediaType, dstMedium, false);
1708 }1708 }
Mtest/ut/testcase/memcache/csrc/config/CMakeLists.txt+1-0
@@ -1,6 +1,7 @@
1file(GLOB_RECURSE config_src1file(GLOB_RECURSE config_src
2 ${CMAKE_CURRENT_SOURCE_DIR}/test_mmc_configuration.cpp2 ${CMAKE_CURRENT_SOURCE_DIR}/test_mmc_configuration.cpp
3 ${CMAKE_CURRENT_SOURCE_DIR}/test_mmc_configuration_url_resolve.cpp3 ${CMAKE_CURRENT_SOURCE_DIR}/test_mmc_configuration_url_resolve.cpp
4+ ${CMAKE_CURRENT_SOURCE_DIR}/test_mmc_kv_parser.cpp
4)5)
5ADD_UNIT_TEST(test_config6ADD_UNIT_TEST(test_config
6 FILES ${config_src} ${TEST_SRC_PATHS}/ut_main.cpp7 FILES ${config_src} ${TEST_SRC_PATHS}/ut_main.cpp
Atest/ut/testcase/memcache/csrc/config/test_mmc_kv_parser.cpp+184-0
@@ -0,0 +1,184 @@
1+/*
2+ * Copyright (c) Huawei Technologies Co., Ltd. 2025-2025. All rights reserved.
3+ * MemCache_Hybrid is licensed under Mulan PSL v2.
4+ * You can use this software according to the terms and conditions of the Mulan PSL v2.
5+ * You may obtain a copy of Mulan PSL v2 at:
6+ * http://license.coscl.org.cn/MulanPSL2
7+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
8+ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
9+ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
10+ * See the Mulan PSL v2 for more details.
11+*/
12+ 
13+#include "gtest/gtest.h"
14+ 
15+#include <cstdio>
16+#include <fstream>
17+#include <string>
18+#include <unistd.h>
19+ 
20+#define private public
21+#include "mmc_kv_parser.h"
22+#undef private
23+ 
24+using namespace testing;
25+using namespace std;
26+using namespace ock::mmc;
27+ 
28+class TestKVParser : public testing::Test {
29+public:
30+ void SetUp() override {}
31+ void TearDown() override {}
32+};
33+ 
34+// ======================== ParseLine prefix filter ========================
35+ 
36+TEST_F(TestKVParser, ParseLine_AcceptsMmcPrefixedKey)
37+{
38+ KVParser parser;
39+ string line = "ock.mmc.test_key = test_value";
40+ EXPECT_EQ(parser.ParseLine(line), MMC_OK);
41+ EXPECT_EQ(parser.Size(), 1U);
42+}
43+ 
44+TEST_F(TestKVParser, ParseLine_SkipsNonMmcPrefixedKey)
45+{
46+ KVParser parser;
47+ string line = "ock.ubs.test_key = test_value";
48+ EXPECT_EQ(parser.ParseLine(line), MMC_OK);
49+ EXPECT_EQ(parser.Size(), 0U);
50+}
51+ 
52+TEST_F(TestKVParser, ParseLine_SkipsMmcWithoutTrailingDot)
53+{
54+ KVParser parser;
55+ string line = "ock.mmc_extra = value";
56+ EXPECT_EQ(parser.ParseLine(line), MMC_OK);
57+ EXPECT_EQ(parser.Size(), 0U);
58+}
59+ 
60+TEST_F(TestKVParser, ParseLine_SkipsKeyWithoutOckPrefix)
61+{
62+ KVParser parser;
63+ string line = "random_key = value";
64+ EXPECT_EQ(parser.ParseLine(line), MMC_OK);
65+ EXPECT_EQ(parser.Size(), 0U);
66+}
67+ 
68+TEST_F(TestKVParser, ParseLine_CommentAndEmptyStillSkipped)
69+{
70+ KVParser parser;
71+ string commentLine = "# ock.mmc.key = value";
72+ EXPECT_EQ(parser.ParseLine(commentLine), MMC_OK);
73+ string emptyLine = "";
74+ EXPECT_EQ(parser.ParseLine(emptyLine), MMC_OK);
75+ string spaces = " ";
76+ EXPECT_EQ(parser.ParseLine(spaces), MMC_OK);
77+ EXPECT_EQ(parser.Size(), 0U);
78+}
79+ 
80+TEST_F(TestKVParser, ParseLine_KeyWithOnlyMmcPrefixNoDot)
81+{
82+ KVParser parser;
83+ string line = "ock.mmc = value";
84+ EXPECT_EQ(parser.ParseLine(line), MMC_OK);
85+ EXPECT_EQ(parser.Size(), 0U);
86+}
87+ 
88+// ======================== FromFile integration ========================
89+ 
90+namespace {
91+ 
92+string WriteTempConfigFile(const string &content)
93+{
94+ char tmpl[] = "/tmp/mmc_kv_parser_test_XXXXXX";
95+ int fd = mkstemp(tmpl);
96+ if (fd == -1) {
97+ return {};
98+ }
99+ close(fd);
100+ 
101+ string filePath = string(tmpl) + ".conf";
102+ ofstream ofs(filePath, ios::trunc);
103+ if (!ofs.is_open()) {
104+ return {};
105+ }
106+ ofs << content;
107+ ofs.close();
108+ remove(tmpl);
109+ return filePath;
110+}
111+ 
112+} // namespace
113+ 
114+TEST_F(TestKVParser, FromFile_MixedKeys_OnlyMmcLoaded)
115+{
116+ string content = "# comment line\n"
117+ "ock.mmc.meta_service_url = tcp://127.0.0.1:5000\n"
118+ "ock.ubs.config = some_value\n"
119+ "\n"
120+ "ock.mmc.log_level = info\n"
121+ "random_key = random_value\n"
122+ "ock.mmc.local_service.world_size = 256\n";
123+ 
124+ string filePath = WriteTempConfigFile(content);
125+ ASSERT_FALSE(filePath.empty());
126+ 
127+ KVParser parser;
128+ EXPECT_EQ(parser.FromFile(filePath), MMC_OK);
129+ EXPECT_EQ(parser.Size(), 3U);
130+ 
131+ string key, value;
132+ parser.GetI(0, key, value);
133+ EXPECT_EQ(key, "ock.mmc.meta_service_url");
134+ parser.GetI(1, key, value);
135+ EXPECT_EQ(key, "ock.mmc.log_level");
136+ parser.GetI(2, key, value);
137+ EXPECT_EQ(key, "ock.mmc.local_service.world_size");
138+ 
139+ remove(filePath.c_str());
140+}
141+ 
142+TEST_F(TestKVParser, FromFile_AllNonMmc_Skipped)
143+{
144+ string content = "ock.ubs.key1 = value1\n"
145+ "other.key2 = value2\n";
146+ 
147+ string filePath = WriteTempConfigFile(content);
148+ ASSERT_FALSE(filePath.empty());
149+ 
150+ KVParser parser;
151+ EXPECT_EQ(parser.FromFile(filePath), MMC_OK);
152+ EXPECT_EQ(parser.Size(), 0U);
153+ 
154+ remove(filePath.c_str());
155+}
156+ 
157+TEST_F(TestKVParser, FromFile_AllMmc_AllLoaded)
158+{
159+ string content = "ock.mmc.key1 = value1\n"
160+ "ock.mmc.key2 = value2\n";
161+ 
162+ string filePath = WriteTempConfigFile(content);
163+ ASSERT_FALSE(filePath.empty());
164+ 
165+ KVParser parser;
166+ EXPECT_EQ(parser.FromFile(filePath), MMC_OK);
167+ EXPECT_EQ(parser.Size(), 2U);
168+ 
169+ remove(filePath.c_str());
170+}
171+ 
172+TEST_F(TestKVParser, FromFile_DuplicateMmcKey_ReturnsError)
173+{
174+ string content = "ock.mmc.key1 = value1\n"
175+ "ock.mmc.key1 = value2\n";
176+ 
177+ string filePath = WriteTempConfigFile(content);
178+ ASSERT_FALSE(filePath.empty());
179+ 
180+ KVParser parser;
181+ EXPECT_NE(parser.FromFile(filePath), MMC_OK);
182+ 
183+ remove(filePath.c_str());
184+}