fix：修复检视bug-memcache-AtomGit

M3rdparty/memfabric_hybrid+1-1

	@@ -1 +1 @@
1	-Subproject commit ba113e7a70552778de53dd633d886f6823a562ab	1	+Subproject commit 8a8699dce340c7488aea9b133877624d002cbf9f

Msrc/memcache/csrc/config/mmc_config_const.h+1-0

	@@ -110,6 +110,7 @@ constexpr uint16_t DEFAULT_REWARM_WATERMARK_VAL = 95U;
110	constexpr auto OCK_MMC_REWARM_DRAM_WATERMARK =	110	constexpr auto OCK_MMC_REWARM_DRAM_WATERMARK =
111	std::make_pair("ock.mmc.rewarm.dram_watermark", DEFAULT_REWARM_WATERMARK_VAL);	111	std::make_pair("ock.mmc.rewarm.dram_watermark", DEFAULT_REWARM_WATERMARK_VAL);
112	constexpr auto OCK_MMC_PREFETCH_ENABLED = std::make_pair("ock.mmc.storage.prefetch.enabled", false);	112	constexpr auto OCK_MMC_PREFETCH_ENABLED = std::make_pair("ock.mmc.storage.prefetch.enabled", false);
		113	+constexpr auto OCK_MMC_PENDING_WAIT_TIMEOUT_MS = std::make_pair("ock.mmc.storage.pending_wait.timeout", 300U);
113	constexpr auto OCK_MMC_LOCAL_SERVICE_STORAGE_ENABLED = std::make_pair("ock.mmc.local_service.storage.enabled", false);	114	constexpr auto OCK_MMC_LOCAL_SERVICE_STORAGE_ENABLED = std::make_pair("ock.mmc.local_service.storage.enabled", false);
114	constexpr auto OCK_MMC_LOCAL_SERVICE_DRAM_BEST_EFFORT =	115	constexpr auto OCK_MMC_LOCAL_SERVICE_DRAM_BEST_EFFORT =
115	std::make_pair("ock.mmc.local_service.dram.best_effort.enabled", false);	116	std::make_pair("ock.mmc.local_service.dram.best_effort.enabled", false);

Msrc/memcache/csrc/config/mmc_configuration.h+3-0

Msrc/memcache/csrc/entities/mmc_mem_obj_meta.cpp+6-2

	@@ -22,13 +22,17 @@ static const uint16_t MAX_NUM_BLOB_CHAINS = 5; // to make sure MmcMemObjMeta <=
22		22
23	Result MmcMemObjMeta::AddBlob(const MmcMemBlobPtr &blob)	23	Result MmcMemObjMeta::AddBlob(const MmcMemBlobPtr &blob)
24	{	24	{
		25	+ if (blob == nullptr) {
		26	+ MMC_LOG_ERROR("add blob: blob is nullptr");
		27	+ return MMC_ERROR;
		28	+ }
25	if (numBlobs_ != 0 && size_ != blob->Size()) {	29	if (numBlobs_ != 0 && size_ != blob->Size()) {
26	MMC_LOG_ERROR("add blob size:" << blob->Size() << " != meta size:" << size_);	30	MMC_LOG_ERROR("add blob size:" << blob->Size() << " != meta size:" << size_);
27	return MMC_ERROR;	31	return MMC_ERROR;
28	}	32	}
29	for (const auto &old : blobs_) {	33	for (const auto &old : blobs_) {
30	- if (old == nullptr \|\| blob == nullptr) {	34	+ if (old == nullptr) {
31	- MMC_LOG_ERROR("null ptr find: " << (old == nullptr));	35	+ MMC_LOG_ERROR("null ptr find in blobs_");
32	return MMC_ERROR;	36	return MMC_ERROR;
33	}	37	}
34	if (old->GetDesc() == blob->GetDesc()) {	38	if (old->GetDesc() == blob->GetDesc()) {

Msrc/memcache/csrc/local_service/mmc_local_service_default.cpp+37-10

	@@ -86,6 +86,10 @@ Result MmcLocalServiceDefault::Start(const mmc_local_service_config_t &config)
86		86
87	if (RegisterBm() != MMC_OK) {	87	if (RegisterBm() != MMC_OK) {
88	MMC_LOG_ERROR("Failed to register bm, name=" << name_ << ", bmRankId=" << options_.rankId);	88	MMC_LOG_ERROR("Failed to register bm, name=" << name_ << ", bmRankId=" << options_.rankId);
		89	+ if (ubsIoProxyPtr_ != nullptr) {
		90	+ ubsIoProxyPtr_->SetMetaEventCallback(nullptr);
		91	+ ubsIoProxyPtr_->DestroyUbsIo();
		92	+ }
89	ubsioEventPool_->Destroy();	93	ubsioEventPool_->Destroy();
90	DestroyBm();	94	DestroyBm();
91	metaNetClient_->Stop();	95	metaNetClient_->Stop();
	@@ -116,20 +120,34 @@ void MmcLocalServiceDefault::Stop()
116	MMC_LOG_WARN("MmcLocalServiceDefault has not been started" << ", rank: " << options_.rankId);	120	MMC_LOG_WARN("MmcLocalServiceDefault has not been started" << ", rank: " << options_.rankId);
117	return;	121	return;
118	}	122	}
		123	+ started_ = false;
119	StopConfigPolling();	124	StopConfigPolling();
		125	+
		126	+ // 先解绑 UBSIO callback，防止 UBSIO 线程继续投递新事件
		127	+ if (ubsIoProxyPtr_ != nullptr) {
		128	+ ubsIoProxyPtr_->SetMetaEventCallback(nullptr);
		129	+ }
		130	+
		131	+ // 排空已入队的事件
		132	+ if (ubsioEventPool_ != nullptr) {
		133	+ ubsioEventPool_->Destroy();
		134	+ }
		135	+
		136	+ // 回调已解绑、事件已排空，安全停止 UBSIO
		137	+ if (ubsIoProxyPtr_ != nullptr) {
		138	+ ubsIoProxyPtr_->DestroyUbsIo();
		139	+ ubsIoProxyPtr_ = nullptr;
		140	+ }
		141	+
120	DestroyBm();	142	DestroyBm();
121	if (metaNetClient_ != nullptr) {	143	if (metaNetClient_ != nullptr) {
122	metaNetClient_->Stop();	144	metaNetClient_->Stop();
123	metaNetClient_ = nullptr;	145	metaNetClient_ = nullptr;
124	}	146	}
125	- if (ubsioEventPool_ != nullptr) {
126	- ubsioEventPool_->Destroy();
127	- }
128	std::lock_guard<std::mutex> guardBlob(blobMutex_);	147	std::lock_guard<std::mutex> guardBlob(blobMutex_);
129	blobMap_.clear();	148	blobMap_.clear();
130	MMC_LOG_INFO("Stop MmcLocalServiceDefault (" << name_ << ") server " << options_.discoveryURL	149	MMC_LOG_INFO("Stop MmcLocalServiceDefault (" << name_ << ") server " << options_.discoveryURL
131	<< ", rank: " << options_.rankId);	150	<< ", rank: " << options_.rankId);
132	- started_ = false;
133	}	151	}
134		152
135	Result MmcLocalServiceDefault::InitBm()	153	Result MmcLocalServiceDefault::InitBm()
	@@ -234,15 +252,17 @@ Result MmcLocalServiceDefault::RegisterBm()
234	}	252	}
235	}	253	}
236		254
237	- for (auto &desc : descs) {	255	+ for (auto descIt = descs.begin(); descIt != descs.end();) {
238	- if (desc.mediaType_ == MEDIA_SSD) {	256	+ if (descIt->mediaType_ == MEDIA_SSD) {
239	if (ubsIoProxyPtr_ != nullptr && !ubsIoProxyPtr_->Exist(key)) {	257	if (ubsIoProxyPtr_ != nullptr && !ubsIoProxyPtr_->Exist(key)) {
240	MMC_LOG_WARN("SSD blob " << key << " no longer exists on SSD, removing from rebuild");	258	MMC_LOG_WARN("SSD blob " << key << " no longer exists on SSD, removing from rebuild");
		259	+ descIt = descs.erase(descIt);
241	continue;	260	continue;
242	}	261	}
243	}	262	}
244	- req.blobList_.push_back({key, desc});	263	+ req.blobList_.push_back({key, *descIt});
245	++count;	264	++count;
		265	+ ++descIt;
246	}	266	}
247		267
248	if (!hasSsdInMap && ubsIoProxyPtr_ != nullptr && ubsIoProxyPtr_->Exist(key)) {	268	if (!hasSsdInMap && ubsIoProxyPtr_ != nullptr && ubsIoProxyPtr_->Exist(key)) {
	@@ -257,7 +277,11 @@ Result MmcLocalServiceDefault::RegisterBm()
257	}	277	}
258	}	278	}
259	}	279	}
260	- ++it;	280	+ if (descs.empty()) {
		281	+ it = blobMap_.erase(it);
		282	+ } else {
		283	+ ++it;
		284	+ }
261		285
262	if (count >= blobRebuildSendMaxCount \|\| it == end) {	286	if (count >= blobRebuildSendMaxCount \|\| it == end) {
263	MMC_LOG_INFO("mmc meta blob rebuild count " << req.blobList_.size());	287	MMC_LOG_INFO("mmc meta blob rebuild count " << req.blobList_.size());
	@@ -282,12 +306,15 @@ Result MmcLocalServiceDefault::InitUbsIo(int32_t deviceId, const std::string &co
282	"metaNetClient_ not ready when registering UBS IO callback", MMC_NOT_INITIALIZED);	306	"metaNetClient_ not ready when registering UBS IO callback", MMC_NOT_INITIALIZED);
283	ubsIoProxyPtr_ = ubsIoProxy;	307	ubsIoProxyPtr_ = ubsIoProxy;
284		308
285	- // Register UBS IO metadata event callback (before InitUbsIo to avoid missing recovery events)
286	ubsIoProxy->SetMetaEventCallback([this](int type, const std::vector<std::string> &keys) {	309	ubsIoProxy->SetMetaEventCallback([this](int type, const std::vector<std::string> &keys) {
287	ubsioEventPool_->Enqueue([this, type, keys]() { HandleUbsIoMetaEvents(type, keys); });	310	ubsioEventPool_->Enqueue([this, type, keys]() { HandleUbsIoMetaEvents(type, keys); });
288	});	311	});
289		312
290	- return ubsIoProxy->InitUbsIo(deviceId, confPath);	313	+ auto ret = ubsIoProxy->InitUbsIo(deviceId, confPath);
		314	+ if (ret != MMC_OK) {
		315	+ ubsIoProxy->SetMetaEventCallback(nullptr);
		316	+ }
		317	+ return ret;
291	}	318	}
292		319
293	Result MmcLocalServiceDefault::UpdateMetaBackup(const std::vector<uint32_t> &ops, const std::vector<std::string> &keys,	320	Result MmcLocalServiceDefault::UpdateMetaBackup(const std::vector<uint32_t> &ops, const std::vector<std::string> &keys,

Msrc/memcache/csrc/meta_service/mmc_meta_manager.cpp+108-83

	@@ -146,11 +146,13 @@ static BlobClassification ClassifyBlobs(const MmcMemObjMetaPtr &objMeta, MmcBlob
146	}	146	}
147		147
148	static Result WaitPendingRewarm(const std::string &key, MmcMemBlobPtr &selectedBlob, const MmcMemBlobPtr &pendingBlob,	148	static Result WaitPendingRewarm(const std::string &key, MmcMemBlobPtr &selectedBlob, const MmcMemBlobPtr &pendingBlob,
149	- const MmcMemBlobPtr &lowerBlob, std::unique_lock<std::mutex> &guard, int waitMs)	149	+ const MmcMemBlobPtr &lowerBlob, std::unique_lock<std::mutex> &guard,
		150	+ const MmcMetaExtConfig &extConfig)
150	{	151	{
151	- MMC_LOG_DEBUG("rewarm in progress for key " << key << ", waiting " << waitMs << "ms");	152	+ auto timeout = std::chrono::milliseconds(extConfig.pendingWaitTimeoutMs);
		153	+ MMC_LOG_DEBUG("rewarm in progress for key " << key << ", waiting " << timeout.count() << "ms");
152	TP_TRACE_BEGIN(TP_MMC_META_GET_WAIT_REWARM);	154	TP_TRACE_BEGIN(TP_MMC_META_GET_WAIT_REWARM);
153	- bool waitOk = pendingBlob->WaitUntilReadable(guard, std::chrono::milliseconds(waitMs));	155	+ bool waitOk = pendingBlob->WaitUntilReadable(guard, timeout);
154	TP_TRACE_END(TP_MMC_META_GET_WAIT_REWARM, waitOk ? MMC_OK : MMC_TIMEOUT);	156	TP_TRACE_END(TP_MMC_META_GET_WAIT_REWARM, waitOk ? MMC_OK : MMC_TIMEOUT);
155	if (!waitOk) {	157	if (!waitOk) {
156	MMC_LOG_ERROR("rewarm wait timeout for key " << key);	158	MMC_LOG_ERROR("rewarm wait timeout for key " << key);
	@@ -207,8 +209,6 @@ Result MmcMetaManager::TryRewarmForGet(const std::string &key, uint64_t operateI
207	Result MmcMetaManager::ResolveAndFillMetaDesc(const std::string &key, uint64_t operateId, MmcBlobFilterPtr filterPtr,	209	Result MmcMetaManager::ResolveAndFillMetaDesc(const std::string &key, uint64_t operateId, MmcBlobFilterPtr filterPtr,
208	const MmcMemObjMetaPtr &memObj, MmcMemMetaDesc &objMeta)	210	const MmcMemObjMetaPtr &memObj, MmcMemMetaDesc &objMeta)
209	{	211	{
210	- constexpr int rewarmWaitMs = 500;
211	-
212	MMC_LOG_DEBUG("ResolveAndFillMetaDesc key=" << key);	212	MMC_LOG_DEBUG("ResolveAndFillMetaDesc key=" << key);
213		213
214	std::unique_lock<std::mutex> guard(memObj->Mutex());	214	std::unique_lock<std::mutex> guard(memObj->Mutex());
	@@ -222,7 +222,7 @@ Result MmcMetaManager::ResolveAndFillMetaDesc(const std::string &key, uint64_t o
222	}	222	}
223		223
224	if (selectedBlob == nullptr && pendingBlob != nullptr && lowerBlob != nullptr) {	224	if (selectedBlob == nullptr && pendingBlob != nullptr && lowerBlob != nullptr) {
225	- auto waitRet = WaitPendingRewarm(key, selectedBlob, pendingBlob, lowerBlob, guard, rewarmWaitMs);	225	+ auto waitRet = WaitPendingRewarm(key, selectedBlob, pendingBlob, lowerBlob, guard, extConfig_);
226	if (waitRet != MMC_OK) {	226	if (waitRet != MMC_OK) {
227	return waitRet;	227	return waitRet;
228	}	228	}
	@@ -245,6 +245,15 @@ Result MmcMetaManager::ResolveAndFillMetaDesc(const std::string &key, uint64_t o
245	}	245	}
246	}	246	}
247		247
		248	+ if (selectedBlob == nullptr) {
		249	+ MMC_LOG_WARN("Get: no blob available for key " << key);
		250	+ objMeta.prot_ = memObj->Prot();
		251	+ objMeta.priority_ = memObj->Priority();
		252	+ objMeta.size_ = memObj->Size();
		253	+ objMeta.numBlobs_ = 0;
		254	+ return MMC_OK;
		255	+ }
		256	+
248	objMeta.prot_ = memObj->Prot();	257	objMeta.prot_ = memObj->Prot();
249	objMeta.priority_ = memObj->Priority();	258	objMeta.priority_ = memObj->Priority();
250	objMeta.size_ = memObj->Size();	259	objMeta.size_ = memObj->Size();
	@@ -274,74 +283,37 @@ Result MmcMetaManager::GetByRank(const std::vector<std::string> &keys, uint64_t
274	uint32_t opRankId = GetRankIdByOperateId(operateId);	283	uint32_t opRankId = GetRankIdByOperateId(operateId);
275	uint32_t opSeq = GetSequenceByOperateId(operateId);	284	uint32_t opSeq = GetSequenceByOperateId(operateId);
276		285
		286	+ CheckAndEvict(MEDIA_DRAM, 0);
		287	+
277	std::map<uint32_t, std::vector<RewarmEntry>> rankGroups;	288	std::map<uint32_t, std::vector<RewarmEntry>> rankGroups;
278	std::vector<PendingRewarmWait> pendingWaitList;	289	std::vector<PendingRewarmWait> pendingWaitList;
		290	+ std::vector<DeferredLockEntry> deferredLockList;
279	TP_TRACE_BEGIN(TP_MMC_META_BATCH_GET_CLASSIFY);	291	TP_TRACE_BEGIN(TP_MMC_META_BATCH_GET_CLASSIFY);
280	- ClassifyAndGroupKeys(keys, opRankId, opSeq, objMetas, rankGroups, pendingWaitList);	292	+ ClassifyAndGroupKeys(keys, opRankId, opSeq, objMetas, rankGroups, pendingWaitList, deferredLockList);
281	TP_TRACE_END(TP_MMC_META_BATCH_GET_CLASSIFY, MMC_OK);	293	TP_TRACE_END(TP_MMC_META_BATCH_GET_CLASSIFY, MMC_OK);
282		294
		295	+ // 先异步提交 rewarm/pendingWait，后台启动 RPC；主线程同步给 selectedBlob 加读锁，两者并行缩小时隙
283	std::vector<std::future<void>> futures;	296	std::vector<std::future<void>> futures;
284	- if (!rankGroups.empty() \|\| !pendingWaitList.empty()) {	297	+ for (auto &[rank, group] : rankGroups) {
285	- TP_TRACE_BEGIN(TP_MMC_META_BATCH_GET_REWARM_WAIT);	298	+ futures.push_back(rewarmThreadPool_->Enqueue([this, rank, &group, &keys, opRankId, opSeq, &objMetas]() {
286	- for (auto &[rank, group] : rankGroups) {	299	+ RewarmRankGroup(rank, group, keys, opRankId, opSeq, objMetas);
287	- futures.push_back(rewarmThreadPool_->Enqueue([this, rank, &group, &keys, opRankId, opSeq, &objMetas]() {	300	+ }));
288	- RewarmRankGroup(rank, group, keys, opRankId, opSeq, objMetas);	301	+ }
289	- }));	302	+ for (auto &w : pendingWaitList) {
290	- }	303	+ futures.push_back(rewarmThreadPool_->Enqueue([this, &keys, opRankId, opSeq, &objMetas, &w]() {
291	-	304	+ PendingWaitAndFill(keys, opRankId, opSeq, objMetas, w);
292	- for (auto &w : pendingWaitList) {	305	+ }));
293	- futures.push_back(rewarmThreadPool_->Enqueue([this, &keys, opRankId, opSeq, &objMetas, &w]() {
294	- PendingWaitAndFill(keys, opRankId, opSeq, objMetas, w);
295	- }));
296	- }
297	-
298	- for (auto &f : futures) {
299	- try {
300	- f.get();
301	- } catch (const std::exception &e) {
302	- MMC_LOG_WARN("GetByRank future failed: " << e.what());
303	- }
304	- }
305	- TP_TRACE_END(TP_MMC_META_BATCH_GET_REWARM_WAIT, MMC_OK);
306	}	306	}
307		307
308	- size_t metaCount = objMetas.size();	308	+ AttachReadLocks(keys, opRankId, opSeq, objMetas, deferredLockList);
309	- TP_TRACE_BEGIN(TP_MMC_META_BATCH_GET_READ_START);	309	+
310	- for (size_t i = 0; i < keys.size() && i < metaCount; ++i) {	310	+ for (auto &f : futures) {
311	- auto &objMeta = objMetas[i];	311	+ try {
312	- if (objMeta.numBlobs_ == 0 \|\| objMeta.blobs_.empty()) {	312	+ f.get();
313	- continue;	313	+ } catch (const std::exception &e) {
314	- }	314	+ MMC_LOG_WARN("GetByRank future failed: " << e.what());
315	- MmcMemObjMetaPtr memObj;
316	- if (metaContainer_->Get(keys[i], memObj) != MMC_OK \|\| memObj == nullptr) {
317	- MMC_LOG_WARN("GetByRank: deferred READ_START memObj not found for key=" << keys[i]);
318	- objMeta.blobs_.clear();
319	- objMeta.numBlobs_ = 0;
320	- continue;
321	- }
322	- {
323	- std::unique_lock<std::mutex> guard(memObj->Mutex());
324	- for (auto &desc : objMeta.blobs_) {
325	- MmcBlobFilterPtr filter =
326	- MmcMakeRef<MmcBlobFilter>(desc.rank_, static_cast<MediaType>(desc.mediaType_), READABLE);
327	- auto blobs = memObj->GetBlobs(filter);
328	- if (blobs.empty()) {
329	- MMC_LOG_WARN("GetByRank: deferred READ_START blob not found for key=" << keys[i]);
330	- objMeta.blobs_.clear();
331	- objMeta.numBlobs_ = 0;
332	- break;
333	- }
334	- auto ret = blobs[0]->UpdateState(keys[i], opRankId, opSeq, MMC_READ_START);
335	- if (ret != MMC_OK) {
336	- MMC_LOG_WARN("GetByRank: deferred READ_START failed for key=" << keys[i] << ", ret=" << ret);
337	- objMeta.blobs_.clear();
338	- objMeta.numBlobs_ = 0;
339	- break;
340	- }
341	- }
342	}	315	}
343	}	316	}
344	- TP_TRACE_END(TP_MMC_META_BATCH_GET_READ_START, MMC_OK);
345		317
346	return MMC_OK;	318	return MMC_OK;
347	}	319	}
	@@ -349,7 +321,8 @@ Result MmcMetaManager::GetByRank(const std::vector<std::string> &keys, uint64_t
349	void MmcMetaManager::ClassifyAndGroupKeys(const std::vector<std::string> &keys, uint32_t opRankId, uint32_t opSeq,	321	void MmcMetaManager::ClassifyAndGroupKeys(const std::vector<std::string> &keys, uint32_t opRankId, uint32_t opSeq,
350	std::vector<MmcMemMetaDesc> &objMetas,	322	std::vector<MmcMemMetaDesc> &objMetas,
351	std::map<uint32_t, std::vector<RewarmEntry>> &rankGroups,	323	std::map<uint32_t, std::vector<RewarmEntry>> &rankGroups,
352	- std::vector<PendingRewarmWait> &pendingWaitList)	324	+ std::vector<PendingRewarmWait> &pendingWaitList,
		325	+ std::vector<DeferredLockEntry> &deferredLockList)
353	{	326	{
354	size_t keyCount = keys.size();	327	size_t keyCount = keys.size();
355		328
	@@ -395,11 +368,8 @@ void MmcMetaManager::ClassifyAndGroupKeys(const std::vector<std::string> &keys,
395		368
396	if (selectedBlob != nullptr) {	369	if (selectedBlob != nullptr) {
397	MmcMetaMetricManager::GetInstance().IncrementGetHitDramCounter(selectedBlob->GetDesc().rank_);	370	MmcMetaMetricManager::GetInstance().IncrementGetHitDramCounter(selectedBlob->GetDesc().rank_);
398	- objMetas[i].prot_ = memObj->Prot();	371	+ objMetas[i].FillFrom(memObj, selectedBlob);
399	- objMetas[i].priority_ = memObj->Priority();	372	+ deferredLockList.push_back({i, memObj});
400	- objMetas[i].size_ = memObj->Size();
401	- objMetas[i].blobs_.push_back(selectedBlob->GetDesc());
402	- objMetas[i].numBlobs_ = 1;
403	} else if (pendingBlob != nullptr) {	373	} else if (pendingBlob != nullptr) {
404	// rewarm already in progress by another thread, wait concurrently	374	// rewarm already in progress by another thread, wait concurrently
405	pendingWaitList.push_back({i, memObj, pendingBlob});	375	pendingWaitList.push_back({i, memObj, pendingBlob});
	@@ -408,11 +378,8 @@ void MmcMetaManager::ClassifyAndGroupKeys(const std::vector<std::string> &keys,
408	MediaType dstMedia = MoveUp(srcMedia);	378	MediaType dstMedia = MoveUp(srcMedia);
409	if (dstMedia == MEDIA_HBM \|\| dstMedia == MEDIA_NONE) {	379	if (dstMedia == MEDIA_HBM \|\| dstMedia == MEDIA_NONE) {
410	MmcMetaMetricManager::GetInstance().IncrementGetHitDramCounter(lowerBlob->GetDesc().rank_);	380	MmcMetaMetricManager::GetInstance().IncrementGetHitDramCounter(lowerBlob->GetDesc().rank_);
411	- objMetas[i].prot_ = memObj->Prot();	381	+ objMetas[i].FillFrom(memObj, lowerBlob);
412	- objMetas[i].priority_ = memObj->Priority();	382	+ deferredLockList.push_back({i, memObj});
413	- objMetas[i].size_ = memObj->Size();
414	- objMetas[i].blobs_.push_back(lowerBlob->GetDesc());
415	- objMetas[i].numBlobs_ = 1;
416	} else {	383	} else {
417	auto readRet = lowerBlob->UpdateState(keys[i], opRankId, opSeq, MMC_READ_START);	384	auto readRet = lowerBlob->UpdateState(keys[i], opRankId, opSeq, MMC_READ_START);
418	if (readRet != MMC_OK) {	385	if (readRet != MMC_OK) {
	@@ -438,6 +405,18 @@ void MmcMetaManager::ClassifyAndGroupKeys(const std::vector<std::string> &keys,
438	continue;	405	continue;
439	}	406	}
440	newBlobs[0]->SetRewarmOrigin();	407	newBlobs[0]->SetRewarmOrigin();
		408	+ auto statusRet = newBlobs[0]->UpdateState(keys[i], lowerBlob->GetDesc().rank_, 0, MMC_ALLOCATED_OK);
		409	+ if (statusRet != MMC_OK) {
		410	+ MMC_LOG_WARN("key: " << keys[i] << " rewarm ALLOCATED_OK failed, ret: " << statusRet);
		411	+ globalAllocator_->Free(newBlobs);
		412	+ auto finishRet = lowerBlob->UpdateState(keys[i], opRankId, opSeq, MMC_READ_FINISH);
		413	+ if (finishRet != MMC_OK) {
		414	+ MMC_LOG_WARN("key: " << keys[i]
		415	+ << " READ_FINISH rollback after ALLOCATED_OK fail, ret: " << finishRet);
		416	+ }
		417	+ MmcMetaMetricManager::GetInstance().IncrementRewarmFailCounter(lowerBlob->GetDesc().rank_);
		418	+ continue;
		419	+ }
441	auto addRet = memObj->AddBlob(newBlobs[0]);	420	auto addRet = memObj->AddBlob(newBlobs[0]);
442	if (addRet != MMC_OK) {	421	if (addRet != MMC_OK) {
443	MMC_LOG_WARN("key: " << keys[i] << " rewarm AddBlob failed, ret: " << addRet);	422	MMC_LOG_WARN("key: " << keys[i] << " rewarm AddBlob failed, ret: " << addRet);
	@@ -450,7 +429,6 @@ void MmcMetaManager::ClassifyAndGroupKeys(const std::vector<std::string> &keys,
450	MmcMetaMetricManager::GetInstance().IncrementRewarmFailCounter(lowerBlob->GetDesc().rank_);	429	MmcMetaMetricManager::GetInstance().IncrementRewarmFailCounter(lowerBlob->GetDesc().rank_);
451	continue;	430	continue;
452	}	431	}
453	- // 与 FreeBlobs 的 DecrementRewarmBytesCurrent 配对，否则量表下溢
454	MmcMetaMetricManager::GetInstance().IncrementRewarmBytesCurrent(newBlobs[0]->Size(),	432	MmcMetaMetricManager::GetInstance().IncrementRewarmBytesCurrent(newBlobs[0]->Size(),
455	newBlobs[0]->GetDesc().rank_);	433	newBlobs[0]->GetDesc().rank_);
456	RewarmEntry entry;	434	RewarmEntry entry;
	@@ -539,6 +517,19 @@ Result MmcMetaManager::ApplyRewarm(const std::string &key, RewarmEntry &entry, M
539	return ret;	517	return ret;
540	}	518	}
541		519
		520	+ auto readStartRet = dstBlob->UpdateState(key, ctx.opRankId, ctx.opSeq, MMC_READ_START);
		521	+ if (readStartRet != MMC_OK) {
		522	+ MMC_LOG_WARN("READ_START failed after rewarm, key=" << key << ", ret=" << readStartRet);
		523	+ MmcBlobFilterPtr rbFilter = MmcMakeRef<MmcBlobFilter>(dstBlob->GetDesc().rank_, ctx.dstMedia, NONE);
		524	+ entry.memObj->FreeBlobs(key, globalAllocator_, rbFilter, false);
		525	+ auto finishRet = entry.ssdBlob->UpdateState(key, entry.opRankId, entry.opSeq, MMC_READ_FINISH);
		526	+ if (finishRet != MMC_OK) {
		527	+ MMC_LOG_WARN("Failed to release SSD read lease after READ_START failed, key=" << key
		528	+ << ", ret=" << finishRet);
		529	+ }
		530	+ return readStartRet;
		531	+ }
		532	+
542	// 保留源 SSD blob 作为冗余副本，若 DRAM 被淘汰则无需重复 CopyBlob	533	// 保留源 SSD blob 作为冗余副本，若 DRAM 被淘汰则无需重复 CopyBlob
543	objMeta.prot_ = entry.memObj->Prot();	534	objMeta.prot_ = entry.memObj->Prot();
544	objMeta.priority_ = entry.memObj->Priority();	535	objMeta.priority_ = entry.memObj->Priority();
	@@ -615,21 +606,55 @@ void MmcMetaManager::RewarmRankGroup(uint32_t rank, std::vector<RewarmEntry> &gr
615	MMC_LOG_DEBUG("finalized " << okCnt << "/" << groupSize << " keys for rank=" << rank);	606	MMC_LOG_DEBUG("finalized " << okCnt << "/" << groupSize << " keys for rank=" << rank);
616	}	607	}
617		608
		609	+void MmcMetaManager::AttachReadLocks(const std::vector<std::string> &keys, uint32_t opRankId, uint32_t opSeq,
		610	+ std::vector<MmcMemMetaDesc> &objMetas,
		611	+ std::vector<DeferredLockEntry> &deferredLockList)
		612	+{
		613	+ TP_TRACE_BEGIN(TP_MMC_META_BATCH_GET_READ_START);
		614	+ for (auto &entry : deferredLockList) {
		615	+ auto &objMeta = objMetas[entry.index];
		616	+ if (objMeta.numBlobs_ == 0 \|\| objMeta.blobs_.empty()) {
		617	+ continue;
		618	+ }
		619	+ std::unique_lock<std::mutex> guard(entry.memObj->Mutex());
		620	+ for (auto &desc : objMeta.blobs_) {
		621	+ MmcBlobFilterPtr filter =
		622	+ MmcMakeRef<MmcBlobFilter>(desc.rank_, static_cast<MediaType>(desc.mediaType_), READABLE);
		623	+ auto blobs = entry.memObj->GetBlobs(filter);
		624	+ if (blobs.empty()) {
		625	+ MMC_LOG_WARN("GetByRank: deferred READ_START blob not found for key=" << keys[entry.index]);
		626	+ objMeta.blobs_.clear();
		627	+ objMeta.numBlobs_ = 0;
		628	+ break;
		629	+ }
		630	+ auto ret = blobs[0]->UpdateState(keys[entry.index], opRankId, opSeq, MMC_READ_START);
		631	+ if (ret != MMC_OK) {
		632	+ MMC_LOG_WARN("GetByRank: deferred READ_START failed for key=" << keys[entry.index] << ", ret=" << ret);
		633	+ objMeta.blobs_.clear();
		634	+ objMeta.numBlobs_ = 0;
		635	+ break;
		636	+ }
		637	+ }
		638	+ }
		639	+ TP_TRACE_END(TP_MMC_META_BATCH_GET_READ_START, MMC_OK);
		640	+}
		641	+
618	void MmcMetaManager::PendingWaitAndFill(const std::vector<std::string> &keys, uint32_t opRankId, uint32_t opSeq,	642	void MmcMetaManager::PendingWaitAndFill(const std::vector<std::string> &keys, uint32_t opRankId, uint32_t opSeq,
619	std::vector<MmcMemMetaDesc> &objMetas, PendingRewarmWait &w)	643	std::vector<MmcMemMetaDesc> &objMetas, PendingRewarmWait &w)
620	{	644	{
621	- static constexpr auto kPendingWaitTimeout = std::chrono::milliseconds(300);	645	+ auto timeout = std::chrono::milliseconds(extConfig_.pendingWaitTimeoutMs);
622	std::unique_lock<std::mutex> guard(w.memObj->Mutex());	646	std::unique_lock<std::mutex> guard(w.memObj->Mutex());
623	if (w.pendingBlob->State() != READABLE) {	647	if (w.pendingBlob->State() != READABLE) {
624	- w.pendingBlob->WaitUntilReadable(guard, kPendingWaitTimeout);	648	+ w.pendingBlob->WaitUntilReadable(guard, timeout);
625	}	649	}
626		650
627	if (w.pendingBlob->State() == READABLE) {	651	if (w.pendingBlob->State() == READABLE) {
628	- objMetas[w.index].prot_ = w.memObj->Prot();	652	+ auto readStartRet = w.pendingBlob->UpdateState(keys[w.index], opRankId, opSeq, MMC_READ_START);
629	- objMetas[w.index].priority_ = w.memObj->Priority();	653	+ if (readStartRet != MMC_OK) {
630	- objMetas[w.index].size_ = w.memObj->Size();	654	+ MMC_LOG_WARN("key: " << keys[w.index] << " READ_START failed after pending rewarm, ret=" << readStartRet);
631	- objMetas[w.index].blobs_.push_back(w.pendingBlob->GetDesc());	655	+ return;
632	- objMetas[w.index].numBlobs_ = 1;	656	+ }
		657	+ objMetas[w.index].FillFrom(w.memObj, w.pendingBlob);
633	MmcMetaMetricManager::GetInstance().IncrementGetHitDramCounter(w.pendingBlob->GetDesc().rank_);	658	MmcMetaMetricManager::GetInstance().IncrementGetHitDramCounter(w.pendingBlob->GetDesc().rank_);
634	} else {	659	} else {
635	MMC_LOG_WARN("key: " << keys[w.index] << " pending rewarm timeout or state not readable, state="	660	MMC_LOG_WARN("key: " << keys[w.index] << " pending rewarm timeout or state not readable, state="

Msrc/memcache/csrc/meta_service/mmc_meta_manager.h+21-1

	@@ -82,6 +82,16 @@ struct MmcMemMetaDesc {
82	}	82	}
83	}	83	}
84		84
		85	+ void FillFrom(const MmcMemObjMetaPtr &memObj, const MmcMemBlobPtr &blob)
		86	+ {
		87	+ prot_ = memObj->Prot();
		88	+ priority_ = memObj->Priority();
		89	+ size_ = memObj->Size();
		90	+ blobs_.clear();
		91	+ blobs_.push_back(blob->GetDesc());
		92	+ numBlobs_ = static_cast<uint8_t>(blobs_.size());
		93	+ }
		94	+
85	uint16_t Prot()	95	uint16_t Prot()
86	{	96	{
87	return prot_;	97	return prot_;
	@@ -105,6 +115,7 @@ struct MmcMemMetaDesc {
105		115
106	struct MmcMetaExtConfig {	116	struct MmcMetaExtConfig {
107	bool prefetchEnabled = false;	117	bool prefetchEnabled = false;
		118	+ uint64_t pendingWaitTimeoutMs = 300U;
108	};	119	};
109		120
110	class MmcMetaManager : public MmcReferable {	121	class MmcMetaManager : public MmcReferable {
	@@ -425,6 +436,11 @@ private:
425	MmcMemBlobPtr pendingBlob;	436	MmcMemBlobPtr pendingBlob;
426	};	437	};
427		438
		439	+ struct DeferredLockEntry {
		440	+ size_t index;
		441	+ MmcMemObjMetaPtr memObj;
		442	+ };
		443	+
428	struct BatchRpcData {	444	struct BatchRpcData {
429	std::vector<std::string> keys;	445	std::vector<std::string> keys;
430	std::vector<MmcMemBlobDesc> srcBlobs;	446	std::vector<MmcMemBlobDesc> srcBlobs;
	@@ -442,11 +458,15 @@ private:
442	void ClassifyAndGroupKeys(const std::vector<std::string> &keys, uint32_t opRankId, uint32_t opSeq,	458	void ClassifyAndGroupKeys(const std::vector<std::string> &keys, uint32_t opRankId, uint32_t opSeq,
443	std::vector<MmcMemMetaDesc> &objMetas,	459	std::vector<MmcMemMetaDesc> &objMetas,
444	std::map<uint32_t, std::vector<RewarmEntry>> &rankGroups,	460	std::map<uint32_t, std::vector<RewarmEntry>> &rankGroups,
445	- std::vector<PendingRewarmWait> &pendingWaitList);	461	+ std::vector<PendingRewarmWait> &pendingWaitList,
		462	+ std::vector<DeferredLockEntry> &deferredLockList);
446		463
447	void RewarmRankGroup(uint32_t rank, std::vector<RewarmEntry> &group, const std::vector<std::string> &keys,	464	void RewarmRankGroup(uint32_t rank, std::vector<RewarmEntry> &group, const std::vector<std::string> &keys,
448	uint32_t opRankId, uint32_t opSeq, std::vector<MmcMemMetaDesc> &objMetas);	465	uint32_t opRankId, uint32_t opSeq, std::vector<MmcMemMetaDesc> &objMetas);
449		466
		467	+ void AttachReadLocks(const std::vector<std::string> &keys, uint32_t opRankId, uint32_t opSeq,
		468	+ std::vector<MmcMemMetaDesc> &objMetas, std::vector<DeferredLockEntry> &deferredLockList);
		469	+
450	void PendingWaitAndFill(const std::vector<std::string> &keys, uint32_t opRankId, uint32_t opSeq,	470	void PendingWaitAndFill(const std::vector<std::string> &keys, uint32_t opRankId, uint32_t opSeq,
451	std::vector<MmcMemMetaDesc> &objMetas, PendingRewarmWait &w);	471	std::vector<MmcMemMetaDesc> &objMetas, PendingRewarmWait &w);
452		472

Msrc/memcache/csrc/meta_service/mmc_meta_service.cpp+1-0

	@@ -81,6 +81,7 @@ Result MmcMetaService::Start(const mmc_meta_service_config_t &options)
81	metaMgrProxy_ = MmcMakeRef<MmcMetaMgrProxy>(metaNetServer_).Get();	81	metaMgrProxy_ = MmcMakeRef<MmcMetaMgrProxy>(metaNetServer_).Get();
82	MmcMetaExtConfig extConfig{};	82	MmcMetaExtConfig extConfig{};
83	extConfig.prefetchEnabled = options.prefetchEnabled;	83	extConfig.prefetchEnabled = options.prefetchEnabled;
		84	+ extConfig.pendingWaitTimeoutMs = options.pendingWaitTimeoutMs;
84	MMC_RETURN_ERROR(metaMgrProxy_->Start(options_.leaseTtlMs, options.evictThresholdHigh, options.evictThresholdLow,	85	MMC_RETURN_ERROR(metaMgrProxy_->Start(options_.leaseTtlMs, options.evictThresholdHigh, options.evictThresholdLow,
85	options.rewarmDramWatermark, extConfig),	86	options.rewarmDramWatermark, extConfig),
86	"Failed to start meta mgr proxy of meta service " << name_);	87	"Failed to start meta mgr proxy of meta service " << name_);

Msrc/memcache/csrc/mmc.cpp+1-0

	@@ -48,6 +48,7 @@ mmc_meta_service_config_t create_default_meta_config()
48	config.evictThresholdLow = 80U;	48	config.evictThresholdLow = 80U;
49	config.rewarmDramWatermark = DEFAULT_REWARM_HIGH_WATERMARK;	49	config.rewarmDramWatermark = DEFAULT_REWARM_HIGH_WATERMARK;
50	config.prefetchEnabled = false;	50	config.prefetchEnabled = false;
		51	+ config.pendingWaitTimeoutMs = 300U;
51	config.leaseTtlMs = MMC_DATA_TTL_MS;	52	config.leaseTtlMs = MMC_DATA_TTL_MS;
52	config.accTlsConfig.tlsEnable = false;	53	config.accTlsConfig.tlsEnable = false;
53	config.configStoreTlsConfig.tlsEnable = false;	54	config.configStoreTlsConfig.tlsEnable = false;

Msrc/memcache/csrc/mmc_client.cpp+2-0

	@@ -460,6 +460,7 @@ MMC_API int32_t mmcc_batch_get(const char **keys, uint32_t keys_count, mmc_buffe
460	MMC_VALIDATE_RETURN(keys != nullptr, "invalid param, keys is null", MMC_INVALID_PARAM);	460	MMC_VALIDATE_RETURN(keys != nullptr, "invalid param, keys is null", MMC_INVALID_PARAM);
461	MMC_VALIDATE_RETURN(keys_count != 0, "invalid param, keys_count: " << keys_count, MMC_INVALID_PARAM);	461	MMC_VALIDATE_RETURN(keys_count != 0, "invalid param, keys_count: " << keys_count, MMC_INVALID_PARAM);
462	MMC_VALIDATE_RETURN(bufs != nullptr, "invalid param, bufs is null", MMC_INVALID_PARAM);	462	MMC_VALIDATE_RETURN(bufs != nullptr, "invalid param, bufs is null", MMC_INVALID_PARAM);
		463	+ MMC_VALIDATE_RETURN(results != nullptr, "invalid param, results is null", MMC_INVALID_PARAM);
463	MMC_VALIDATE_RETURN(MmcClientDefault::GetInstance() != nullptr, "client is not initialize", MMC_CLIENT_NOT_INIT);	464	MMC_VALIDATE_RETURN(MmcClientDefault::GetInstance() != nullptr, "client is not initialize", MMC_CLIENT_NOT_INIT);
464		465
465	std::vector<std::string> keys_vector;	466	std::vector<std::string> keys_vector;
	@@ -495,6 +496,7 @@ MMC_API int32_t mmcc_batch_put(const char **keys, uint32_t keys_count, const mmc
495	MMC_VALIDATE_RETURN(keys != nullptr, "invalid param, keys is null", MMC_INVALID_PARAM);	496	MMC_VALIDATE_RETURN(keys != nullptr, "invalid param, keys is null", MMC_INVALID_PARAM);
496	MMC_VALIDATE_RETURN(keys_count != 0, "invalid param, keys_count: " << keys_count, MMC_INVALID_PARAM);	497	MMC_VALIDATE_RETURN(keys_count != 0, "invalid param, keys_count: " << keys_count, MMC_INVALID_PARAM);
497	MMC_VALIDATE_RETURN(bufs != nullptr, "invalid param, bufs is null", MMC_INVALID_PARAM);	498	MMC_VALIDATE_RETURN(bufs != nullptr, "invalid param, bufs is null", MMC_INVALID_PARAM);
		499	+ MMC_VALIDATE_RETURN(results != nullptr, "invalid param, results is null", MMC_INVALID_PARAM);
498	MMC_VALIDATE_RETURN(MmcClientDefault::GetInstance() != nullptr, "client is not initialize", MMC_CLIENT_NOT_INIT);	500	MMC_VALIDATE_RETURN(MmcClientDefault::GetInstance() != nullptr, "client is not initialize", MMC_CLIENT_NOT_INIT);
499		501
500	std::vector<std::string> keys_vector;	502	std::vector<std::string> keys_vector;

Msrc/memcache/csrc/mmcache_store.cpp+22-2

	@@ -914,6 +914,10 @@ std::vector<mmc_buffer> MmcacheStore::GetBatch(const std::vector<std::string> &k
914		914
915	// 1. Query KeyInfo for all keys	915	// 1. Query KeyInfo for all keys
916	auto keyInfos = BatchGetKeyInfo(keys);	916	auto keyInfos = BatchGetKeyInfo(keys);
		917	+ if (keyInfos.size() != count) {
		918	+ MMC_LOG_ERROR("BatchGetKeyInfo returned " << keyInfos.size() << " results, expected " << count);
		919	+ return {};
		920	+ }
917		921
918	// 2. alloc memory and assign value to the buffers	922	// 2. alloc memory and assign value to the buffers
919	for (size_t i = 0; i < count; ++i) {	923	for (size_t i = 0; i < count; ++i) {
	@@ -929,7 +933,7 @@ std::vector<mmc_buffer> MmcacheStore::GetBatch(const std::vector<std::string> &k
929	buffers[j].len = 0;	933	buffers[j].len = 0;
930	}	934	}
931	MMC_LOG_ERROR("Failed to allocate dynamic memory for key: " << keys[i].c_str());	935	MMC_LOG_ERROR("Failed to allocate dynamic memory for key: " << keys[i].c_str());
932	- return buffers;	936	+ return {};
933	}	937	}
934	buffers[i] = {	938	buffers[i] = {
935	.addr = reinterpret_cast<uint64_t>(dataPtr),	939	.addr = reinterpret_cast<uint64_t>(dataPtr),
	@@ -942,7 +946,23 @@ std::vector<mmc_buffer> MmcacheStore::GetBatch(const std::vector<std::string> &k
942	TP_TRACE_BEGIN(TP_MMC_PY_BATCH_GET);	946	TP_TRACE_BEGIN(TP_MMC_PY_BATCH_GET);
943	auto ret = mmcc_batch_get(keyArray.data(), count, buffers.data(), 0, results.data());	947	auto ret = mmcc_batch_get(keyArray.data(), count, buffers.data(), 0, results.data());
944	TP_TRACE_END(TP_MMC_PY_BATCH_GET, ret);	948	TP_TRACE_END(TP_MMC_PY_BATCH_GET, ret);
945	- (void)ret;	949	+ if (ret != MMC_OK) {
		950	+ for (size_t i = 0; i < count; ++i) {
		951	+ auto tmpPtr = reinterpret_cast<char *>(buffers[i].addr);
		952	+ delete[] tmpPtr;
		953	+ buffers[i] = {0, 0, 0, 0};
		954	+ }
		955	+ MMC_LOG_ERROR("mmcc_batch_get failed, ret=" << ret);
		956	+ return {};
		957	+ }
		958	+ // 4. zero out buffers for individually failed keys
		959	+ for (size_t i = 0; i < count; ++i) {
		960	+ if (results[i] != MMC_OK) {
		961	+ auto tmpPtr = reinterpret_cast<char *>(buffers[i].addr);
		962	+ delete[] tmpPtr;
		963	+ buffers[i] = {0, 0, 0, 0};
		964	+ }
		965	+ }
946	return buffers;	966	return buffers;
947	}	967	}
948		968

Msrc/memcache/csrc/under_api/ubs_io/dl_ubsio_api.cpp+7-1

	@@ -20,7 +20,7 @@ namespace mmc {
20	bool DlUbsioApi::gLoaded = false;	20	bool DlUbsioApi::gLoaded = false;
21	std::mutex DlUbsioApi::gMutex;	21	std::mutex DlUbsioApi::gMutex;
22	void *DlUbsioApi::ubsioHandle = nullptr;	22	void *DlUbsioApi::ubsioHandle = nullptr;
23	-const std::string DlUbsioApi::gUbsioLibName = "libubsio_kvc.so";	23	+const std::string DlUbsioApi::gUbsioLibName = "libubsio_kvc.so.1";
24		24
25	ubsio_client_initFunc DlUbsioApi::pUbsioClientInit = nullptr;	25	ubsio_client_initFunc DlUbsioApi::pUbsioClientInit = nullptr;
26	ubsio_putFunc DlUbsioApi::pUbsioPut = nullptr;	26	ubsio_putFunc DlUbsioApi::pUbsioPut = nullptr;
	@@ -36,6 +36,7 @@ ubsio_batch_deleteFunc DlUbsioApi::pUbsioBatchDelete = nullptr;
36	ubsio_batch_get_lengthFunc DlUbsioApi::pUbsioBatchGetLength = nullptr;	36	ubsio_batch_get_lengthFunc DlUbsioApi::pUbsioBatchGetLength = nullptr;
37	ubsio_batch_free_addressFunc DlUbsioApi::pUbsioBatchFreeAddress = nullptr;	37	ubsio_batch_free_addressFunc DlUbsioApi::pUbsioBatchFreeAddress = nullptr;
38	ubsio_register_meta_event_callbackFunc DlUbsioApi::pUbsioRegisterMetaEventCallback = nullptr;	38	ubsio_register_meta_event_callbackFunc DlUbsioApi::pUbsioRegisterMetaEventCallback = nullptr;
		39	+ubsio_kv_cache_exitFunc DlUbsioApi::pUbsioKvCacheExit = nullptr;
39	ubsio_get_resource_infoFunc DlUbsioApi::pUbsioGetResourceInfo = nullptr;	40	ubsio_get_resource_infoFunc DlUbsioApi::pUbsioGetResourceInfo = nullptr;
40		41
41	Result DlUbsioApi::UbsioClientInit(int32_t deviceId, const std::string &confPath)	42	Result DlUbsioApi::UbsioClientInit(int32_t deviceId, const std::string &confPath)
	@@ -83,6 +84,7 @@ Result DlUbsioApi::LoadLibrary()
83	DL_LOAD_SYM(pUbsioBatchFreeAddress, ubsio_batch_free_addressFunc, ubsioHandle, "UbsioKvCacheBatchFree");	84	DL_LOAD_SYM(pUbsioBatchFreeAddress, ubsio_batch_free_addressFunc, ubsioHandle, "UbsioKvCacheBatchFree");
84	DL_LOAD_SYM(pUbsioRegisterMetaEventCallback, ubsio_register_meta_event_callbackFunc, ubsioHandle,	85	DL_LOAD_SYM(pUbsioRegisterMetaEventCallback, ubsio_register_meta_event_callbackFunc, ubsioHandle,
85	"UbsioKvCacheRegisterMetaEventCallback");	86	"UbsioKvCacheRegisterMetaEventCallback");
		87	+ DL_LOAD_SYM(pUbsioKvCacheExit, ubsio_kv_cache_exitFunc, ubsioHandle, "UbsioKvCacheExit");
86	DlLoadSymOptional(pUbsioGetResourceInfo, ubsioHandle, "UbsioGetResourceInfo");	88	DlLoadSymOptional(pUbsioGetResourceInfo, ubsioHandle, "UbsioGetResourceInfo");
87		89
88	gLoaded = true;	90	gLoaded = true;
	@@ -112,6 +114,10 @@ void DlUbsioApi::CleanupLibrary()
112	pUbsioRegisterMetaEventCallback = nullptr;	114	pUbsioRegisterMetaEventCallback = nullptr;
113	pUbsioGetResourceInfo = nullptr;	115	pUbsioGetResourceInfo = nullptr;
114		116
		117	+ if (pUbsioKvCacheExit != nullptr) {
		118	+ pUbsioKvCacheExit();
		119	+ pUbsioKvCacheExit = nullptr;
		120	+ }
115	if (ubsioHandle != nullptr) {	121	if (ubsioHandle != nullptr) {
116	dlclose(ubsioHandle);	122	dlclose(ubsioHandle);
117	ubsioHandle = nullptr;	123	ubsioHandle = nullptr;

Msrc/memcache/csrc/under_api/ubs_io/dl_ubsio_api.h+2-0

	@@ -55,6 +55,7 @@ typedef struct {
55	typedef void (UbsioMetaEventCallbackC)(void context, const UbsioMetaEventC *events, uint32_t count);	55	typedef void (UbsioMetaEventCallbackC)(void context, const UbsioMetaEventC *events, uint32_t count);
56		56
57	using ubsio_register_meta_event_callbackFunc = int32_t ()(UbsioMetaEventCallbackC callback, void context);	57	using ubsio_register_meta_event_callbackFunc = int32_t ()(UbsioMetaEventCallbackC callback, void context);
		58	+using ubsio_kv_cache_exitFunc = void (*)();
58		59
59	class DlUbsioApi {	60	class DlUbsioApi {
60	public:	61	public:
	@@ -200,6 +201,7 @@ private:
200	static ubsio_batch_get_lengthFunc pUbsioBatchGetLength;	201	static ubsio_batch_get_lengthFunc pUbsioBatchGetLength;
201	static ubsio_batch_free_addressFunc pUbsioBatchFreeAddress;	202	static ubsio_batch_free_addressFunc pUbsioBatchFreeAddress;
202	static ubsio_register_meta_event_callbackFunc pUbsioRegisterMetaEventCallback;	203	static ubsio_register_meta_event_callbackFunc pUbsioRegisterMetaEventCallback;
		204	+ static ubsio_kv_cache_exitFunc pUbsioKvCacheExit;
203	static ubsio_get_resource_infoFunc pUbsioGetResourceInfo;	205	static ubsio_get_resource_infoFunc pUbsioGetResourceInfo;
204	};	206	};
205	} // namespace mmc	207	} // namespace mmc

Msrc/memcache/include/mmc_def.h+1-0

	@@ -76,6 +76,7 @@ typedef struct {
76	mmc_kv_events_config_t kvEvents;	76	mmc_kv_events_config_t kvEvents;
77	uint16_t rewarmDramWatermark;	77	uint16_t rewarmDramWatermark;
78	bool prefetchEnabled;	78	bool prefetchEnabled;
		79	+ uint64_t pendingWaitTimeoutMs;
79	} mmc_meta_service_config_t;	80	} mmc_meta_service_config_t;
80		81
81	typedef struct {	82	typedef struct {

Msrc/memcache/python/memcache_hybrid/__init__.py+1-1

	@@ -45,7 +45,7 @@ lib_dir = os.path.join(current_dir, "lib")
45	# Preload with absolute paths so that subsequent dlopen can resolve without LD_LIBRARY_PATH.	45	# Preload with absolute paths so that subsequent dlopen can resolve without LD_LIBRARY_PATH.
46	_preload_list = [	46	_preload_list = [
47	"libmf_memcache.so",	47	"libmf_memcache.so",
48	- "libubsio_kvc.so",	48	+ "libubsio_kvc.so.1",
49	]	49	]
50	for lib_name in _preload_list:	50	for lib_name in _preload_list:
51	so_path = os.path.join(lib_dir, lib_name)	51	so_path = os.path.join(lib_dir, lib_name)

Mtest/ut/mock/ubsio/CMakeLists.txt+5-1

	@@ -10,4 +10,8 @@
10		10
11	add_library(mock_ubsio SHARED mock_ubsio_api.cpp)	11	add_library(mock_ubsio SHARED mock_ubsio_api.cpp)
12	target_include_directories(mock_ubsio PRIVATE ${CMAKE_SOURCE_DIR}/src/memcache/csrc/under_api/ubs_io)	12	target_include_directories(mock_ubsio PRIVATE ${CMAKE_SOURCE_DIR}/src/memcache/csrc/under_api/ubs_io)
13	-set_target_properties(mock_ubsio PROPERTIES OUTPUT_NAME "ubsio_kvc")	13	+set_target_properties(mock_ubsio PROPERTIES
		14	+ OUTPUT_NAME "ubsio_kvc"
		15	+ VERSION 1.0.0
		16	+ SOVERSION 1
		17	+)

Mtest/ut/mock/ubsio/mock_ubsio_api.cpp+6-0

	@@ -301,6 +301,12 @@ extern "C" int32_t UbsioKvCacheBatchFree(void **bufs, uint32_t keys_count)
301	return 0;	301	return 0;
302	}	302	}
303		303
		304	+// 退出清理函数
		305	+extern "C" int32_t UbsioKvCacheExit()
		306	+{
		307	+ return 0;
		308	+}
		309	+
304	// 批量直接读取函数（带HBM）	310	// 批量直接读取函数（带HBM）
305	extern "C" int32_t UbsioKvCacheBatchGetDirect(const char keys, uint32_t keys_count, void bufs, size_t *lengths,	311	extern "C" int32_t UbsioKvCacheBatchGetDirect(const char keys, uint32_t keys_count, void bufs, size_t *lengths,
306	uint32_t lengths_rows, uint32_t lengths_cols, int *results,	312	uint32_t lengths_rows, uint32_t lengths_cols, int *results,

	@@ -218,6 +218,8 @@ public:
218	AddIntConf(OCK_MMC_REWARM_DRAM_WATERMARK,	218	AddIntConf(OCK_MMC_REWARM_DRAM_WATERMARK,
219	VIntRange::Create(OCK_MMC_REWARM_DRAM_WATERMARK.first, MIN_PERCENT, MAX_PERCENT), 0);	219	VIntRange::Create(OCK_MMC_REWARM_DRAM_WATERMARK.first, MIN_PERCENT, MAX_PERCENT), 0);
220	AddBoolConf(OCK_MMC_PREFETCH_ENABLED, VStrEnum::Create(OCK_MMC_PREFETCH_ENABLED.first, BOOL_ENUM_STR), 0);	220	AddBoolConf(OCK_MMC_PREFETCH_ENABLED, VStrEnum::Create(OCK_MMC_PREFETCH_ENABLED.first, BOOL_ENUM_STR), 0);
		221	+ AddIntConf(OCK_MMC_PENDING_WAIT_TIMEOUT_MS, VIntRange::Create(OCK_MMC_PENDING_WAIT_TIMEOUT_MS.first, 1, 60000U),
		222	+ 0);
221	AddIntConf(OCK_MMC_META_LEASE_TTL_MS,	223	AddIntConf(OCK_MMC_META_LEASE_TTL_MS,
222	VIntRange::Create(OCK_MMC_META_LEASE_TTL_MS.first, MIN_LEASE_TTL_MS, MAX_LEASE_TTL_MS), 0);	224	VIntRange::Create(OCK_MMC_META_LEASE_TTL_MS.first, MIN_LEASE_TTL_MS, MAX_LEASE_TTL_MS), 0);
223		225
	@@ -287,6 +289,7 @@ public:
287	config.evictThresholdLow = GetInt(ConfConstant::OKC_MMC_EVICT_THRESHOLD_LOW);	289	config.evictThresholdLow = GetInt(ConfConstant::OKC_MMC_EVICT_THRESHOLD_LOW);
288	config.rewarmDramWatermark = GetInt(ConfConstant::OCK_MMC_REWARM_DRAM_WATERMARK);	290	config.rewarmDramWatermark = GetInt(ConfConstant::OCK_MMC_REWARM_DRAM_WATERMARK);
289	config.prefetchEnabled = GetBool(ConfConstant::OCK_MMC_PREFETCH_ENABLED);	291	config.prefetchEnabled = GetBool(ConfConstant::OCK_MMC_PREFETCH_ENABLED);
		292	+ config.pendingWaitTimeoutMs = static_cast<uint64_t>(GetInt(ConfConstant::OCK_MMC_PENDING_WAIT_TIMEOUT_MS));
290	config.leaseTtlMs = static_cast<uint64_t>(GetInt(ConfConstant::OCK_MMC_META_LEASE_TTL_MS));	293	config.leaseTtlMs = static_cast<uint64_t>(GetInt(ConfConstant::OCK_MMC_META_LEASE_TTL_MS));
291	config.logRotationFileSize = GetInt(ConfConstant::OCK_MMC_LOG_ROTATION_FILE_SIZE) * MB_NUM;	294	config.logRotationFileSize = GetInt(ConfConstant::OCK_MMC_LOG_ROTATION_FILE_SIZE) * MB_NUM;
292	config.logRotationFileCount = GetInt(ConfConstant::OCK_MMC_LOG_ROTATION_FILE_COUNT);	295	config.logRotationFileCount = GetInt(ConfConstant::OCK_MMC_LOG_ROTATION_FILE_COUNT);