[feature]mooncake standalone部署模式适配-MindIE-Motor-AtomGit

M.agent/skills/motor-dev/references/nodeman.md+1-0

	@@ -409,6 +409,7 @@ barriers heartbeat until the container checkpoint is done. During container snap
409	\| `motor/common/utils/ai_cube.py` \| `npu-smi info watch -s u` AI Cube usage sampling (shared, no engine_server dependency) \|	409	\| `motor/common/utils/ai_cube.py` \| `npu-smi info watch -s u` AI Cube usage sampling (shared, no engine_server dependency) \|
410	\| `motor/node_manager/core/services/registry.py` \| Service registration and backend discovery \|	410	\| `motor/node_manager/core/services/registry.py` \| Service registration and backend discovery \|
411	\| `motor/node_manager/core/services/memcache/` \| Optional KV-store service implementation \|	411	\| `motor/node_manager/core/services/memcache/` \| Optional KV-store service implementation \|
		412	+\| `motor/node_manager/core/services/mooncake/` \| Mooncake standalone store service (`store_mode="standalone"`: prepare phase writes the store config; the official `mooncake_store_service` subprocess is launched after engines and restarted in place on death) \|
412	\| `motor/node_manager/core/heartbeat_manager.py` \| Native state polling, status mapping, heartbeat and suicide threshold \|	413	\| `motor/node_manager/core/heartbeat_manager.py` \| Native state polling, status mapping, heartbeat and suicide threshold \|
413	\| `motor/node_manager/core/register_manager.py` \| Controller registration, StartCmdMsg validation, ranktable and snapshot metadata/restore helpers \|	414	\| `motor/node_manager/core/register_manager.py` \| Controller registration, StartCmdMsg validation, ranktable and snapshot metadata/restore helpers \|
414	\| `motor/node_manager/api_client/controller_api_client.py` \| Controller register, reregister and heartbeat HTTP client \|	415	\| `motor/node_manager/api_client/controller_api_client.py` \| Controller register, reregister and heartbeat HTTP client \|

Mdocs/zh/user_guide/configuration/config_reference.md+4-0

	@@ -614,6 +614,8 @@ motor_engine_union_config字段用于PD混部场景，配置同一类 union
614	\| port_allocator_config.probe_timeout_seconds \|float\|探测超时时间，默认值：0.5。\|	614	\| port_allocator_config.probe_timeout_seconds \|float\|探测超时时间，默认值：0.5。\|
615	\| port_allocator_config.remote_check_timeout_seconds \|float\|远程检测超时时间，默认值：1.0。\|	615	\| port_allocator_config.remote_check_timeout_seconds \|float\|远程检测超时时间，默认值：1.0。\|
616	\| port_allocator_config.bind_host \|string\|绑定主机地址，默认值：0.0.0.0。\|	616	\| port_allocator_config.bind_host \|string\|绑定主机地址，默认值：0.0.0.0。\|
		617	+\| kv_cache_store_config字段 \|-\|-\|
		618	+\| kv_cache_store_config \| object \| KV 池化配置（`enable`/`backend`/`store_mode`/`global_segment_size`/`local_buffer_size`/`store_http_port` 等），未配置则不启用池化。完整字段与默认值见 [KV 池化 README — kv_cache_store_config](../features/kv_cache_store/README.md#kv_cache_store_config全局配置)；Mooncake `standalone` 部署模式说明见 [Mooncake 后端文档](../features/kv_cache_store/backend/mooncake.md#standalone-模式独立-store-进程)。 \|
617		619
618	---	620	---
619		621
	@@ -813,6 +815,8 @@ motor_engine_prefill_config和motor_engine_decode_config字段用于**PD分离
813	\| port_allocator_config.probe_timeout_seconds \|float\|探测超时时间，默认值：0.5。\|	815	\| port_allocator_config.probe_timeout_seconds \|float\|探测超时时间，默认值：0.5。\|
814	\| port_allocator_config.remote_check_timeout_seconds \|float\|远程检测超时时间，默认值：1.0。\|	816	\| port_allocator_config.remote_check_timeout_seconds \|float\|远程检测超时时间，默认值：1.0。\|
815	\| port_allocator_config.bind_host \|string\|绑定主机地址，默认值：0.0.0.0。\|	817	\| port_allocator_config.bind_host \|string\|绑定主机地址，默认值：0.0.0.0。\|
		818	+\| kv_cache_store_config字段 \|-\|-\|
		819	+\| kv_cache_store_config \| object \| KV 池化配置（`enable`/`backend`/`store_mode`/`global_segment_size`/`local_buffer_size`/`store_http_port` 等），未配置则不启用池化。完整字段与默认值见 [KV 池化 README — kv_cache_store_config](../features/kv_cache_store/README.md#kv_cache_store_config全局配置)；Mooncake `standalone` 部署模式说明见 [Mooncake 后端文档](../features/kv_cache_store/backend/mooncake.md#standalone-模式独立-store-进程)。 \|
816		820
817	PD模式下P与D各自独立配置"health_check_config"，未配置时使用代码默认值。	821	PD模式下P与D各自独立配置"health_check_config"，未配置时使用代码默认值。
818		822

Mdocs/zh/user_guide/features/kv_cache_store/README.md+5-2

	@@ -140,8 +140,11 @@ KV池化主要通过 `user_config.json` 配置；使用 UCM 功能时还需要
140	\| `global_segment_size` \| string \| `1GB` \| 全局共享显存段大小 \|	140	\| `global_segment_size` \| string \| `1GB` \| 全局共享显存段大小 \|
141	\| `port` \| int（可选） \| `50088` \| KV Pool 服务端口；未配置时 deploy.py 将按默认值补齐 \|	141	\| `port` \| int（可选） \| `50088` \| KV Pool 服务端口；未配置时 deploy.py 将按默认值补齐 \|
142	\| `default_kv_lease_ttl` \| int（可选） \| `11000` \| KV 对象默认租约 TTL（毫秒）；配置值需大于 `env.json` 中 vllm 实例的 `ASCEND_CONNECT_TIMEOUT` 和 `ASCEND_TRANSFER_TIMEOUT` \|	142	\| `default_kv_lease_ttl` \| int（可选） \| `11000` \| KV 对象默认租约 TTL（毫秒）；配置值需大于 `env.json` 中 vllm 实例的 `ASCEND_CONNECT_TIMEOUT` 和 `ASCEND_TRANSFER_TIMEOUT` \|
143	-\| `eviction_high_watermark_ratio` \| float \| 0.9 \| 池化空间高水位驱逐线，传递给 `mooncake_master` 进程 \|	143	+\| `eviction_high_watermark_ratio` \| float \| 无（必填） \| 池化空间高水位驱逐线，传递给 `mooncake_master` 进程；deploy.py 对 mooncake 后端强制校验，缺失报错，建议值 0.9 \|
144	-\| `eviction_ratio` \| float \| 0.1 \| 单次驱逐比例，传递给 `mooncake_master` 进程 \|	144	+\| `eviction_ratio` \| float \| 无（必填） \| 单次驱逐比例，传递给 `mooncake_master` 进程；deploy.py 对 mooncake 后端强制校验，缺失报错，建议值 0.1 \|
		145	+\| `store_mode` \| string（可选） \| `embedded` \| store 部署模式：`embedded`（引擎进程贡献内存，旧行为）或 `standalone`（每个引擎 Pod 由 NodeManager 拉起独立的 `mooncake_store_service` 进程贡献内存，引擎故障不清池，store 故障原地重拉）。详见 [Mooncake 后端文档](backend/mooncake.md) \|
		liu8月25日 `store_mode` / `local_buffer_size` / `store_http_port` 已在本表和 `backend/mooncake.md` 写清，但 `docs/zh/user_guide/configuration/config_reference.md` 未收录这些字段。若 `config_reference` 仍作为配置总表，建议在 `kv_cache_store_config` 相关处补一行（或链到本表），避免只查总表的用户漏配 standalone。 zhinengwu 8月25日评论： 8月25日评论：
		146	+\| `local_buffer_size` \| string（可选） \| `1GB` \| standalone 模式下引擎侧传输 staging buffer \|
		147	+\| `store_http_port` \| int（可选） \| `0` \| standalone 模式下 store 进程的 REST 端口；默认 `0` 由内核分配临时端口（该接口无消费者，避免 hostNetwork 下同机端口冲突） \|
145		148
146	MemCache 专属参数	149	MemCache 专属参数
147		150

Mdocs/zh/user_guide/features/kv_cache_store/backend/mooncake.md+133-17

	@@ -2,44 +2,160 @@
2		2
3	Mooncake 池化后端，由 vllm-ascend 天然集成，无需额外安装任何组件。	3	Mooncake 池化后端，由 vllm-ascend 天然集成，无需额外安装任何组件。
4		4
5	-## 配置	5	+## 配置（embedded 模式）
6		6
7	-`AscendStoreConnector` 中配置 `"backend": "mooncake"`：	7	+Mooncake 池化有两种部署方式（`store_mode` 取值），区别在池化内存由谁贡献：
8		8
9	-```json	9	+- embedded 模式（默认，`store_mode` 为空或 `"embedded"`）：引擎进程自身贡献 `global_segment_size` 的池化内存，配置最简，适合小规模验证；引擎进程挂掉则其贡献的池化内存随之失效。本节介绍该模式，standalone 模式见[下节](#standalone-模式独立-store-进程)。
10	-"backend": "mooncake"	10	+- standalone 模式（`store_mode="standalone"`）：由独立 `mooncake_store_service` 进程贡献池化内存，与引擎生命周期解耦，适合生产部署。
11	-```
12		11
13	`kv_cache_store_config` 中配置 `"backend": "mooncake"`。可选配置 `target_job_id` 复用其他推理服务的 kv_store（值为目标服务的 `job_id`），行为说明见 [KV 池化 README — 多套服务共享 kv_store](../README.md#多套服务共享-kv_store)。	12	`kv_cache_store_config` 中配置 `"backend": "mooncake"`。可选配置 `target_job_id` 复用其他推理服务的 kv_store（值为目标服务的 `job_id`），行为说明见 [KV 池化 README — 多套服务共享 kv_store](../README.md#多套服务共享-kv_store)。
14		13
15	```json	14	```json
16	"kv_cache_store_config": {	15	"kv_cache_store_config": {
		16	+ "enable": true,
17	"backend": "mooncake",	17	"backend": "mooncake",
18	- "metadata_server": "P2PHANDSHAKE",	18	+ "global_segment_size": "2GB",
19	- "protocol": "ascend",
20	- "device_name": "",
21	- "global_segment_size": "1GB",
22	"eviction_high_watermark_ratio": 0.9,	19	"eviction_high_watermark_ratio": 0.9,
23	- "eviction_ratio": 0.1,	20	+ "eviction_ratio": 0.1
24	- "target_job_id": "service-a"
25	}	21	}
26	```	22	```
27		23
28	-`eviction_high_watermark_ratio` 和 `eviction_ratio` 为 Mooncake 专属参数，会传递给 `mooncake_master` 进程。	24	+`eviction_high_watermark_ratio` 和 `eviction_ratio` 为 Mooncake 专属参数，会传递给 `mooncake_master` 进程；deploy.py 对 mooncake 后端强制校验这两项，缺失会直接报错，必须显式配置（建议值 0.9 / 0.1）。
		25	+
		26	+该模式下需显式配置 `enable`（默认 `false`）、`backend`（默认 `memcache`）、`global_segment_size`（无默认）与上述两个驱逐参数，其余字段均有默认值。
		27	+
		28	+## standalone 模式（独立 store 进程）
		29	+
		30	+默认（embedded）模式下，`global_segment_size` 由每个引擎进程自己贡献，引擎进程挂掉则其贡献的池化内存随之失效。standalone 模式把内存贡献者从引擎进程中剥离：
		31	+
		32	+- 每个 PD 实例 Pod 内由 NodeManager 拉起一个独立的 `mooncake_store_service` 进程（Mooncake 官方 store 入口），负责向池里贡献 `global_segment_size` 内存；
		33	+- 引擎进程 `global_segment_size=0`，仅作为请求方（配置由部署脚本自动生成，无需手工改）；
		34	+- `mooncake_master` 仍然运行在独立的 kv-store Pod 中。
		35	+
		36	+收益：store 进程独立申请大内存（先于引擎启动，避免与引擎权重/KV 内存竞争），且引擎故障/重建不影响池内已有数据；store 进程故障由 NodeManager 原地重拉（受 `MOTOR_RESTART_LOCAL_SERVICE` 控制，默认开启），重新注册回 master。
		37	+
		38	+### 配置
		39	+
		40	+以下为 P/D 分离 + standalone 部署在 `user_config.json` 中的关键配置：
		41	+
		42	+```json
		43	+"kv_cache_store_config": {
		44	+ "enable": true,
		45	+ "backend": "mooncake",
		46	+ "store_mode": "standalone",
		47	+ "global_segment_size": "200GB",
		48	+ "eviction_high_watermark_ratio": 0.9,
		49	+ "eviction_ratio": 0.1
		50	+}
		51	+
		52	+"motor_engine_prefill_config": {
		53	+ "engine_type": "vllm",
		54	+ "engine_config": {
		55	+ // 模型、并行度等常规配置省略
		56	+ "kv_transfer_config": {
		57	+ "kv_connector": "MultiConnector",
		58	+ "kv_role": "kv_producer",
		59	+ "kv_port": "30001",
		60	+ "engine_id": "0",
		61	+ "kv_connector_extra_config": {
		62	+ "connectors": [
		63	+ {
		64	+ "kv_connector": "MooncakeConnectorV1",
		65	+ "kv_role": "kv_producer",
		66	+ "kv_port": "30001",
		67	+ "kv_connector_extra_config": {
		68	+ "prefill": {"dp_size": 1, "tp_size": 2, "pp_size": 1},
		69	+ "decode": {"dp_size": 1, "tp_size": 2, "pp_size": 1}
		70	+ }
		71	+ },
		72	+ {
		73	+ "kv_connector": "AscendStoreConnector",
		74	+ "kv_role": "kv_producer",
		75	+ "kv_connector_extra_config": {
		76	+ "backend": "mooncake"
		77	+ }
		78	+ }
		79	+ ]
		80	+ }
		81	+ }
		82	+ }
		83	+}
		84	+
		85	+"motor_engine_decode_config": {
		86	+ "engine_type": "vllm",
		87	+ "engine_config": {
		88	+ // 结构同 prefill，仅 kv_role 相反：
		89	+ // "kv_role": "kv_consumer"
		90	+ // MooncakeConnectorV1 与 AscendStoreConnector 的 kv_role 均为 "kv_consumer"
		91	+ }
		92	+}
		93	+```
		94	+
		95	+### 字段说明
		96	+
		97	+`kv_cache_store_config`：
		98	+
		99	+\| 字段 \| 默认值 \| 说明 \|
		100	+\|------\|--------\|------\|
		101	+\| `enable` \| `false` \| 池化总开关，需配 `true` \|
		102	+\| `backend` \| `memcache` \| 需配 `"mooncake"` \|
		103	+\| `store_mode` \| `embedded` \| standalone 模式需配 `"standalone"` \|
		104	+\| `global_segment_size` \| 无 \| standalone 下为 store 进程贡献的池化内存（如 `"200GB"`） \|
		105	+\| `eviction_high_watermark_ratio` / `eviction_ratio` \| 无（必填） \| 驱逐水位与单次驱逐比例，传递给 `mooncake_master`；deploy.py 强制校验，缺失报错（建议 0.9 / 0.1） \|
		106	+
		107	+`engine_config.kv_transfer_config`：
		108	+
		109	+\| 字段 \| 默认值 \| 说明 \|
		110	+\|------\|--------\|------\|
		111	+\| `kv_connector` \| — \| 需配 `"MultiConnector"` \|
		112	+\| `kv_role` \| — \| 需配：prefill = `kv_producer`，decode = `kv_consumer` \|
		113	+\| `engine_id` \| — \| 池化域标识，需配且 P/D 保持一致 \|
		114	+\| `connectors[].MooncakeConnectorV1` \| — \| P2P 直传（prefill → decode 直接传 KV，不落池）；`prefill`/`decode` 拓扑需与部署一致 \|
		115	+\| `connectors[].AscendStoreConnector` \| — \| 存池（prefill 入池 / decode 出池）；`backend` 需配 `"mooncake"` \|
		116	+\| `kv_port` \| — \| 建议显式指定（如 `"30001"`），避免多服务端口冲突 \|
		117	+
		118	+> store 进程的配置文件由 NodeManager 自动生成在引擎配置同目录（`mooncake_store_config.json`），`local_hostname` 取 `POD_IP`，master 地址取 `KVS_MASTER_SERVICE`。
		119	+>
		120	+> store 进程的通信环境同样由 NodeManager 自动配置，无需手工干预：store 独占 HIXL `comm_resource_config.listen_port=26666`（写入 ranktable 的 `device_port`，与同卡引擎 worker 区分，避免 HCCL `EI0014`），HCCL socket 端口段相对引擎偏移（A2：`HCCL_NPU_SOCKET_PORT_RANGE=16700-16800`；A5：host socket 段 +2000），避免与引擎的 16666/RA socket 冲突（`EI0020`）。
29		121
30	## 环境变量配置说明	122	## 环境变量配置说明
31		123
32	-按硬件在 `env.json` 的 `motor_engine_prefill_env`、`motor_engine_decode_env` 中配置下列环境变量，Prefill 与 Decode 保持一致。	124	+在 `env.json` 的 `motor_engine_prefill_env`、`motor_engine_decode_env` 中配置下列环境变量，Prefill 与 Decode 保持一致。
		125	+
		126	+所有硬件均需配置：
		127	+
		128	+\| 环境变量（写入 `env.json`） \| 说明 \|
		129	+\|------\|------\|
		130	+\| `HCCL_INTRA_ROCE_ENABLE=1` \| 必须。HIXL 底层直连传输走 RoCE 协议，需显式使能才能建连，未配置会导致 KV 传输失败。注意该变量需配在 `motor_engine_prefill_env` / `motor_engine_decode_env` 中并随部署下发到引擎 Pod，仅在部署节点 shell 中 export 不生效 \|
		131	+\| `ASCEND_LOCAL_COMM_RES={"version":"1.3"}` \| 必须。使 ascend_transport 按 v1.3 格式生成本地通信资源，走 client-server 单边通信，ranktable 携带 `device_port`。所有硬件（A2/A5 等）均需配置；standalone 模式下缺失时，store 进程与引擎 worker 共用同一 NPU 会因合并 ranktable 出现重复 device_ip 报 `EI0014: IP is used repeatedly`，与 store 同卡的 worker（如 TP rank 0）block 入池失败 \|
		132	+
		133	+`env.json` 配置示例（所有硬件通用，Prefill 与 Decode 相同）：
		134	+
		135	+```json
		136	+"motor_engine_prefill_env": {
		137	+ "HCCL_INTRA_ROCE_ENABLE": "1",
		138	+ "ASCEND_LOCAL_COMM_RES": "{\"version\":\"1.3\"}"
		139	+}
		140	+```
		141	+
		142	+各硬件依赖与差异化配置（CANN 版本均需 >= 9.1.0）：
33		143
34	\| 硬件 \| 依赖 \| 环境变量（写入 `env.json`） \| 说明 \|	144	\| 硬件 \| 依赖 \| 环境变量（写入 `env.json`） \| 说明 \|
35	\|------\|------\|------------------------------\|------\|	145	\|------\|------\|------------------------------\|------\|
36	\| Ascend 950 系列产品 \| HDK >= 25.6 且 mooncake >= v0.3.11<br>CANN >= 9.1.0 \| UBOE：`ASCEND_GLOBAL_RESOURCE_CONFIG={"comm_resource_config.protocol_desc":["uboe:device"]}`<br>UB：`ASCEND_LOCAL_COMM_RES={"version":"1.3"}` \| 按实际使用的通信协议配置对应环境变量（UBOE / UB 二选一） \|	146	\| Ascend 950 系列产品 \| HDK >= 25.6 且 mooncake >= v0.3.11<br>CANN >= 9.1.0 \| UBOE：`ASCEND_GLOBAL_RESOURCE_CONFIG={"comm_resource_config.protocol_desc":["uboe:device"]}`<br>UB：`ASCEND_LOCAL_COMM_RES={"version":"1.3"}` \| 按实际使用的通信协议配置对应环境变量（UBOE / UB 二选一） \|
37	-\| Atlas 800I/T A3 超节点服务器 \| HDK >= 26.0<br>或 HDK >= 25.5 且 mooncake >= v0.3.11<br>CANN >= 9.0.0<br>灵衢算力网络 >= 1.5 \| `ASCEND_ENABLE_USE_FABRIC_MEM=1` \| 推荐。启用统一内存地址直传方案。若开启 SSD offload，相关内存大小需按 1GB 对齐，详见 vllm-ascend 文档 [Fabric memory size alignment](https://docs.vllm.ai/projects/ascend/en/latest/user_guide/feature_guide/kv_pool.html#fabric-memory-size-alignment-a3-ascend-enable-use-fabric-mem-1) \|	147	+\| Atlas 800I/T A3 超节点服务器 \| HDK >= 26.0<br>或 HDK >= 25.5 且 mooncake >= v0.3.11<br>CANN >= 9.1.0<br>灵衢算力网络 >= 1.5 \| `ASCEND_ENABLE_USE_FABRIC_MEM=1` \| 推荐。启用统一内存地址直传方案。若开启 SSD offload，相关内存大小需按 1GB 对齐，详见 vllm-ascend 文档 [Fabric memory size alignment](https://docs.vllm.ai/projects/ascend/en/latest/user_guide/feature_guide/kv_pool.html#fabric-memory-size-alignment-a3-ascend-enable-use-fabric-mem-1) \|
38	\| Atlas 800I/T A3 超节点服务器 \| 上述依赖不满足时 \| `ASCEND_BUFFER_POOL=4:8` \| 配置 NPU Device 上用于聚合与 KV 传输的 buffer 个数与大小（例如 `4:8` 表示 4 个 8MB buffer） \|	148	\| Atlas 800I/T A3 超节点服务器 \| 上述依赖不满足时 \| `ASCEND_BUFFER_POOL=4:8` \| 配置 NPU Device 上用于聚合与 KV 传输的 buffer 个数与大小（例如 `4:8` 表示 4 个 8MB buffer） \|
39	-\| Atlas 800I/T A2 推理服务器 \| 推荐 HDK >= 25.5 \| `HCCL_INTRA_ROCE_ENABLE=1` \| 800 I/T A2 系列直连传输方案所需 \|	149	+\| Atlas 800I/T A2 推理服务器 \| HDK >= 25.5<br>CANN >= 9.1.0 \| — \| 无需额外环境变量，通用必配项即可 \|
40		150
41	> 更多原理与排障请参考 [vllm-ascend KV Pool 文档](https://docs.vllm.ai/projects/ascend/zh-cn/main/user_guide/feature_guide/kv_pool.html)。	151	> 更多原理与排障请参考 [vllm-ascend KV Pool 文档](https://docs.vllm.ai/projects/ascend/zh-cn/main/user_guide/feature_guide/kv_pool.html)。
42		152
		153	+## 入池条件
		154	+
		155	+block 入池的前提是请求的 prompt 长度不小于 128 token：`AscendStoreConnector` 按 128 token 的 chunk 粒度做入池判定（`can_save`），只有 token 数达到一个完整 chunk 时才真正触发存池 put。短请求（prompt < 128 token）不会入池，也不会产生 put 流量，这是 vllm-ascend 的设计行为，不是故障。
		156	+
		157	+验证池化是否生效时请使用长 prompt（≥ 128 token，建议 500+ token）发起请求，并通过 master 侧观测 `PutStart`/`Keys` 等指标确认 block 确实入池。
		158	+
43	## 调优建议	159	## 调优建议
44		160
45	- `global_segment_size`：根据模型大小和并发量调整，过小会导致频繁驱逐；过大则浪费显存。建议设为模型 KV Cache 预估大小的 1.5~2 倍。	161	- `global_segment_size`：根据模型大小和并发量调整，过小会导致频繁驱逐；过大则浪费显存。建议设为模型 KV Cache 预估大小的 1.5~2 倍。
	@@ -81,8 +197,8 @@ Mooncake 池化后端，由 vllm-ascend 天然集成，**无需额外安装任
81	ip="${HOST_IP:-$POD_IP}"	197	ip="${HOST_IP:-$POD_IP}"
82		198
83	if [ -z "$if_name" ]; then	199	if [ -z "$if_name" ]; then
		200	+ # Skip auto-detection only; never unset — the user may have exported these explicitly.
84	echo "Warning: failed to detect default route interface from /proc/net/route, skip GLOO/TP/HCCL socket ifname env" >&2	201	echo "Warning: failed to detect default route interface from /proc/net/route, skip GLOO/TP/HCCL socket ifname env" >&2
85	- unset GLOO_SOCKET_IFNAME TP_SOCKET_IFNAME HCCL_SOCKET_IFNAME
86	else	202	else
87	export GLOO_SOCKET_IFNAME="$if_name"	203	export GLOO_SOCKET_IFNAME="$if_name"
88	export TP_SOCKET_IFNAME="$if_name"	204	export TP_SOCKET_IFNAME="$if_name"
	@@ -90,8 +206,8 @@ Mooncake 池化后端，由 vllm-ascend 天然集成，**无需额外安装任
90	fi	206	fi
91		207
92	if [ -z "$ip" ]; then	208	if [ -z "$ip" ]; then
		209	+ # Skip auto-detection only; never unset — the user may have exported it explicitly.
93	echo "Warning: HOST_IP and POD_IP are both empty, skip HCCL_IF_IP env" >&2	210	echo "Warning: HOST_IP and POD_IP are both empty, skip HCCL_IF_IP env" >&2
94	- unset HCCL_IF_IP
95	else	211	else
96	export HCCL_IF_IP="$ip"	212	export HCCL_IF_IP="$ip"
97	fi	213	fi

Mexamples/deployer/startup/common.sh+21-0

	@@ -291,8 +291,29 @@ is_a5_hardware() {
291	}	291	}
292		292
293	set_a5_engine_env() {	293	set_a5_engine_env() {
		294	+ local if_name ip
		295	+ if_name=$(awk '$2 == "00000000" {print $1; exit}' /proc/net/route)
		296	+ ip="${POD_IP:-$HOST_IP}" # POD_IP is the pod-net IP valid in the pod; HOST_IP (host IP) only works under hostNetwork
		297	+
		298	+ if [ -z "$if_name" ]; then
		299	+ # Skip auto-detection only; never unset — the user may have exported these explicitly.
		300	+ echo "Warning: failed to detect default route interface from /proc/net/route, skip GLOO/TP/HCCL socket ifname env" >&2
		301	+ else
		302	+ export GLOO_SOCKET_IFNAME="$if_name"
		303	+ export TP_SOCKET_IFNAME="$if_name"
		304	+ export HCCL_SOCKET_IFNAME="$if_name"
		305	+ fi
		306	+
		307	+ if [ -z "$ip" ]; then
		308	+ # Skip auto-detection only; never unset — the user may have exported it explicitly.
		309	+ echo "Warning: HOST_IP and POD_IP are both empty, skip HCCL_IF_IP env" >&2
		310	+ else
		311	+ export HCCL_IF_IP="$ip"
		312	+ fi
		313	+
294	export PATH="$PATH:/usr/local/go/bin"	314	export PATH="$PATH:/usr/local/go/bin"
295	export LD_LIBRARY_PATH="/usr/local/lib:/usr/lib64:/lib64:${LD_LIBRARY_PATH:-}"	315	export LD_LIBRARY_PATH="/usr/local/lib:/usr/lib64:/lib64:${LD_LIBRARY_PATH:-}"
		316	+ export ASCEND_LOCAL_COMM_RES_PATH="${ASCEND_LOCAL_COMM_RES_PATH:-/etc/hixlep}"
296	}	317	}
297		318
298	gen_ranktable_config() {	319	gen_ranktable_config() {

Mexamples/deployer/startup/roles/kv_store_backends/mooncake/mooncake_config.py+7-0

	@@ -94,6 +94,13 @@ def generate_kv_cache_store_config(output_path: str, user_config_path: str) -> b
94		94
95	out_cfg: dict[str, Any] = dict(kv_cfg)	95	out_cfg: dict[str, Any] = dict(kv_cfg)
96		96
		97	+ out_cfg.setdefault("metadata_server", "P2PHANDSHAKE")
		98	+
		99	+ # Standalone: the store process contributes memory; engines contribute nothing but keep a staging buffer.
		100	+ if kv_cfg.get("store_mode") == "standalone":
		101	+ out_cfg["global_segment_size"] = 0
		102	+ out_cfg["local_buffer_size"] = kv_cfg.get("local_buffer_size") or "1GB"
		103	+
97	kvp_master_service = os.getenv("KVS_MASTER_SERVICE", "")	104	kvp_master_service = os.getenv("KVS_MASTER_SERVICE", "")
98	if not kvp_master_service:	105	if not kvp_master_service:
99	logging.error("Env KVS_MASTER_SERVICE is required but not set, cannot generate kv_cache_store_config")	106	logging.error("Env KVS_MASTER_SERVICE is required but not set, cannot generate kv_cache_store_config")

Mmotor/common/utils/env.py+6-0

	@@ -108,6 +108,12 @@ class _Environment:
108	def mmc_local_config_path(self):	108	def mmc_local_config_path(self):
109	return os.getenv("MMC_LOCAL_CONFIG_PATH", "")	109	return os.getenv("MMC_LOCAL_CONFIG_PATH", "")
110		110
		111	+ # --- Mooncake store ---
		112	+
		113	+ @property
		114	+ def mooncake_config_path(self):
		115	+ return os.getenv("MOONCAKE_CONFIG_PATH", "")
		116	+
111	@property	117	@property
112	def mmc_local_service_mode(self):	118	def mmc_local_service_mode(self):
113	return os.getenv("MMC_LOCAL_SERVICE_MODE", "")	119	return os.getenv("MMC_LOCAL_SERVICE_MODE", "")

Mmotor/config/node_manager.py+40-0

	@@ -338,6 +338,24 @@ class KVCacheStoreConfig:
338	# "standalone" / "inprocess" — the only user-facing config key for memcache.	338	# "standalone" / "inprocess" — the only user-facing config key for memcache.
339	local_config_path: str = "/usr/local/Ascend/pyMotor/conf/mmc-local-inprocess.conf"	339	local_config_path: str = "/usr/local/Ascend/pyMotor/conf/mmc-local-inprocess.conf"
340		340
		341	+ # --- Mooncake ---
		342	+ store_mode: str = ""
		343	+ # "" / "embedded" (default): engines contribute memory themselves.
		344	+ # "standalone": the daemon runs a dedicated store process; engines are pure requesters.
		345	+ global_segment_size: str = ""
		346	+ # Pool memory contributed by this pod, e.g. "600GB"; consumed by the engine
		347	+ # connector in embedded mode, by the store process in standalone mode.
		348	+ local_buffer_size: str = ""
		349	+ # Engine-side staging buffer in standalone mode (default "1GB"); the store always uses 0.
		350	+ store_http_port: int = 0
		351	+ # REST port of mooncake_store_service (its --port arg). 0 = ephemeral port:
		352	+ # the API has no consumer in Motor deployments and only needs to bind
		353	+ # successfully, so let the kernel pick a free port to avoid conflicts
		354	+ # (hostNetwork pods on the same node share the host port space).
		355	+ metadata_server: str = "P2PHANDSHAKE"
		356	+ protocol: str = "ascend"
		357	+ device_name: str = ""
		358	+
341		359
342	@dataclass	360	@dataclass
343	class NodeManagerConfig:	361	class NodeManagerConfig:
	@@ -723,6 +741,26 @@ class NodeManagerConfig:
723	if config_path:	741	if config_path:
724	kcfg.local_config_path = config_path	742	kcfg.local_config_path = config_path
725		743
		744	+ # --- Mooncake ---
		745	+ if kcfg.backend == "mooncake":
		746	+ if "store_mode" in kv:
		747	+ kcfg.store_mode = kv["store_mode"]
		748	+ if kcfg.store_mode not in ("", "embedded", "standalone"):
		749	+ logger.warning(
		750	+ "kv_cache_store_config.store_mode=%r is invalid, falling back to 'embedded'",
		751	+ kcfg.store_mode,
		752	+ )
		753	+ kcfg.store_mode = "embedded"
		754	+ kcfg.global_segment_size = kv.get("global_segment_size", "") or kcfg.global_segment_size
		755	+ kcfg.local_buffer_size = kv.get("local_buffer_size", "") or kcfg.local_buffer_size
		756	+ kcfg.metadata_server = kv.get("metadata_server", "") or kcfg.metadata_server
		757	+ kcfg.protocol = kv.get("protocol", "") or kcfg.protocol
		758	+ if "device_name" in kv:
		759	+ kcfg.device_name = kv["device_name"]
		760	+ store_http_port = kv.get("store_http_port", 0)
		761	+ if store_http_port:
		762	+ kcfg.store_http_port = int(store_http_port)
		763	+
726	@classmethod	764	@classmethod
727	def _set_device_count_for_single_container(cls, config: "NodeManagerConfig"):	765	def _set_device_count_for_single_container(cls, config: "NodeManagerConfig"):
728	"""Set device count for single container mode using parallel_config.world_size"""	766	"""Set device count for single container mode using parallel_config.world_size"""
	@@ -913,6 +951,8 @@ class NodeManagerConfig:
913	f" ├─ Service: {self.kv_cache_store_config.service or '(env: KVS_MASTER_SERVICE)'}\n"	951	f" ├─ Service: {self.kv_cache_store_config.service or '(env: KVS_MASTER_SERVICE)'}\n"
914	f" ├─ Deploy Mode: {self.kv_cache_store_config.mode}\n"	952	f" ├─ Deploy Mode: {self.kv_cache_store_config.mode}\n"
915	f" ├─ Runtime Mode: {self.kv_cache_store_config.local_service_mode or '(default)'}\n"	953	f" ├─ Runtime Mode: {self.kv_cache_store_config.local_service_mode or '(default)'}\n"
		954	+ f" ├─ Store Mode: {self.kv_cache_store_config.store_mode or '(embedded)'}\n"
		955	+ f" ├─ Global Segment Size: {self.kv_cache_store_config.global_segment_size or '(default)'}\n"
916	f" ├─ Port: {self.kv_cache_store_config.port}\n"	956	f" ├─ Port: {self.kv_cache_store_config.port}\n"
917	f" └─ Local Config Path: {self.kv_cache_store_config.local_config_path}\n"	957	f" └─ Local Config Path: {self.kv_cache_store_config.local_config_path}\n"
918	f"{'=' * 80}"	958	f"{'=' * 80}"

Amotor/node_manager/core/services/mooncake/__init__.py+11-0

@@ -0,0 +1,11 @@
	1	+# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved.
	2	+# MindIE is licensed under Mulan PSL v2.
	3	+# You can use this software according to the terms and conditions of the Mulan PSL v2.
	4	+# You may obtain a copy of Mulan PSL v2 at:
	5	+# http://license.coscl.org.cn/MulanPSL2
	6	+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
	7	+# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
	8	+# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
	9	+# See the Mulan PSL v2 for more details.
	10	+
	11	+"""Mooncake KV-store backend services managed by the NodeManager daemon."""

Amotor/node_manager/core/services/mooncake/bootstrap/__init__.py+17-0

@@ -0,0 +1,17 @@
	1	+# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved.
	2	+# MindIE is licensed under Mulan PSL v2.
	3	+# You can use this software according to the terms and conditions of the Mulan PSL v2.
	4	+# You may obtain a copy of Mulan PSL v2 at:
	5	+# http://license.coscl.org.cn/MulanPSL2
	6	+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
	7	+# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
	8	+# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
	9	+# See the Mulan PSL v2 for more details.
	10	+
	11	+"""Store-process bootstraps run inside the store subprocess (``python -m``),
	12	+setting up what the store process needs before ``mooncake_store_service``
	13	+starts (an ACL context never survives the Popen boundary).
	14	+
	15	+- ``ascend_850`` — 850: ACL context only, comm-free like 800I (pending A5 re-validation).
	16	+- ``ascend_800I`` — 800I: ACL context only, deliberately no comm (EI0014).
	17	+"""

Amotor/node_manager/core/services/mooncake/bootstrap/ascend_800I.py+62-0

@@ -0,0 +1,62 @@
	1	+# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved.
	2	+# MindIE is licensed under Mulan PSL v2.
	3	+# You can use this software according to the terms and conditions of the Mulan PSL v2.
	4	+# You may obtain a copy of Mulan PSL v2 at:
	5	+# http://license.coscl.org.cn/MulanPSL2
	6	+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
	7	+# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
	8	+# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
	9	+# See the Mulan PSL v2 for more details.
	10	+
	11	+"""800I-A2 / 800I-A3 store bootstrap: ACL context only, deliberately no HCCL comm.
	12	+
	13	+A comm would make the store's adxl engine merge its root info into the P2P
	14	+handshake; on same-node PD the store shares devices with the vllm workers, the
	15	+merged rank table repeats a device IP and HCCL rejects it with EI0014. Comm-free
	16	+handshake is TP-agnostic. ACL context is still required (the store never calls
	17	+``aclrtSetDevice`` itself).
	18	+"""
	19	+
	20	+import logging
	21	+import runpy
	22	+import sys
	23	+
	24	+logger = logging.getLogger("motor.mooncake.store_bootstrap")
	25	+
	26	+
	27	+def _init_logging() -> None:
	28	+ """Ensure the store subprocess logs to stderr.
	29	+
	30	+ The store runs as a bare subprocess (see mooncake/lifecycle.py) without the
	31	+ motor logging setup, so configure a minimal stderr handler up front.
	32	+ """
	33	+ logging.basicConfig(
	34	+ stream=sys.stderr,
	35	+ level=logging.INFO,
	36	+ format="[mooncake store bootstrap] %(asctime)s %(levelname)s %(message)s",
	37	+ )
	38	+
	39	+
	40	+def _init_acl_context() -> None:
	41	+ """Create a device-0 ACL context for the store process (best effort)."""
	42	+ try:
	43	+ import acl
	44	+
	45	+ acl.init()
	46	+ acl.rt.set_device(0)
	47	+ except Exception as e: # pylint: disable=broad-except
	48	+ # Non-fatal by design: the store must still start and serve TCP-only clients.
	49	+ logger.warning("ACL context init failed (non-fatal): %s", e)
	50	+
	51	+
	52	+def main() -> None:
	53	+ """Bootstrap the device context, then run the official store service
	54	+ (``--config``/``--port`` in ``sys.argv`` are consumed by the store itself).
	55	+ """
	56	+ _init_logging()
	57	+ _init_acl_context()
	58	+ runpy.run_module("mooncake.mooncake_store_service", run_name="__main__")
	59	+
	60	+
	61	+if __name__ == "__main__":
	62	+ main()

Amotor/node_manager/core/services/mooncake/bootstrap/ascend_850.py+59-0

@@ -0,0 +1,59 @@
	1	+# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved.
	2	+# MindIE is licensed under Mulan PSL v2.
	3	+# You can use this software according to the terms and conditions of the Mulan PSL v2.
	4	+# You may obtain a copy of Mulan PSL v2 at:
	5	+# http://license.coscl.org.cn/MulanPSL2
	6	+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
	7	+# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
	8	+# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
	9	+# See the Mulan PSL v2 for more details.
	10	+
	11	+"""850 (350/850/950) store bootstrap: ACL context only, comm-free like 800I.
	12	+
	13	+The HCCL comm mirror was dropped pending A5 re-validation. The store never
	14	+calls ``aclrtSetDevice`` itself, so the device context is still required.
	15	+"""
	16	+
	17	+import logging
	18	+import runpy
	19	+import sys
	20	+
	21	+logger = logging.getLogger("motor.mooncake.store_bootstrap")
	22	+
	23	+
	24	+def _init_logging() -> None:
	25	+ """Ensure the store subprocess logs to stderr.
	26	+
	27	+ The store runs as a bare subprocess (see mooncake/lifecycle.py) without the
	28	+ motor logging setup, so configure a minimal stderr handler up front.
	29	+ """
	30	+ logging.basicConfig(
	31	+ stream=sys.stderr,
	32	+ level=logging.INFO,
	33	+ format="[mooncake store bootstrap] %(asctime)s %(levelname)s %(message)s",
	34	+ )
	35	+
	36	+
	37	+def _init_acl_context() -> None:
	38	+ """Create a device-0 ACL context for the store process (best effort)."""
	39	+ try:
	40	+ import acl
	41	+
	42	+ acl.init()
	43	+ acl.rt.set_device(0)
	44	+ except Exception as e: # pylint: disable=broad-except
	45	+ # Non-fatal by design: the store must still start and serve TCP-only clients.
	46	+ logger.warning("ACL context init failed (non-fatal): %s", e)
	47	+
	48	+
	49	+def main() -> None:
	50	+ """Bootstrap the device context, then run the official store service
	51	+ (``--config``/``--port`` in ``sys.argv`` are consumed by the store itself).
	52	+ """
	53	+ _init_logging()
	54	+ _init_acl_context()
	55	+ runpy.run_module("mooncake.mooncake_store_service", run_name="__main__")
	56	+
	57	+
	58	+if __name__ == "__main__":
	59	+ main()

Amotor/node_manager/core/services/mooncake/lifecycle.py+307-0

@@ -0,0 +1,307 @@
	1	+# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved.
	2	+# MindIE is licensed under Mulan PSL v2.
	3	+# You can use this software according to the terms and conditions of the Mulan PSL v2.
	4	+# You may obtain a copy of Mulan PSL v2 at:
	5	+# http://license.coscl.org.cn/MulanPSL2
	6	+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
	7	+# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
	8	+# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
	9	+# See the Mulan PSL v2 for more details.
	10	+
	11	+"""Mooncake standalone store lifecycle manager (mirrors memcache LocalService).
	12	+
	13	+Each engine pod runs the official ``mooncake_store_service`` contributing
	14	+``global_segment_size``; the process is launched after engines (``pull_kv_store``)
	15	+and restarted in place on death, so engines never wait on it.
	16	+"""
	17	+
	18	+import json
	19	+import os
	20	+import signal
	21	+import subprocess
	22	+import sys
	23	+import threading
	24	+import time
	25	+
	26	+from motor.common.logger import get_logger
	27	+from motor.common.utils.env import Env
	28	+from motor.common.utils.net import format_address
	29	+from motor.config.node_manager import HardwareType, KVCacheStoreConfig
	30	+from motor.node_manager.core.services.registry import SERVICE_KV_STORE, register_service
	31	+
	32	+logger = get_logger(__name__)
	33	+
	34	+_DEFAULT_CONF_DIR = "/usr/local/Ascend/Motor/conf"
	35	+_STORE_CONFIG_FILENAME = "mooncake_store_config.json"
	36	+_STARTUP_GRACE_SEC = 2.0 # catch immediate startup failures after spawn
	37	+
	38	+# Store-process bootstraps (run with ``python -m`` in the store subprocess).
	39	+_BOOTSTRAP_ASCEND = "motor.node_manager.core.services.mooncake.bootstrap.ascend_850"
	40	+_BOOTSTRAP_ASCEND_800I = "motor.node_manager.core.services.mooncake.bootstrap.ascend_800I"
	41	+
	42	+# HIXL listen port for the store, off the engine's per-device 16666 (verified on 800I, 2026-08-24).
	43	+_STORE_HIXL_LISTEN_PORT = "26666"
	44	+
	45	+
	46	+def _offset_port_range(port_range: str, offset: int) -> str:
	47	+ """Shift an ``"A-B"`` port range by ``offset``."""
	48	+ try:
	49	+ start_s, _, end_s = port_range.partition("-")
	50	+ return "%d-%d" % (int(start_s) + offset, int(end_s) + offset)
	51	+ except (ValueError, AttributeError):
	52	+ return "%d-%d" % (60000 + offset, 60050 + offset)
	53	+
	54	+
	55	+def _merge_hixl_listen_port(env: dict) -> None:
	56	+ """Set the store's HIXL listen port in ASCEND_GLOBAL_RESOURCE_CONFIG.
	57	+
	58	+ HIXL reads the flat dotted key ``comm_resource_config.listen_port`` (official AGRC form).
	59	+ Merge, never replace: a pre-set AGRC (e.g. protocol_desc for UBOE) must survive.
	60	+ """
	61	+ listen_port = env.get("HIXL_LISTEN_PORT", _STORE_HIXL_LISTEN_PORT)
	62	+ try:
	63	+ agrc = json.loads(env.get("ASCEND_GLOBAL_RESOURCE_CONFIG", "{}"))
	64	+ except json.JSONDecodeError:
	65	+ agrc = {}
	66	+ if not isinstance(agrc, dict):
	67	+ agrc = {}
	68	+ agrc["comm_resource_config.listen_port"] = listen_port
	69	+ env["ASCEND_GLOBAL_RESOURCE_CONFIG"] = json.dumps(agrc)
	70	+
	71	+
	72	+def _apply_850_store_env(env: dict) -> None:
	73	+ """850 store env: move HCCL socket ranges +2000 and the HIXL listen port off the engine's
	74	+ 16666 (the per-device RA socket 60001 collision otherwise kills the store's channels).
	75	+ Explicitly pre-set ranges are respected; only unset ones get the offset store range.
	76	+ """
	77	+ if "ASCEND_SLOG_PRINT_TO_STDOUT" not in env:
	78	+ # Surface device-side (hccp) errors on stdout for log_collect.
	79	+ env["ASCEND_SLOG_PRINT_TO_STDOUT"] = "1"
	80	+ store_range = _offset_port_range("60000-60050", 2000)
	81	+ if "HCCL_HOST_SOCKET_PORT_RANGE" not in env:
	82	+ env["HCCL_HOST_SOCKET_PORT_RANGE"] = store_range
	83	+ if "HCCL_SOCKET_PORT_RANGE" not in env:
	84	+ env["HCCL_SOCKET_PORT_RANGE"] = store_range
	85	+ _merge_hixl_listen_port(env)
	86	+
	87	+
	88	+def _apply_800I_store_env(env: dict) -> None:
	89	+ """800I store env: hccp sockets off the workers' 16666 (EI0020 otherwise) via
	90	+ HCCL_NPU_SOCKET_PORT_RANGE, HIXL listen port via comm_resource_config.
	91	+ """
	92	+ if "HCCL_NPU_SOCKET_PORT_RANGE" not in env:
	93	+ env["HCCL_NPU_SOCKET_PORT_RANGE"] = "16700-16800"
	94	+ _merge_hixl_listen_port(env)
	95	+
	96	+
	97	+def _create_mooncake_store(hardware_type: str, config): # pylint: disable=unused-argument
	98	+ """MooncakeStoreService factory for the daemon registry."""
	99	+ return MooncakeStoreService(
	100	+ hardware_type=hardware_type,
	101	+ kv_cache_store_config=config.kv_cache_store_config,
	102	+ restart_local_service=Env.motor_restart_local_service,
	103	+ )
	104	+
	105	+
	106	+@register_service(
	107	+ SERVICE_KV_STORE,
	108	+ backend="mooncake",
	109	+ prepare_priority=10,
	110	+ factory=_create_mooncake_store,
	111	+)
	112	+class MooncakeStoreService:
	113	+ """Manage the standalone ``mooncake_store_service`` subprocess."""
	114	+
	115	+ def __init__(
	116	+ self,
	117	+ hardware_type: str,
	118	+ kv_cache_store_config: KVCacheStoreConfig \| None = None,
	119	+ restart_local_service: bool = True,
	120	+ ):
	121	+ self.hardware_type = hardware_type
	122	+ self._kv_cfg = kv_cache_store_config or KVCacheStoreConfig()
	123	+ self.restart_local_service = restart_local_service
	124	+
	125	+ self._store_process: subprocess.Popen \| None = None
	126	+ # pull() runs on the API thread while health_check() runs on the monitor
	127	+ # thread; the lock serializes them so a race cannot spawn a duplicate
	128	+ # store (the loser would orphan a process holding the pool segment).
	129	+ self._lock = threading.Lock()
	130	+ # Intent flag: the store should be running. health_check() keys on this
	131	+ # instead of is_started(), so a failed start is retried next cycle
	132	+ # rather than abandoning the store forever.
	133	+ self._desired_running = False
	134	+
	135	+ @property
	136	+ def _can_launch(self) -> bool:
	137	+ return self._kv_cfg.enable and self._kv_cfg.backend == "mooncake" and self._kv_cfg.store_mode == "standalone"
	138	+
	139	+ @staticmethod
	140	+ def _conf_dir() -> str:
	141	+ """Store config lives next to the engine-side mooncake config."""
	142	+ engine_config_path = Env.mooncake_config_path
	143	+ if engine_config_path:
	144	+ return os.path.dirname(engine_config_path)
	145	+ return Env.config_path or _DEFAULT_CONF_DIR
	146	+
	147	+ def _store_config_path(self) -> str:
	148	+ return os.path.join(self._conf_dir(), _STORE_CONFIG_FILENAME)
	149	+
	150	+ def _build_store_config(self) -> dict:
	151	+ """Same schema as the engine config, but the store contributes the segment and needs no staging buffer."""
	152	+ return {
	153	+ "local_hostname": Env.pod_ip or "",
	154	+ "metadata_server": self._kv_cfg.metadata_server,
	155	+ "protocol": self._kv_cfg.protocol,
	156	+ "device_name": self._kv_cfg.device_name,
	157	+ "global_segment_size": self._kv_cfg.global_segment_size,
	158	+ "local_buffer_size": 0,
	159	+ "master_server_address": format_address(self._kv_cfg.service, self._kv_cfg.port),
	160	+ }
	161	+
	162	+ def _ensure_store_config(self) -> str:
	163	+ """(Re)generate the store config file; returns its path or "" on failure."""
	164	+ path = self._store_config_path()
	165	+ try:
	166	+ os.makedirs(os.path.dirname(path), exist_ok=True)
	167	+ with open(path, "w", encoding="utf-8") as f:
	168	+ json.dump(self._build_store_config(), f, indent=2)
	169	+ return path
	170	+ except OSError as e:
	171	+ logger.error("Failed to write mooncake store config %s: %s", path, e)
	172	+ return ""
	173	+
	174	+ def prepare(self, **kwargs) -> None:
	175	+ """Write the store config only — the subprocess is started after engines (pull_kv_store),
	176	+ so the engine launch never serializes on the store.
	177	+ """
	178	+ if not self._kv_cfg.enable or self._kv_cfg.backend != "mooncake":
	179	+ return
	180	+ if self._kv_cfg.store_mode != "standalone":
	181	+ logger.info(
	182	+ "Mooncake store_mode=%r (embedded): no standalone store process", self._kv_cfg.store_mode or "embedded"
	183	+ )
	184	+ return
	185	+ if not self._kv_cfg.global_segment_size:
	186	+ logger.warning("Mooncake standalone store: global_segment_size is empty, using engine built-in default")
	187	+ if self._ensure_store_config():
	188	+ logger.info("Mooncake standalone store config prepared")
	189	+
	190	+ def pull(self) -> None:
	zhinengwu8月25日 pull() 没有并发保护：pull_kv_store 在 API 线程执行，health_check 在 monitor 线程周期执行，两者并发时可能都通过 is_alive() 检查而各自 Popen，_store_process 被后一次覆盖，先启动的 store 进程变成孤儿——持有大段池化内存且永不回收、永不健康检查。建议在 pull/stop/health_check 之间加 threading.Lock 串行化。 zhinengwu 8月25日评论： 8月25日评论：
	191	+ """Start the store subprocess (idempotent); called by the start flow
	192	+ after engines (pull_kv_store) and by monitor restarts.
	193	+ """
	194	+ if not self._can_launch:
	195	+ return
	196	+ with self._lock:
	197	+ self._desired_running = True
	198	+ self._start_locked()
	199	+
	200	+ def _start_locked(self) -> None:
	201	+ """Start the store subprocess if not alive. Caller must hold ``self._lock``."""
	202	+ if self.is_alive():
	203	+ return
	204	+
	205	+ config_path = self._ensure_store_config()
	206	+ if not config_path:
	207	+ return
	208	+
	209	+ is_850 = HardwareType.is_a5(self.hardware_type)
	210	+ if self._kv_cfg.protocol == "ascend":
	211	+ # ACL context must be created inside the store process (does not survive Popen);
	212	+ # both bootstraps stay comm-free (see bootstrap modules).
	213	+ bootstrap = _BOOTSTRAP_ASCEND if is_850 else _BOOTSTRAP_ASCEND_800I
	214	+ cmd = [
	215	+ sys.executable,
	216	+ "-m",
	217	+ bootstrap,
	218	+ "--config",
	219	+ config_path,
	220	+ "--port",
	221	+ str(self._kv_cfg.store_http_port),
	222	+ ]
	223	+ else:
	224	+ cmd = [
	225	+ sys.executable,
	226	+ "-m",
	227	+ "mooncake.mooncake_store_service",
	228	+ "--config",
	229	+ config_path,
	230	+ "--port",
	231	+ str(self._kv_cfg.store_http_port),
	232	+ ]
	233	+ try:
	234	+ logger.info(
	235	+ "Starting mooncake_store_service (segment=%s, master=%s)",
	236	+ self._kv_cfg.global_segment_size,
	237	+ format_address(self._kv_cfg.service, self._kv_cfg.port),
	238	+ )
	239	+ env = os.environ.copy()
	240	+ if is_850:
	241	+ _apply_850_store_env(env)
	242	+ elif self._kv_cfg.protocol == "ascend":
	243	+ _apply_800I_store_env(env)
	244	+ self._store_process = subprocess.Popen( # pylint: disable=consider-using-with
	245	+ cmd, shell=False, env=env
	246	+ )
	247	+ # Config or environment problems surface as an immediate exit.
	248	+ time.sleep(_STARTUP_GRACE_SEC)
	249	+ if self._store_process.poll() is not None:
	250	+ raise RuntimeError(
	251	+ "mooncake_store_service exited immediately with code %s" % self._store_process.returncode
	252	+ )
	253	+ except Exception as e: # pylint: disable=broad-except
	zhinengwu8月25日启动失败（包括 2 秒宽限期内立即退出）后 _store_process 被置为 None，is_started() 随之变成 False，health_check 从此不再重试——一次瞬时故障（配置目录临时不可写、端口临时被占）就让 store 永久退出，和「故障原地重拉」的设计目标相悖。建议引入独立的「应运行」意图标记，health_check 依据它而不是 is_started() 决定是否重拉，必要时加重启退避。 zhinengwu 8月25日评论： 8月25日评论：
	254	+ logger.error("Failed to start mooncake_store_service: %s", e)
	255	+ self._store_process = None
	256	+
	257	+ def stop(self) -> None:
	258	+ with self._lock:
	259	+ self._desired_running = False
	260	+ self._stop_locked()
	261	+
	262	+ def _stop_locked(self) -> None:
	263	+ """Kill the store process. Caller must hold ``self._lock``."""
	264	+ if self._store_process is None:
	265	+ return
	266	+ pid = self._store_process.pid
	267	+ try:
	268	+ os.kill(pid, signal.SIGKILL)
	269	+ self._store_process.wait(timeout=5.0)
	270	+ logger.info("mooncake_store_service terminated (pid=%s)", pid)
	271	+ except ProcessLookupError:
	272	+ logger.info("mooncake_store_service %s already terminated", pid)
	273	+ except subprocess.TimeoutExpired:
	274	+ logger.warning("mooncake_store_service %s did not terminate in time", pid)
	275	+ except Exception as e: # pylint: disable=broad-except
	276	+ logger.error("Failed to kill mooncake_store_service %s: %s", pid, e)
	277	+ finally:
	278	+ self._store_process = None
	279	+
	280	+ def is_started(self) -> bool:
	281	+ return self._store_process is not None
	282	+
	283	+ def is_alive(self) -> bool:
	284	+ if self._store_process is None:
	285	+ return False
	286	+ return self._store_process.poll() is None
	287	+
	288	+ def mark_dead(self) -> None:
	289	+ if self._store_process is not None:
	290	+ self._store_process.poll()
	291	+ self._store_process = None
	292	+
	293	+ def health_check(self) -> None:
	294	+ """Restart the store in place when it should be running but is not; the new
	295	+ process re-registers and remounts. Keys on the intent flag, so a failed
	296	+ start is retried on the next cycle instead of being abandoned.
	297	+ """
	298	+ with self._lock:
	299	+ if not self._desired_running or self.is_alive():
	300	+ return
	301	+ logger.warning(
	302	+ "mooncake_store_service is not running (restart_local_service=%s)",
	303	+ self.restart_local_service,
	304	+ )
	305	+ self.mark_dead()
	306	+ if self.restart_local_service:
	307	+ self._start_locked()

Mmotor/node_manager/core/services/registry.py+1-0

	@@ -51,6 +51,7 @@ SERVICE_KV_STORE: str = "kv_store"
51	_DEFAULT_MODULE_MAP: dict[str \| None, list[str]] = {	51	_DEFAULT_MODULE_MAP: dict[str \| None, list[str]] = {
52	"engine": ["motor.node_manager.core.services.native_engine.service"],	52	"engine": ["motor.node_manager.core.services.native_engine.service"],
53	"memcache": ["motor.node_manager.core.services.memcache.lifecycle"],	53	"memcache": ["motor.node_manager.core.services.memcache.lifecycle"],
		54	+ "mooncake": ["motor.node_manager.core.services.mooncake.lifecycle"],
54	}	55	}
55		56
56		57

Atests/node_manager/core/services/mooncake/test_lifecycle.py+464-0

@@ -0,0 +1,464 @@

2 +# MindIE is licensed under Mulan PSL v2.

3 +# You can use this software according to the terms and conditions of the Mulan PSL v2.

4 +# You may obtain a copy of Mulan PSL v2 at:

5 +# http://license.coscl.org.cn/MulanPSL2

6 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,

7 +# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,

8 +# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.

9 +# See the Mulan PSL v2 for more details.

10 +

11 +"""Tests for the Mooncake standalone store daemon service."""

12 +

13 +import json

14 +import sys

15 +import threading

16 +from unittest.mock import MagicMock, patch

17 +

18 +import pytest

19 +

20 +from motor.config.node_manager import HardwareType, KVCacheStoreConfig

21 +from motor.node_manager.core.services.mooncake.lifecycle import (

22 + MooncakeStoreService,

23 + _BOOTSTRAP_ASCEND,

24 + _BOOTSTRAP_ASCEND_800I,

25 +)

26 +

27 +_850_HW = HardwareType.TYPE_950_SUPERPOD_ATLAS_8.value

28 +_800I_HW = HardwareType.TYPE_800I_A2.value

29 +

30 +_MODULE = "motor.node_manager.core.services.mooncake.lifecycle"

31 +

32 +

33 +# ---------------------------------------------------------------------------

34 +# helpers

35 +# ---------------------------------------------------------------------------

36 +

37 +

38 +def _make_store(**overrides) -> MooncakeStoreService:

39 + """Quick one-off MooncakeStoreService with overrides on top of defaults."""

40 + cfg = KVCacheStoreConfig()

41 + cfg.enable = True

42 + cfg.backend = "mooncake"

43 + cfg.store_mode = "standalone"

44 + cfg.service = "kvs-master"

45 + cfg.port = 50088

46 + cfg.global_segment_size = "600GB"

47 + hw = overrides.pop("hardware_type", _800I_HW)

48 + for k, v in overrides.items():

49 + setattr(cfg, k, v)

50 + return MooncakeStoreService(

51 + hardware_type=hw,

52 + kv_cache_store_config=cfg,

53 + )

54 +

55 +

56 +def _alive_proc() -> MagicMock:

57 + proc = MagicMock()

58 + proc.poll.return_value = None

59 + return proc

60 +

61 +

62 +# ===================================================================

63 +# launch gating — pull() is a no-op unless standalone mode is active

64 +# ===================================================================

65 +

66 +

67 +@patch("subprocess.Popen")

68 +@pytest.mark.parametrize(

69 + "overrides",

70 + [

71 + {"store_mode": "embedded"},

72 + {"store_mode": ""},

73 + {"enable": False},

74 + {"backend": "memcache"},

75 + ],

76 + ids=["embedded", "default_mode", "disabled", "wrong_backend"],

77 +)

78 +def test_pull_skipped_unless_standalone(mock_popen, overrides):

79 + """prepare()+pull() launch nothing unless store_mode == "standalone"."""

80 + store = _make_store(**overrides)

81 + store.prepare()

82 + store.pull()

83 + mock_popen.assert_not_called()

84 +

85 +

86 +@patch("subprocess.Popen")

87 +def test_prepare_does_not_launch_store(mock_popen, tmp_path, monkeypatch):

88 + """prepare() only writes the store config; the store process must be

89 + started after engines (pull_kv_store), never during engine start.

90 + """

91 + monkeypatch.setenv("MOONCAKE_CONFIG_PATH", str(tmp_path / "kv_cache_store_config.json"))

92 + monkeypatch.setenv("POD_IP", "10.0.0.1")

93 +

94 + store = _make_store()

95 + store.prepare()

96 +

97 + mock_popen.assert_not_called()

98 + assert store.is_started() is False

99 + # config file was written for the later pull_kv_store launch

100 + generated = json.loads((tmp_path / "mooncake_store_config.json").read_text(encoding="utf-8"))

101 + assert generated["protocol"] == "ascend"

102 + assert generated["global_segment_size"] == "600GB"

103 +

104 +

105 +# ===================================================================

106 +# pull() — standalone store (Popen)

107 +# ===================================================================

108 +

109 +

110 +@patch(f"{_MODULE}.time.sleep")

111 +@patch("subprocess.Popen")

112 +def test_pull_launches_official_store_service(mock_popen, _sleep, tmp_path, monkeypatch):

113 + """pull() runs the official store via the 850 bootstrap module (python -m),

114 + which sets up the ACL device context + HCCL comm first (the standalone

115 + store process never calls aclrtSetDevice itself, so AscendDirectTransport

116 + would fail with "cannot allocate local segment, ret: -1").

117 + """

118 + mock_popen.return_value = _alive_proc()

119 + monkeypatch.setenv("MOONCAKE_CONFIG_PATH", str(tmp_path / "kv_cache_store_config.json"))

120 + monkeypatch.setenv("POD_IP", "10.0.0.1")

121 +

122 + store = _make_store(hardware_type=_850_HW)

123 + store.pull()

124 +

125 + mock_popen.assert_called_once()

126 + call_args, call_kwargs = mock_popen.call_args

127 + cmd = call_args[0]

128 + assert cmd[0] == sys.executable

129 + assert cmd[1] == "-m"

130 + assert cmd[2] == _BOOTSTRAP_ASCEND

131 + assert cmd[3:] == ["--config", str(tmp_path / "mooncake_store_config.json"), "--port", "0"]

132 + assert call_kwargs["shell"] is False

133 + assert store.is_alive() is True

134 +

135 +

136 +@patch(f"{_MODULE}.time.sleep")

137 +@patch("subprocess.Popen")

138 +def test_pull_injects_hixl_listen_port(mock_popen, _sleep, tmp_path, monkeypatch):

139 + """ascend protocol: point HIXL's device NIC listen port away from 16666

140 + (which vllm workers bind first) so the standalone store can build its

141 + channel. A pre-set ASCEND_GLOBAL_RESOURCE_CONFIG is MERGED, not replaced —

142 + the pod may ship comm_resource_config.protocol_desc (UBOE/UB), and

143 + dropping it would silently lose the listen_port override and put the store

144 + back on the engine's 16666.

145 + """

146 + mock_popen.return_value = _alive_proc()

147 + monkeypatch.setenv("MOONCAKE_CONFIG_PATH", str(tmp_path / "kv_cache_store_config.json"))

148 + monkeypatch.setenv("POD_IP", "10.0.0.1")

149 +

150 + _make_store().pull()

151 + env = mock_popen.call_args.kwargs["env"]

152 + assert env["ASCEND_GLOBAL_RESOURCE_CONFIG"] == '{"comm_resource_config.listen_port": "26666"}'

atomgit-bot8月21日

🟡 Medium Priority

变更行 → 受影响行为：test_pull_injects_hixl_listen_port 第一处断言（tests/node_manager/core/services/mooncake/test_lifecycle.py:150）期望 ASCEND_GLOBAL_RESOURCE_CONFIG == '{"comm_resource_config.listen_port": "26666"}'，而 lifecycle.py:256 的代码默认值是 env.get("HIXL_LISTEN_PORT", "16667")，且该测试只 monkeypatch 了 MOONCAKE_CONFIG_PATH 和 POD_IP，未设置 HIXL_LISTEN_PORT。

失败模式：在干净 CI 环境中 HIXL_LISTEN_PORT 未设置时，pull() 生成的 JSON 为 {"comm_resource_config.listen_port": "16667"}，与断言 "26666" 不匹配，第一个断言必然抛 AssertionError，导致 tests/run_tests.sh 测试失败（与 PR 描述的"全部通过"不符；只有运行环境中恰好存在 HIXL_LISTEN_PORT=26666 时才会偶然通过，属于环境相关的不稳定用例）。

建议：将断言值改为 '{"comm_resource_config.listen_port": "16667"}'，与 lifecycle.py 中 env.get("HIXL_LISTEN_PORT", "16667") 的默认值对齐。

改动建议

152	- assert env["ASCEND_GLOBAL_RESOURCE_CONFIG"] == '{"comm_resource_config.listen_port": "26666"}'
152	+ assert env["ASCEND_GLOBAL_RESOURCE_CONFIG"] == '{"comm_resource_config.listen_port": "16667"}'

应用建议

不准确？

zhinengwu

8月25日评论：

153 +

154 + monkeypatch.setenv("HIXL_LISTEN_PORT", "30000")

155 + _make_store().pull()

156 + env = mock_popen.call_args.kwargs["env"]

157 + assert env["ASCEND_GLOBAL_RESOURCE_CONFIG"] == '{"comm_resource_config.listen_port": "30000"}'

158 +

159 + # user pre-set AGRC keeps its own keys; listen_port is merged in

160 + monkeypatch.setenv("ASCEND_GLOBAL_RESOURCE_CONFIG", '{"fabric_memory.max_capacity": "64"}')

161 + monkeypatch.delenv("HIXL_LISTEN_PORT", raising=False)

162 + _make_store().pull()

163 + env = mock_popen.call_args.kwargs["env"]

164 + assert env["ASCEND_GLOBAL_RESOURCE_CONFIG"] == (

165 + '{"fabric_memory.max_capacity": "64", "comm_resource_config.listen_port": "26666"}'

166 + )

167 +

168 + # pre-set comm_resource_config.* keys (official dotted form, e.g. UBOE)

169 + # survive the merge

170 + monkeypatch.setenv(

171 + "ASCEND_GLOBAL_RESOURCE_CONFIG",

172 + '{"comm_resource_config.protocol_desc": ["uboe:device"]}',

173 + )

174 + _make_store().pull()

175 + env = mock_popen.call_args.kwargs["env"]

176 + assert env["ASCEND_GLOBAL_RESOURCE_CONFIG"] == (

177 + '{"comm_resource_config.protocol_desc": ["uboe:device"], "comm_resource_config.listen_port": "26666"}'

178 + )

179 +

180 +

181 +@patch(f"{_MODULE}.time.sleep")

182 +@patch("subprocess.Popen")

183 +@pytest.mark.parametrize("hw", [_800I_HW, HardwareType.TYPE_800I_A3.value], ids=["800I-A2", "800I-A3"])

184 +def test_pull_800I_acl_context_without_comm(mock_popen, _sleep, tmp_path, monkeypatch, hw):

185 + """800I: the store bootstrap provides ONLY the ACL context — no HCCL comm.

186 + A comm would make the store's adxl engine merge its own root info into the

187 + P2P handshake; on a same-node PD deployment the store shares physical

188 + devices with the vllm workers, so the merged channel rank table repeats a

189 + device IP and HCCL rejects it with EI0014 (TP0 fails every put, TP1

190 + succeeds) — the comm-free handshake is TP-agnostic. The store's hccp

191 + sockets still move off the workers' 16666 (EI0020 otherwise).

192 + """

193 + mock_popen.return_value = _alive_proc()

194 + monkeypatch.setenv("MOONCAKE_CONFIG_PATH", str(tmp_path / "kv_cache_store_config.json"))

195 + monkeypatch.setenv("POD_IP", "10.0.0.1")

196 +

197 + _make_store(hardware_type=hw).pull()

198 +

199 + call_args, call_kwargs = mock_popen.call_args

200 + cmd = call_args[0]

201 + assert cmd[1] == "-m"

202 + assert cmd[2] == _BOOTSTRAP_ASCEND_800I

203 +

204 + env = call_kwargs["env"]

205 + assert env["HCCL_NPU_SOCKET_PORT_RANGE"] == "16700-16800"

206 + assert env["ASCEND_GLOBAL_RESOURCE_CONFIG"] == '{"comm_resource_config.listen_port": "26666"}'

207 +

208 +

209 +@patch(f"{_MODULE}.time.sleep")

210 +@patch("subprocess.Popen")

211 +def test_pull_injects_slog_print_to_stdout(mock_popen, _sleep, tmp_path, monkeypatch):

212 + """store device-side (hccp) errors are surfaced on the store stdout so

213 + log_collect keeps them (they would otherwise go to the host slog, which

214 + the pod cannot read); a pre-set ASCEND_SLOG_PRINT_TO_STDOUT wins.

215 + """

216 + mock_popen.return_value = _alive_proc()

217 + monkeypatch.setenv("MOONCAKE_CONFIG_PATH", str(tmp_path / "kv_cache_store_config.json"))

218 + monkeypatch.setenv("POD_IP", "10.0.0.1")

219 +

220 + _make_store(hardware_type=_850_HW).pull()

221 + env = mock_popen.call_args.kwargs["env"]

222 + assert env["ASCEND_SLOG_PRINT_TO_STDOUT"] == "1"

223 +

224 + monkeypatch.setenv("ASCEND_SLOG_PRINT_TO_STDOUT", "0")

225 + _make_store(hardware_type=_850_HW).pull()

226 + env = mock_popen.call_args.kwargs["env"]

227 + assert env["ASCEND_SLOG_PRINT_TO_STDOUT"] == "0"

228 +

229 +

230 +@patch(f"{_MODULE}.time.sleep")

231 +@patch("subprocess.Popen")

232 +def test_pull_850_respects_preset_socket_ranges(mock_popen, _sleep, tmp_path, monkeypatch):

233 + """850: an explicitly pre-set HCCL_(HOST_)SOCKET_PORT_RANGE is the user's

234 + deliberate choice and must survive; only unset ones get the +2000 store range.

235 + """

236 + mock_popen.return_value = _alive_proc()

237 + monkeypatch.setenv("MOONCAKE_CONFIG_PATH", str(tmp_path / "kv_cache_store_config.json"))

238 + monkeypatch.setenv("POD_IP", "10.0.0.1")

239 + monkeypatch.setenv("HCCL_SOCKET_PORT_RANGE", "50000-50050")

240 +

241 + _make_store(hardware_type=_850_HW).pull()

242 + env = mock_popen.call_args.kwargs["env"]

243 + assert env["HCCL_SOCKET_PORT_RANGE"] == "50000-50050"

244 + assert env["HCCL_HOST_SOCKET_PORT_RANGE"] == "62000-62050"

245 +

246 +

247 +@patch(f"{_MODULE}.time.sleep")

248 +@patch("subprocess.Popen")

249 +def test_pull_non_ascend_protocol_uses_plain_module(mock_popen, _sleep, tmp_path, monkeypatch):

250 + """Non-ascend protocols keep the plain ``python -m`` launch (no ACL needed)."""

251 + mock_popen.return_value = _alive_proc()

252 + monkeypatch.setenv("MOONCAKE_CONFIG_PATH", str(tmp_path / "kv_cache_store_config.json"))

253 +

254 + _make_store(protocol="tcp").pull()

255 +

256 + mock_popen.assert_called_once()

257 + cmd = mock_popen.call_args[0][0]

258 + assert cmd[:3] == [sys.executable, "-m", "mooncake.mooncake_store_service"]

259 + assert cmd[3:] == ["--config", str(tmp_path / "mooncake_store_config.json"), "--port", "0"]

260 +

261 +

262 +@patch(f"{_MODULE}.time.sleep")

263 +@patch("subprocess.Popen")

264 +def test_pull_writes_store_config(mock_popen, _sleep, tmp_path, monkeypatch):

265 + """pull() generates the store config: segment from cfg, no staging buffer."""

266 + mock_popen.return_value = _alive_proc()

267 + monkeypatch.setenv("MOONCAKE_CONFIG_PATH", str(tmp_path / "kv_cache_store_config.json"))

268 + monkeypatch.setenv("POD_IP", "10.0.0.1")

269 +

270 + _make_store(store_http_port=9090).pull()

271 +

272 + generated = json.loads((tmp_path / "mooncake_store_config.json").read_text(encoding="utf-8"))

273 + assert generated["global_segment_size"] == "600GB"

274 + assert generated["local_buffer_size"] == 0

275 + assert generated["master_server_address"] == "kvs-master:50088"

276 + assert generated["local_hostname"] == "10.0.0.1"

277 + assert generated["metadata_server"] == "P2PHANDSHAKE"

278 + assert generated["protocol"] == "ascend"

279 + # store_http_port reached the command line, not the config file

280 + assert mock_popen.call_args[0][0][-1] == "9090"

281 +

282 +

283 +@patch(f"{_MODULE}.time.sleep")

284 +@patch("subprocess.Popen")

285 +def test_pull_brackets_ipv6_master(mock_popen, _sleep, tmp_path, monkeypatch):

286 + """IPv6 master service FQDN is bracketed in master_server_address."""

287 + mock_popen.return_value = _alive_proc()

288 + monkeypatch.setenv("MOONCAKE_CONFIG_PATH", str(tmp_path / "kv_cache_store_config.json"))

289 + monkeypatch.delenv("POD_IP", raising=False)

290 +

291 + _make_store(service="2001:db8::1").pull()

292 +

293 + generated = json.loads((tmp_path / "mooncake_store_config.json").read_text(encoding="utf-8"))

294 + assert generated["master_server_address"] == "[2001:db8::1]:50088"

295 +

296 +

297 +@patch(f"{_MODULE}.time.sleep")

298 +@patch("subprocess.Popen")

299 +def test_pull_idempotent_when_alive(mock_popen, _sleep, tmp_path, monkeypatch):

300 + """Second pull() is a no-op while the store is running."""

301 + mock_popen.return_value = _alive_proc()

302 + monkeypatch.setenv("MOONCAKE_CONFIG_PATH", str(tmp_path / "kv_cache_store_config.json"))

303 +

304 + store = _make_store()

305 + store.pull()

306 + store.pull()

307 +

308 + assert mock_popen.call_count == 1

309 +

310 +

311 +@patch(f"{_MODULE}.time.sleep")

312 +@patch("subprocess.Popen")

313 +def test_pull_process_exits_immediately(mock_popen, _sleep, tmp_path, monkeypatch):

314 + """Immediate startup failure is detected and the store stays unstarted."""

315 + mock_proc = MagicMock()

316 + mock_proc.poll.return_value = 1

317 + mock_popen.return_value = mock_proc

318 + monkeypatch.setenv("MOONCAKE_CONFIG_PATH", str(tmp_path / "kv_cache_store_config.json"))

319 +

320 + store = _make_store()

321 + store.pull()

322 +

323 + assert store.is_alive() is False

324 + assert store.is_started() is False

325 +

326 +

327 +@patch(f"{_MODULE}.time.sleep")

328 +@patch("subprocess.Popen")

329 +def test_health_check_retries_after_start_failure(mock_popen, _sleep, tmp_path, monkeypatch):

330 + """A failed start (is_started()=False) must NOT abandon the store: the intent

331 + flag survives, so the next health_check retries the launch. Otherwise one

332 + transient failure (config dir briefly unwritable, port briefly taken) would

333 + permanently disable pooling, contradicting restart-in-place.

334 + """

335 + monkeypatch.setenv("MOONCAKE_CONFIG_PATH", str(tmp_path / "kv_cache_store_config.json"))

336 + dead_proc = MagicMock()

337 + dead_proc.poll.return_value = 1

338 + mock_popen.side_effect = [dead_proc, _alive_proc()]

339 +

340 + store = _make_store()

341 + store.pull()

342 + assert store.is_started() is False

343 +

344 + store.health_check()

345 +

346 + assert mock_popen.call_count == 2

347 + assert store.is_alive() is True

348 +

349 +

350 +@patch(f"{_MODULE}.time.sleep")

351 +@patch("subprocess.Popen")

352 +def test_health_check_no_restart_after_stop(mock_popen, _sleep, tmp_path, monkeypatch):

353 + """stop() clears the intent flag: health_check must not resurrect the store."""

354 + monkeypatch.setenv("MOONCAKE_CONFIG_PATH", str(tmp_path / "kv_cache_store_config.json"))

355 + mock_popen.return_value = _alive_proc()

356 +

357 + store = _make_store()

358 + store.pull()

359 + store.stop()

360 + store.health_check()

361 +

362 + assert mock_popen.call_count == 1

363 +

364 +

365 +@patch(f"{_MODULE}.time")

366 +@patch("subprocess.Popen")

367 +def test_concurrent_pull_starts_single_store(mock_popen, _time, tmp_path, monkeypatch):

368 + """pull() runs on the API thread while health_check() runs on the monitor

369 + thread; a race must not spawn two stores — the loser would orphan a process

370 + holding the pool segment, never health-checked and never reaped.

371 + """

372 + monkeypatch.setenv("MOONCAKE_CONFIG_PATH", str(tmp_path / "kv_cache_store_config.json"))

373 + entered = threading.Event()

374 + release = threading.Event()

375 +

376 + def _slow_popen(*args, **kwargs):

377 + entered.set()

378 + release.wait(5)

379 + return _alive_proc()

380 +

381 + mock_popen.side_effect = _slow_popen

382 +

383 + store = _make_store()

384 + racer = threading.Thread(target=store.pull)

385 + racer.start()

386 + assert entered.wait(5)

387 + store.pull() # main thread races while the first Popen is still in flight

388 + release.set()

389 + racer.join(timeout=5)

390 +

391 + assert mock_popen.call_count == 1

392 + assert store.is_alive() is True

393 +

394 +

395 +# ===================================================================

396 +# stop()

397 +# ===================================================================

398 +

399 +

400 +def test_stop_kills_process():

401 + store = _make_store()

402 + mock_proc = MagicMock()

403 + mock_proc.pid = 12345

404 + store._store_process = mock_proc

405 +

406 + with patch("os.kill") as mock_kill:

407 + store.stop()

408 + mock_kill.assert_called_once_with(12345, 9) # signal.SIGKILL

409 +

410 +

411 +def test_stop_process_not_found():

412 + store = _make_store()

413 + mock_proc = MagicMock()

414 + mock_proc.pid = 12345

415 + store._store_process = mock_proc

416 +

417 + with patch("os.kill", side_effect=ProcessLookupError):

418 + store.stop()

419 +

420 + assert store._store_process is None

421 +

422 +

423 +# ===================================================================

424 +# health_check() — restart in place

425 +# ===================================================================

426 +

427 +

428 +@patch(f"{_MODULE}.time.sleep")

429 +@patch("subprocess.Popen")

430 +def test_health_check_restarts_dead_store(mock_popen, _sleep, tmp_path, monkeypatch):

431 + """Dead store is re-pulled in place when restart is enabled."""

432 + monkeypatch.setenv("MOONCAKE_CONFIG_PATH", str(tmp_path / "kv_cache_store_config.json"))

433 + proc1, proc2 = _alive_proc(), _alive_proc()

434 + mock_popen.side_effect = [proc1, proc2]

435 +

436 + store = _make_store()

437 + store.pull()

438 + assert mock_popen.call_count == 1

439 +

440 + # Store dies

441 + proc1.poll.return_value = 1

442 + store.health_check()

443 +

444 + assert mock_popen.call_count == 2

445 + assert store._store_process is proc2

446 + assert store.is_alive() is True

447 +

448 +

449 +@patch(f"{_MODULE}.time.sleep")

450 +@patch("subprocess.Popen")

451 +def test_health_check_no_restart_when_disabled(mock_popen, _sleep, tmp_path, monkeypatch):

452 + """MOTOR_RESTART_LOCAL_SERVICE=0 semantics: dead store stays dead."""

453 + monkeypatch.setenv("MOONCAKE_CONFIG_PATH", str(tmp_path / "kv_cache_store_config.json"))

454 + mock_popen.return_value = _alive_proc()

455 +

456 + store = _make_store()

457 + store.restart_local_service = False

458 + store.pull()

459 +

460 + store._store_process.poll.return_value = 1

461 + store.health_check()

462 +

463 + assert mock_popen.call_count == 1

464 + assert store.is_started() is False

Atests/node_manager/core/services/mooncake/test_store_bootstrap.py+94-0

@@ -0,0 +1,94 @@
	1	+# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved.
	2	+# MindIE is licensed under Mulan PSL v2.
	3	+# You can use this software according to the terms and conditions of the Mulan PSL v2.
	4	+# You may obtain a copy of Mulan PSL v2 at:
	5	+# http://license.coscl.org.cn/MulanPSL2
	6	+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
	7	+# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
	8	+# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
	9	+# See the Mulan PSL v2 for more details.
	10	+
	11	+"""Unit tests for the store-process bootstraps (bootstrap/ascend*).
	12	+
	13	+The real ``acl`` is not available in CI; the modules are exercised with a
	14	+stand-in ``acl`` module injected into ``sys.modules``. What matters here: the
	15	+ACL context is created before the store service runs, and every failure is
	16	+non-fatal — the store must always start.
	17	+"""
	18	+
	19	+import sys
	20	+from unittest.mock import MagicMock, patch
	21	+
	22	+import pytest
	23	+
	24	+from motor.node_manager.core.services.mooncake.bootstrap import ascend_800I, ascend_850
	25	+
	26	+
	27	+class _FakeACL:
	28	+ """Minimal stand-in for the real ``acl`` module (not installed in CI)."""
	29	+
	30	+ def __init__(self, device_count=2, device_count_ret=0):
	31	+ self.device_count = device_count
	32	+ self.device_count_ret = device_count_ret
	33	+ self.calls = []
	34	+ self.rt = MagicMock()
	35	+ self.rt.get_device_count.side_effect = lambda: (self.device_count, self.device_count_ret)
	36	+
	37	+ def init(self):
	38	+ self.calls.append("init")
	39	+
	40	+
	41	+@pytest.fixture
	42	+def fake_acl():
	43	+ fake = _FakeACL()
	44	+ with patch.dict(sys.modules, {"acl": fake}):
	45	+ yield fake
	46	+
	47	+
	48	+def _run_module_mock(module):
	49	+ """Context manager patching runpy.run_module inside a bootstrap module."""
	50	+ return patch.object(module.runpy, "run_module")
	51	+
	52	+
	53	+# ===================================================================
	54	+# ascend_800I — ACL context only, no HCCL comm
	55	+# ===================================================================
	56	+
	57	+
	58	+def test_800I_bootstrap_sets_up_acl_and_runs_store(fake_acl):
	59	+ """ACL context first, then the official store service."""
	60	+ with _run_module_mock(ascend_800I) as run_module:
	61	+ ascend_800I.main()
	62	+ run_module.assert_called_once_with("mooncake.mooncake_store_service", run_name="__main__")
	63	+ assert fake_acl.calls == ["init"]
	64	+ fake_acl.rt.set_device.assert_called_once_with(0)
	65	+
	66	+
	67	+def test_800I_bootstrap_continues_when_acl_unavailable(caplog):
	68	+ """acl import/init failure is non-fatal: the store still starts."""
	69	+ with patch.dict(sys.modules, {"acl": None}), _run_module_mock(ascend_800I) as run_module:
	70	+ ascend_800I.main()
	71	+ run_module.assert_called_once()
	72	+ assert any("ACL context init failed (non-fatal)" in record.message for record in caplog.records)
	73	+
	74	+
	75	+# ===================================================================
	76	+# ascend_850 — ACL context only, comm-free like 800I (pending A5 re-validation)
	77	+# ===================================================================
	78	+
	79	+
	80	+def test_850_bootstrap_sets_up_acl_and_runs_store(fake_acl):
	81	+ """ACL context first, then the official store service."""
	82	+ with _run_module_mock(ascend_850) as run_module:
	83	+ ascend_850.main()
	84	+ run_module.assert_called_once_with("mooncake.mooncake_store_service", run_name="__main__")
	85	+ assert fake_acl.calls == ["init"]
	86	+ fake_acl.rt.set_device.assert_called_once_with(0)
	87	+
	88	+
	89	+def test_850_bootstrap_continues_when_acl_unavailable(caplog):
	90	+ """acl import/init failure is non-fatal: the store still starts."""
	91	+ with patch.dict(sys.modules, {"acl": None}), _run_module_mock(ascend_850) as run_module:
	92	+ ascend_850.main()
	93	+ run_module.assert_called_once()
	94	+ assert any("ACL context init failed (non-fatal)" in record.message for record in caplog.records)

Mtests/node_manager/core/services/test_registry.py+5-5

	@@ -220,14 +220,14 @@ class TestServiceRegistry:
220		220
221	def test_add_discovery_path(self):	221	def test_add_discovery_path(self):
222	registry = _ServiceRegistry()	222	registry = _ServiceRegistry()
223	- registry.add_discovery_path("mooncake", "tests.node_manager.test_registry")	223	+ registry.add_discovery_path("memcache", "tests.node_manager.test_registry")
224	- assert "mooncake" in registry._module_map	224	+ assert "memcache" in registry._module_map
225	- assert "tests.node_manager.test_registry" in registry._module_map["mooncake"]	225	+ assert "tests.node_manager.test_registry" in registry._module_map["memcache"]
226		226
227	def test_add_discovery_path_multiple(self):	227	def test_add_discovery_path_multiple(self):
228	registry = _ServiceRegistry()	228	registry = _ServiceRegistry()
229	- registry.add_discovery_path("mooncake", "mod.a", "mod.b")	229	+ registry.add_discovery_path("future_backend", "mod.a", "mod.b")
230	- assert registry._module_map["mooncake"] == ["mod.a", "mod.b"]	230	+ assert registry._module_map["future_backend"] == ["mod.a", "mod.b"]
231		231
232		232
233	# ---------------------------------------------------------------------------	233	# ---------------------------------------------------------------------------

Mtests/node_manager/test_config.py+58-0

	@@ -1154,3 +1154,61 @@ def test_config_engine_restart_validation(attr, value, expected):
1154	config = NodeManagerConfig()	1154	config = NodeManagerConfig()
1155	setattr(config.fault_tolerance_config, attr, value)	1155	setattr(config.fault_tolerance_config, attr, value)
1156	config.validate_config()	1156	config.validate_config()
		1157	+
		1158	+
		1159	+# ===================================================================
		1160	+# kv_cache_store_config — Mooncake standalone store mode
		1161	+# ===================================================================
		1162	+
		1163	+
		1164	+def _parse_kv(raw_kv: dict):
		1165	+ config = NodeManagerConfig()
		1166	+ NodeManagerConfig._parse_kv_cache_store_config(config, {"kv_cache_store_config": raw_kv})
		1167	+ return config.kv_cache_store_config
		1168	+
		1169	+
		1170	+def test_kv_config_mooncake_standalone_fields():
		1171	+ kcfg = _parse_kv(
		1172	+ {
		1173	+ "backend": "mooncake",
		1174	+ "store_mode": "standalone",
		1175	+ "global_segment_size": "600GB",
		1176	+ "local_buffer_size": "2GB",
		1177	+ "store_http_port": 9090,
		1178	+ "metadata_server": "etcd://10.0.0.1:2379",
		1179	+ "protocol": "tcp",
		1180	+ "device_name": "mlx5_0",
		1181	+ }
		1182	+ )
		1183	+ assert kcfg.enable is True
		1184	+ assert kcfg.store_mode == "standalone"
		1185	+ assert kcfg.global_segment_size == "600GB"
		1186	+ assert kcfg.local_buffer_size == "2GB"
		1187	+ assert kcfg.store_http_port == 9090
		1188	+ assert kcfg.metadata_server == "etcd://10.0.0.1:2379"
		1189	+ assert kcfg.protocol == "tcp"
		1190	+ assert kcfg.device_name == "mlx5_0"
		1191	+
		1192	+
		1193	+def test_kv_config_mooncake_defaults():
		1194	+ """Omitted mooncake fields fall back to embedded mode and Ascend defaults."""
		1195	+ kcfg = _parse_kv({"backend": "mooncake"})
		1196	+ assert kcfg.store_mode == ""
		1197	+ assert kcfg.global_segment_size == ""
		1198	+ assert kcfg.local_buffer_size == ""
		1199	+ assert kcfg.store_http_port == 0
		1200	+ assert kcfg.metadata_server == "P2PHANDSHAKE"
		1201	+ assert kcfg.protocol == "ascend"
		1202	+ assert kcfg.device_name == ""
		1203	+
		1204	+
		1205	+def test_kv_config_mooncake_invalid_store_mode_falls_back():
		1206	+ kcfg = _parse_kv({"backend": "mooncake", "store_mode": "bogus"})
		1207	+ assert kcfg.store_mode == "embedded"
		1208	+
		1209	+
		1210	+def test_kv_config_memcache_ignores_mooncake_fields():
		1211	+ """Mooncake-only fields are not parsed for the memcache backend."""
		1212	+ kcfg = _parse_kv({"backend": "memcache", "store_mode": "standalone", "global_segment_size": "1GB"})
		1213	+ assert kcfg.store_mode == ""
		1214	+ assert kcfg.global_segment_size == ""