已合并
vllm_to_motor全量配置转化工具 #471
高鹏创建于 7月10日
vllm_to_motor全量配置转化工具 #471
已合并
共 24 个文件变更+2050-955
| @@ -14,6 +14,8 @@ | |||
| 14 | 14 | ||
| 15 | ### 参数说明 | 15 | ### 参数说明 |
| 16 | 16 | ||
| 17 | +Motor**服务部署**参数说明 | ||
| 18 | + | ||
| 17 | | 参数 | 简写 | 说明 | | 19 | | 参数 | 简写 | 说明 | |
| 18 | |------|------|------| | 20 | |------|------|------| |
| 19 | | `--config_dir` | `--dir` | 配置文件所在目录，目录下需包含 `user_config.json` 和 `env.json` | | 21 | | `--config_dir` | `--dir` | 配置文件所在目录，目录下需包含 `user_config.json` 和 `env.json` | |
| @@ -25,6 +27,16 @@ | |||
| 25 | | `--auto_log_collect` | - | 部署完成后自动启动日志采集 | | 27 | | `--auto_log_collect` | - | 部署完成后自动启动日志采集 | |
| 26 | | `--nostep` | - | 部署完成后不显示服务启动进度条 | | 28 | | `--nostep` | - | 部署完成后不显示服务启动进度条 | |
| 27 | 29 | ||
| 30 | +Motor**配置文件自动生成**参数说明 | ||
| 31 | + | ||
| 32 | +| 参数 | 简写 | 说明 | | ||
| 33 | +|------|------|------| | ||
| 34 | +| `--mode` | - | `deploy`（默认）或 `general_config`（从 vLLM 脚本生成配置） | | ||
| 35 | +| `--deploy-scenario` | - | `general_config` 必填：`hybrid` / `separate` | | ||
| 36 | +| `--hardware-type` | - | `general_config` 必填：`A2` / `A3` | | ||
| 37 | +| `--weight-path` | - | `general_config` 可选：权重挂载路径 | | ||
| 38 | +| `--image-name` | - | `general_config` 可选：镜像名称 | | ||
| 39 | + | ||
| 28 | ### 使用方式 | 40 | ### 使用方式 |
| 29 | 41 | ||
| 30 | #### 方式零：交互式 TUI 模式 | 42 | #### 方式零：交互式 TUI 模式 |
| @@ -74,6 +86,10 @@ python deploy.py --config_dir ../infer_engines/vllm --config /path/to/custom_use | |||
| 74 | 86 | ||
| 75 | 当同时指定 `--config_dir` 和 `--config`/`--env` 时，以 `--config` 和 `--env` 为准。 | 87 | 当同时指定 `--config_dir` 和 `--config`/`--env` 时，以 `--config` 和 `--env` 为准。 |
| 76 | 88 | ||
| 89 | +#### 方式四：基于vllm部署脚本生成Motor全量配置文件 | ||
| 90 | + | ||
| 91 | +使用方式请参阅[Motor配置自动生成指导](../infer_engines/vllm/models/README.md)。 | ||
| 92 | + | ||
| 77 | ### 其他操作 | 93 | ### 其他操作 |
| 78 | 94 | ||
| 79 | #### 更新配置 | 95 | #### 更新配置 |
Rexamples/infer_engines/vllm/models/deepseek/v4/pro/env.json→examples/deployer/config_tool/output_config/env.json+8-18
| @@ -3,24 +3,20 @@ | |||
| 3 | "motor_common_env": { | 3 | "motor_common_env": { |
| 4 | "CANN_INSTALL_PATH": "/usr/local/Ascend" | 4 | "CANN_INSTALL_PATH": "/usr/local/Ascend" |
| 5 | }, | 5 | }, |
| 6 | - "motor_controller_env": { | 6 | + "motor_controller_env": {}, |
| 7 | - }, | 7 | + "motor_coordinator_env": {}, |
| 8 | - "motor_coordinator_env": { | ||
| 9 | - }, | ||
| 10 | "motor_engine_prefill_env": { | 8 | "motor_engine_prefill_env": { |
| 11 | "VLLM_RPC_TIMEOUT": 3600000, | 9 | "VLLM_RPC_TIMEOUT": 3600000, |
| 12 | "VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS": 30000, | 10 | "VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS": 30000, |
| 13 | "HCCL_EXEC_TIMEOUT": 204, | 11 | "HCCL_EXEC_TIMEOUT": 204, |
| 14 | - "HCCL_CONNECT_TIMEOUT": 1200, | 12 | + "HCCL_CONNECT_TIMEOUT": 120, |
| 15 | "OMP_PROC_BIND": "false", | 13 | "OMP_PROC_BIND": "false", |
| 16 | "OMP_NUM_THREADS": 10, | 14 | "OMP_NUM_THREADS": 10, |
| 17 | "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", | 15 | "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", |
| 18 | - "HCCL_BUFFSIZE": 1024, | 16 | + "HCCL_BUFFSIZE": 2560, |
| 19 | "TASK_QUEUE_ENABLE": 1, | 17 | "TASK_QUEUE_ENABLE": 1, |
| 20 | "HCCL_OP_EXPANSION_MODE": "AIV", | 18 | "HCCL_OP_EXPANSION_MODE": "AIV", |
| 21 | - "ASCEND_BUFFER_POOL": "4:8", | 19 | + "VLLM_ASCEND_ENABLE_FLASHCOMM1": 1 |
| 22 | - "USE_MULTI_GROUPS_KV_CACHE": 1, | ||
| 23 | - "USE_MULTI_BLOCK_POOL": 1 | ||
| 24 | }, | 20 | }, |
| 25 | "motor_engine_decode_env": { | 21 | "motor_engine_decode_env": { |
| 26 | "HCCL_OP_EXPANSION_MODE": "AIV", | 22 | "HCCL_OP_EXPANSION_MODE": "AIV", |
| @@ -29,16 +25,10 @@ | |||
| 29 | "VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS": 30000, | 25 | "VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS": 30000, |
| 30 | "HCCL_EXEC_TIMEOUT": 204, | 26 | "HCCL_EXEC_TIMEOUT": 204, |
| 31 | "HCCL_CONNECT_TIMEOUT": 1200, | 27 | "HCCL_CONNECT_TIMEOUT": 1200, |
| 32 | - "VLLM_ASCEND_ENABLE_FUSED_MC2": 1, | ||
| 33 | "OMP_PROC_BIND": "false", | 28 | "OMP_PROC_BIND": "false", |
| 34 | "OMP_NUM_THREADS": 10, | 29 | "OMP_NUM_THREADS": 10, |
| 35 | "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", | 30 | "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", |
| 36 | - "HCCL_BUFFSIZE": 1024, | 31 | + "HCCL_BUFFSIZE": 1024 |
| 37 | - "ASCEND_BUFFER_POOL": "4:8", | ||
| 38 | - "VLLM_TORCH_PROFILER_DIR": "./vllm_profile", | ||
| 39 | - "USE_MULTI_GROUPS_KV_CACHE": 1, | ||
| 40 | - "USE_MULTI_BLOCK_POOL": 1 | ||
| 41 | }, | 32 | }, |
| 42 | - "motor_kv_cache_pool_env": { | 33 | + "motor_kv_cache_pool_env": {} |
| 43 | - } | 34 | +} |
| 44 | -} | ||
Rexamples/infer_engines/vllm/models/deepseek/v4/flash/user_config.json→examples/deployer/config_tool/output_config/user_config.json+130-119
| @@ -1,119 +1,130 @@ | |||
| 1 | -{ | 1 | +{ |
| 2 | - "version": "v2.0", | 2 | + "version": "v2.0", |
| 3 | - "motor_deploy_config": { | 3 | + "motor_deploy_config": { |
| 4 | - "p_instances_num": 1, | 4 | + "p_instances_num": 1, |
| 5 | - "d_instances_num": 1, | 5 | + "d_instances_num": 1, |
| 6 | - "single_p_instance_pod_num": 1, | 6 | + "single_p_instance_pod_num": 1, |
| 7 | - "single_d_instance_pod_num": 1, | 7 | + "single_d_instance_pod_num": 1, |
| 8 | - "p_pod_npu_num": 16, | 8 | + "p_pod_npu_num": 16, |
| 9 | - "d_pod_npu_num": 16, | 9 | + "d_pod_npu_num": 16, |
| 10 | - "image_name": "vllm_motor:motor", | 10 | + "image_name": "<请手动填写镜像名称，例如：mindie-motor-vllm:dev-26.1.0.B081-800I-A3-py311-Ubuntu24.04-lts-aarch64>", |
| 11 | - "job_id": "mindie-motor", | 11 | + "job_id": "mindie-motor", |
| 12 | - "hardware_type": "800I_A3", | 12 | + "hardware_type": "800I_A3", |
| 13 | - "env_path": "./conf/env.json", | 13 | + "weight_mount_path": "<请按实际情况填写模型权重文件的访问路径>" |
| 14 | - "weight_mount_path": "/mnt/share/v4_w8a8", | 14 | + }, |
| 15 | - "deploy_mode": "multi_deployment" | 15 | + "motor_controller_config": {}, |
| 16 | - }, | 16 | + "motor_coordinator_config": {}, |
| 17 | - "motor_controller_config": {}, | 17 | + "motor_engine_prefill_config": { |
| 18 | - "motor_coordinator_config": {}, | 18 | + "engine_type": "vllm", |
| 19 | - "motor_engine_prefill_config": { | 19 | + "engine_config": { |
| 20 | - "engine_type": "vllm", | 20 | + "data_parallel_size": 4, |
| 21 | - "motor_nodemanger_config": {}, | 21 | + "tensor_parallel_size": 4, |
| 22 | - "engine_config": { | 22 | + "pipeline_parallel_size": 1, |
| 23 | - "served_model_name": "dsv4", | 23 | + "data_parallel_rpc_port": 9000, |
| 24 | - "model": "/mnt/share/v4_w8a8", | 24 | + "served_model_name": "dsv4", |
| 25 | - "gpu_memory_utilization": 0.85, | 25 | + "model": "<请按实际情况填写模型权重文件的访问路径>", |
| 26 | - "data_parallel_size": 16, | 26 | + "seed": 1024, |
| 27 | - "tensor_parallel_size": 1, | 27 | + "enable_expert_parallel": true, |
| 28 | - "pipeline_parallel_size": 1, | 28 | + "max-model-len": 1048576, |
| 29 | - "enable_expert_parallel": true, | 29 | + "max-num-batched-tokens": 8192, |
| 30 | - "data_parallel_rpc_port": 9000, | 30 | + "max-num-seqs": 16, |
| 31 | - "seed": 1024, | 31 | + "block-size": 128, |
| 32 | - "max-model-len": 13000, | 32 | + "enforce-eager": true, |
| 33 | - "max-num-batched-tokens": 8192, | 33 | + "no_disable_hybrid_kv_cache_manager": true, |
| 34 | - "max-num-seqs": 4, | 34 | + "trust-remote-code": true, |
| 35 | - "no_disable_hybrid_kv_cache_manager": true, | 35 | + "gpu_memory_utilization": 0.9, |
| 36 | - "safetensors-load-strategy": "prefetch", | 36 | + "quantization": "ascend", |
| 37 | - "trust-remote-code": true, | 37 | + "safetensors-load-strategy": "prefetch", |
| 38 | - "tokenizer-mode": "deepseek_v4", | 38 | + "model-loader-extra-config": { |
| 39 | - "tool-call-parser": "deepseek_v4", | 39 | + "enable_multithread_load": "true", |
| 40 | - "enable-auto-tool-choice": true, | 40 | + "num_threads": 128 |
| 41 | - "reasoning-parser": "deepseek_v4", | 41 | + }, |
| 42 | - "quantization": "ascend", | 42 | + "tokenizer-mode": "deepseek_v4", |
| 43 | - "profiler-config": { | 43 | + "tool-call-parser": "deepseek_v4", |
| 44 | - "profiler": "torch", | 44 | + "enable-auto-tool-choice": true, |
| 45 | - "torch_profiler_dir": "./vllm_profile", | 45 | + "reasoning-parser": "deepseek_v4", |
| 46 | - "torch_profiler_with_stack": false | 46 | + "speculative-config": { |
| 47 | - }, | 47 | + "num_speculative_tokens": 1, |
| 48 | - "enforce-eager": true, | 48 | + "method": "mtp", |
| 49 | - "additional-config": { | 49 | + "enforce_eager": true |
| 50 | - "enable_cpu_binding": true | 50 | + }, |
| 51 | - }, | 51 | + "profiler-config": { |
| 52 | - "kv_transfer_config": { | 52 | + "profiler": "torch", |
| 53 | - "kv_connector": "MooncakeHybridConnector", | 53 | + "torch_profiler_dir": "./vllm_profile", |
| 54 | - "kv_role": "kv_producer", | 54 | + "torch_profiler_with_stack": false |
| 55 | - "kv_port": "30001", | 55 | + }, |
| 56 | - "engine_id": "0" | 56 | + "additional-config": { |
| 57 | - } | 57 | + "enable_cpu_binding": true, |
| 58 | - } | 58 | + "enable_shared_expert_dp": true, |
| 59 | - }, | 59 | + "enable_dsa_cp": true |
| 60 | - "motor_engine_decode_config": { | 60 | + }, |
| 61 | - "engine_type": "vllm", | 61 | + "kv_transfer_config": { |
| 62 | - "motor_nodemanger_config": {}, | 62 | + "kv_connector": "MooncakeHybridConnector", |
| 63 | - "engine_config": { | 63 | + "kv_role": "kv_producer", |
| 64 | - "served_model_name": "auto", | 64 | + "kv_port": "30000", |
| 65 | - "model": "/mnt/share/v4_w8a8", | 65 | + "engine_id": "0" |
| 66 | - "gpu_memory_utilization": 0.88, | 66 | + }, |
| 67 | - "data_parallel_size": 16, | 67 | + "no-enable-prefix-caching": true |
| 68 | - "tensor_parallel_size": 1, | 68 | + } |
| 69 | - "pipeline_parallel_size": 1, | 69 | + }, |
| 70 | - "enable_expert_parallel": true, | 70 | + "motor_engine_decode_config": { |
| 71 | - "data_parallel_rpc_port": 9000, | 71 | + "engine_type": "vllm", |
| 72 | - "seed": 1024, | 72 | + "engine_config": { |
| 73 | - "max-model-len": 13500, | 73 | + "data_parallel_size": 16, |
| 74 | - "max-num-batched-tokens": 120, | 74 | + "tensor_parallel_size": 1, |
| 75 | - "max-num-seqs": 60, | 75 | + "pipeline_parallel_size": 1, |
| 76 | - "async-scheduling": true, | 76 | + "data_parallel_rpc_port": 9000, |
| 77 | - "no_disable_hybrid_kv_cache_manager": true, | 77 | + "served_model_name": "dsv4", |
| 78 | - "safetensors-load-strategy": "prefetch", | 78 | + "model": "<请按实际情况填写模型权重文件的访问路径>", |
| 79 | - "trust-remote-code": true, | 79 | + "seed": 1024, |
| 80 | - "tokenizer-mode": "deepseek_v4", | 80 | + "enable_expert_parallel": true, |
| 81 | - "tool-call-parser": "deepseek_v4", | 81 | + "max-model-len": 1048576, |
| 82 | - "enable-auto-tool-choice": true, | 82 | + "max-num-batched-tokens": 120, |
| 83 | - "reasoning-parser": "deepseek_v4", | 83 | + "max-num-seqs": 60, |
| 84 | - "quantization": "ascend", | 84 | + "async-scheduling": true, |
| 85 | - "profiler-config": { | 85 | + "block-size": 128, |
| 86 | - "profiler": "torch", | 86 | + "no-enable-prefix-caching": true, |
| 87 | - "torch_profiler_dir": "./vllm_profile", | 87 | + "no_disable_hybrid_kv_cache_manager": true, |
| 88 | - "torch_profiler_with_stack": false | 88 | + "safetensors-load-strategy": "prefetch", |
| 89 | - }, | 89 | + "model-loader-extra-config": { |
| 90 | - "speculative-config": {"num_speculative_tokens": 1, "method":"deepseek_mtp"}, | 90 | + "enable_multithread_load": "true", |
| 91 | - "compilation_config": { | 91 | + "num_threads": 128 |
| 92 | - "cudagraph_mode": "FULL_DECODE_ONLY" | 92 | + }, |
| 93 | - }, | 93 | + "trust-remote-code": true, |
| 94 | - "kv_transfer_config": { | 94 | + "tokenizer-mode": "deepseek_v4", |
| 95 | - "kv_connector": "MooncakeHybridConnector", | 95 | + "tool-call-parser": "deepseek_v4", |
| 96 | - "kv_role": "kv_consumer", | 96 | + "enable-auto-tool-choice": true, |
| 97 | - "kv_port": "30001", | 97 | + "reasoning-parser": "deepseek_v4", |
| 98 | - "engine_id": "0" | 98 | + "gpu_memory_utilization": 0.9, |
| 99 | - }, | 99 | + "quantization": "ascend", |
| 100 | - "additional-config": { | 100 | + "compilation-config": { |
| 101 | - "ascend_compilation_config": { | 101 | + "cudagraph_mode": "FULL_DECODE_ONLY" |
| 102 | - "enable_npugraph_ex": true, | 102 | + }, |
| 103 | - "enable_static_kernel": false | 103 | + "speculative-config": { |
| 104 | - }, | 104 | + "num_speculative_tokens": 1, |
| 105 | - "eplb_config": { | 105 | + "method": "mtp", |
| 106 | - "dynamic_eplb":true, | 106 | + "enforce_eager": true |
| 107 | - "expert_heat_collection_interval":600, | 107 | + }, |
| 108 | - "algorithm_execution_interval":50, | 108 | + "profiler-config": { |
| 109 | - "eplb_policy_type":2, | 109 | + "profiler": "torch", |
| 110 | - "num_redundant_experts":16 | 110 | + "torch_profiler_dir": "./vllm_profile", |
| 111 | - }, | 111 | + "torch_profiler_with_stack": false |
| 112 | - "enable_cpu_binding": true, | 112 | + }, |
| 113 | - "multistream_dsa_preprocess": false, | 113 | + "additional-config": { |
| 114 | - "multistream_overlap_shared_expert":false, | 114 | + "enable_cpu_binding": true, |
| 115 | - "recompute_scheduler_enable":true | 115 | + "multistream_overlap_shared_expert": true, |
| 116 | - } | 116 | + "recompute_scheduler_enable": true, |
| 117 | - } | 117 | + "ascend_compilation_config": { |
| 118 | - } | 118 | + "enable_npugraph_ex": true, |
| 119 | -} | 119 | + "enable_static_kernel": false |
| 120 | + } | ||
| 121 | + }, | ||
| 122 | + "kv_transfer_config": { | ||
| 123 | + "kv_connector": "MooncakeHybridConnector", | ||
| 124 | + "kv_role": "kv_consumer", | ||
| 125 | + "kv_port": "30100", | ||
| 126 | + "engine_id": "1" | ||
| 127 | + } | ||
| 128 | + } | ||
| 129 | + } | ||
| 130 | +} | ||
| @@ -0,0 +1,74 @@ | |||||||||||||||||||||||||
| 1 | +nic_name="xxxx" # change to your own nic name | ||||||||||||||||||||||||
 🟡 Medium Priority 脚本 建议：在脚本开头（第 1 行   不准确？ | |||||||||||||||||||||||||
| 2 | +local_ip=xx.xx.xx.2 # change to your own ip | ||||||||||||||||||||||||
| 3 | + | ||||||||||||||||||||||||
| 4 | +export LD_PRELOAD=/usr/lib/aarch64-linux-gnu/libjemalloc.so.2:$LD_PRELOAD | ||||||||||||||||||||||||
| 5 | +export HCCL_OP_EXPANSION_MODE="AIV" | ||||||||||||||||||||||||
| 6 | +export TASK_QUEUE_ENABLE=1 | ||||||||||||||||||||||||
| 7 | +export VLLM_RPC_TIMEOUT=3600000 | ||||||||||||||||||||||||
| 8 | +export VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS=30000 | ||||||||||||||||||||||||
| 9 | +export HCCL_EXEC_TIMEOUT=204 | ||||||||||||||||||||||||
| 10 | +export HCCL_CONNECT_TIMEOUT=1200 | ||||||||||||||||||||||||
| 11 | +export HCCL_IF_IP=$local_ip | ||||||||||||||||||||||||
| 12 | +export GLOO_SOCKET_IFNAME=$nic_name | ||||||||||||||||||||||||
| 13 | +export TP_SOCKET_IFNAME=$nic_name | ||||||||||||||||||||||||
| 14 | +export HCCL_SOCKET_IFNAME=$nic_name | ||||||||||||||||||||||||
| 15 | +export OMP_PROC_BIND=false | ||||||||||||||||||||||||
| 16 | +export OMP_NUM_THREADS=10 | ||||||||||||||||||||||||
| 17 | +export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True | ||||||||||||||||||||||||
| 18 | +export HCCL_BUFFSIZE=1024 | ||||||||||||||||||||||||
| 19 | +export ASCEND_RT_VISIBLE_DEVICES=$1 | ||||||||||||||||||||||||
| 20 | + | ||||||||||||||||||||||||
| 21 | +vllm serve /root/.cache/modelscope/hub/models/vllm-ascend/DeepSeek-V4-Flash-w8a8-mtp \ | ||||||||||||||||||||||||
| 22 | + --host 0.0.0.0 \ | ||||||||||||||||||||||||
| 23 | + --port $2 \ | ||||||||||||||||||||||||
| 24 | + --data-parallel-size $3 \ | ||||||||||||||||||||||||
| 25 | + --data-parallel-rank $4 \ | ||||||||||||||||||||||||
| 26 | + --data-parallel-address $5 \ | ||||||||||||||||||||||||
| 27 | + --data-parallel-rpc-port $6 \ | ||||||||||||||||||||||||
| 28 | + --tensor-parallel-size $7 \ | ||||||||||||||||||||||||
🟡 Medium Priority 脚本中存在多处未加引号的变量展开，可能因单词拆分（word splitting）或 glob 展开导致非预期行为：
用户需要将模板中的占位符（ 建议：为所有变量展开加双引号： 不准确？ 🟡 Medium Priority 建议：对所有 改动建议
不准确？ | |||||||||||||||||||||||||
| 29 | + --enable-expert-parallel \ | ||||||||||||||||||||||||
| 30 | + --seed 1024 \ | ||||||||||||||||||||||||
| 31 | + --served-model-name dsv4 \ | ||||||||||||||||||||||||
| 32 | + --max-model-len 1048576 \ | ||||||||||||||||||||||||
| 33 | + --max-num-batched-tokens 120 \ | ||||||||||||||||||||||||
| 34 | + --max-num-seqs 60 \ | ||||||||||||||||||||||||
| 35 | + --async-scheduling \ | ||||||||||||||||||||||||
| 36 | + --block-size 128 \ | ||||||||||||||||||||||||
| 37 | + --no-disable-hybrid-kv-cache-manager \ | ||||||||||||||||||||||||
| 38 | + --no-enable-prefix-caching \ | ||||||||||||||||||||||||
| 39 | + --safetensors-load-strategy 'prefetch' \ | ||||||||||||||||||||||||
| 40 | + --trust-remote-code \ | ||||||||||||||||||||||||
| 41 | + --tokenizer-mode deepseek_v4 \ | ||||||||||||||||||||||||
| 42 | + --model-loader-extra-config='{"enable_multithread_load": "true", "num_threads": 128}' \ | ||||||||||||||||||||||||
| 43 | + --tool-call-parser deepseek_v4 \ | ||||||||||||||||||||||||
| 44 | + --enable-auto-tool-choice \ | ||||||||||||||||||||||||
| 45 | + --reasoning-parser deepseek_v4 \ | ||||||||||||||||||||||||
| 46 | + --gpu-memory-utilization 0.9 \ | ||||||||||||||||||||||||
| 47 | + --quantization ascend \ | ||||||||||||||||||||||||
| 48 | + --speculative-config '{"num_speculative_tokens": 1,"method": "mtp","enforce_eager": true}' \ | ||||||||||||||||||||||||
| 49 | + --compilation-config '{"cudagraph_mode": "FULL_DECODE_ONLY"}' \ | ||||||||||||||||||||||||
| 50 | + --kv-transfer-config \ | ||||||||||||||||||||||||
| 51 | + '{"kv_connector": "MooncakeHybridConnector", | ||||||||||||||||||||||||
| 52 | + "kv_role": "kv_consumer", | ||||||||||||||||||||||||
| 53 | + "kv_port": "30100", | ||||||||||||||||||||||||
| 54 | + "engine_id": "1", | ||||||||||||||||||||||||
| 55 | + "kv_connector_extra_config": { | ||||||||||||||||||||||||
| 56 | + "prefill": { | ||||||||||||||||||||||||
| 57 | + "dp_size": 4, | ||||||||||||||||||||||||
| 58 | + "tp_size": 4 | ||||||||||||||||||||||||
| 59 | + }, | ||||||||||||||||||||||||
| 60 | + "decode": { | ||||||||||||||||||||||||
| 61 | + "dp_size": 16, | ||||||||||||||||||||||||
| 62 | + "tp_size": 1 | ||||||||||||||||||||||||
| 63 | + } | ||||||||||||||||||||||||
| 64 | + } | ||||||||||||||||||||||||
| 65 | + }' \ | ||||||||||||||||||||||||
| 66 | + --additional-config '{ | ||||||||||||||||||||||||
| 67 | + "ascend_compilation_config":{ | ||||||||||||||||||||||||
| 68 | + "enable_npugraph_ex":true, | ||||||||||||||||||||||||
| 69 | + "enable_static_kernel":false | ||||||||||||||||||||||||
| 70 | + }, | ||||||||||||||||||||||||
| 71 | + "enable_cpu_binding":true, | ||||||||||||||||||||||||
| 72 | + "multistream_overlap_shared_expert":true, | ||||||||||||||||||||||||
| 73 | + "recompute_scheduler_enable":true | ||||||||||||||||||||||||
| 74 | + }' | ||||||||||||||||||||||||
| @@ -0,0 +1,38 @@ | |||
| 1 | +export OMP_PROC_BIND=false | ||
🟡 Medium Priority 与 建议：在第 1 行后添加 不准确？ 🟡 Medium Priority 建议：在脚本第1行之前添加 不准确？ | |||
| 2 | +export OMP_NUM_THREADS=10 | ||
| 3 | +export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True | ||
| 4 | +export LD_PRELOAD=/usr/lib/aarch64-linux-gnu/libjemalloc.so.2:$LD_PRELOAD | ||
| 5 | +export HCCL_BUFFSIZE=1024 | ||
| 6 | +export VLLM_ASCEND_ENABLE_FLASHCOMM1=1 | ||
| 7 | +export TASK_QUEUE_ENABLE=1 | ||
| 8 | +export HCCL_OP_EXPANSION_MODE="AIV" | ||
| 9 | + | ||
| 10 | +vllm serve /root/.cache/modelscope/hub/models/vllm-ascend/DeepSeek-V4-Flash-w8a8-mtp \ | ||
| 11 | + --max-model-len 1048576 \ | ||
| 12 | + --max-num-batched-tokens 10240 \ | ||
| 13 | + --served-model-name dsv4 \ | ||
| 14 | + --gpu-memory-utilization 0.9 \ | ||
| 15 | + --api-server-count 1 \ | ||
| 16 | + --max-num-seqs 64 \ | ||
| 17 | + --data-parallel-size 4 \ | ||
| 18 | + --tensor-parallel-size 4 \ | ||
| 19 | + --enable-expert-parallel \ | ||
| 20 | + --tokenizer-mode deepseek_v4 \ | ||
| 21 | + --tool-call-parser deepseek_v4 \ | ||
| 22 | + --enable-auto-tool-choice \ | ||
| 23 | + --reasoning-parser deepseek_v4 \ | ||
| 24 | + --safetensors-load-strategy 'prefetch' \ | ||
| 25 | + --model-loader-extra-config='{"enable_multithread_load": "true", "num_threads": 128}' \ | ||
| 26 | + --quantization ascend \ | ||
| 27 | + --port 8900 \ | ||
| 28 | + --block-size 128 \ | ||
| 29 | + --speculative-config '{"num_speculative_tokens": 1,"method": "mtp","enforce_eager": true}' \ | ||
| 30 | + --compilation-config '{"cudagraph_mode": "FULL_DECODE_ONLY"}' \ | ||
| 31 | + --async-scheduling \ | ||
| 32 | + --additional-config ' | ||
| 33 | + {"ascend_compilation_config":{ | ||
| 34 | + "enable_npugraph_ex":true, | ||
| 35 | + "enable_static_kernel":false | ||
| 36 | + }, | ||
| 37 | + "enable_cpu_binding": true, | ||
| 38 | + "multistream_overlap_shared_expert":true}' | ||
| @@ -0,0 +1,66 @@ | |||||||||||||||||||||||||
| 1 | +nic_name="xxxx" # change to your own nic name | ||||||||||||||||||||||||
🟡 Medium Priority 建议：在脚本第1行之前添加 不准确？ 🟡 Medium Priority 脚本 建议：在第 1 行后添加 不准确？ | |||||||||||||||||||||||||
| 2 | +local_ip=xx.xx.xx.1 # change to your own ip | ||||||||||||||||||||||||
| 3 | + | ||||||||||||||||||||||||
| 4 | +export HCCL_IF_IP=$local_ip | ||||||||||||||||||||||||
| 5 | +export GLOO_SOCKET_IFNAME=$nic_name | ||||||||||||||||||||||||
| 6 | +export TP_SOCKET_IFNAME=$nic_name | ||||||||||||||||||||||||
| 7 | +export HCCL_SOCKET_IFNAME=$nic_name | ||||||||||||||||||||||||
| 8 | +export VLLM_RPC_TIMEOUT=3600000 | ||||||||||||||||||||||||
| 9 | +export VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS=30000 | ||||||||||||||||||||||||
| 10 | +export HCCL_EXEC_TIMEOUT=204 | ||||||||||||||||||||||||
| 11 | +export HCCL_CONNECT_TIMEOUT=120 | ||||||||||||||||||||||||
| 12 | +export OMP_PROC_BIND=false | ||||||||||||||||||||||||
| 13 | +export OMP_NUM_THREADS=10 | ||||||||||||||||||||||||
| 14 | +export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True | ||||||||||||||||||||||||
| 15 | +export HCCL_BUFFSIZE=2560 | ||||||||||||||||||||||||
| 16 | +export TASK_QUEUE_ENABLE=1 | ||||||||||||||||||||||||
| 17 | +export VLLM_ASCEND_ENABLE_FLASHCOMM1=1 | ||||||||||||||||||||||||
| 18 | +export HCCL_OP_EXPANSION_MODE="AIV" | ||||||||||||||||||||||||
| 19 | +export LD_PRELOAD=/usr/lib/aarch64-linux-gnu/libjemalloc.so.2:$LD_PRELOAD | ||||||||||||||||||||||||
| 20 | +export ASCEND_RT_VISIBLE_DEVICES=$1 | ||||||||||||||||||||||||
| 21 | + | ||||||||||||||||||||||||
| 22 | +vllm serve /root/.cache/modelscope/hub/models/vllm-ascend/DeepSeek-V4-Flash-w8a8-mtp \ | ||||||||||||||||||||||||
| 23 | + --host 0.0.0.0 \ | ||||||||||||||||||||||||
| 24 | + --port $2 \ | ||||||||||||||||||||||||
| 25 | + --data-parallel-size $3 \ | ||||||||||||||||||||||||
| 26 | + --data-parallel-rank $4 \ | ||||||||||||||||||||||||
| 27 | + --data-parallel-address $5 \ | ||||||||||||||||||||||||
| 28 | + --data-parallel-rpc-port $6 \ | ||||||||||||||||||||||||
| 29 | + --tensor-parallel-size $7 \ | ||||||||||||||||||||||||
🟡 Medium Priority 建议：对所有 改动建议
不准确？ 🟡 Medium Priority 脚本中存在多处未加引号的变量展开：
若用户替换占位符后的值含有空格，变量将被错误拆分。 建议：为所有变量展开加双引号： 不准确？ | |||||||||||||||||||||||||
| 30 | + --enable-expert-parallel \ | ||||||||||||||||||||||||
| 31 | + --seed 1024 \ | ||||||||||||||||||||||||
| 32 | + --served-model-name dsv4 \ | ||||||||||||||||||||||||
| 33 | + --max-model-len 1048576 \ | ||||||||||||||||||||||||
| 34 | + --max-num-batched-tokens 8192 \ | ||||||||||||||||||||||||
| 35 | + --max-num-seqs 16 \ | ||||||||||||||||||||||||
| 36 | + --no-disable-hybrid-kv-cache-manager \ | ||||||||||||||||||||||||
| 37 | + --model-loader-extra-config='{"enable_multithread_load": "true", "num_threads": 128}' \ | ||||||||||||||||||||||||
| 38 | + --no-enable-prefix-caching \ | ||||||||||||||||||||||||
| 39 | + --safetensors-load-strategy 'prefetch' \ | ||||||||||||||||||||||||
| 40 | + --speculative-config '{"num_speculative_tokens": 1,"method": "mtp","enforce_eager": true}' \ | ||||||||||||||||||||||||
| 41 | + --trust-remote-code \ | ||||||||||||||||||||||||
| 42 | + --block-size 128 \ | ||||||||||||||||||||||||
| 43 | + --tokenizer-mode deepseek_v4 \ | ||||||||||||||||||||||||
| 44 | + --tool-call-parser deepseek_v4 \ | ||||||||||||||||||||||||
| 45 | + --enable-auto-tool-choice \ | ||||||||||||||||||||||||
| 46 | + --reasoning-parser deepseek_v4 \ | ||||||||||||||||||||||||
| 47 | + --gpu-memory-utilization 0.9 \ | ||||||||||||||||||||||||
| 48 | + --quantization ascend \ | ||||||||||||||||||||||||
| 49 | + --enforce-eager \ | ||||||||||||||||||||||||
| 50 | + --additional-config '{"enable_cpu_binding": true, "enable_shared_expert_dp": true, "enable_dsa_cp": true}' \ | ||||||||||||||||||||||||
| 51 | + --kv-transfer-config \ | ||||||||||||||||||||||||
| 52 | + '{"kv_connector": "MooncakeHybridConnector", | ||||||||||||||||||||||||
| 53 | + "kv_role": "kv_producer", | ||||||||||||||||||||||||
| 54 | + "kv_port": "30000", | ||||||||||||||||||||||||
| 55 | + "engine_id": "0", | ||||||||||||||||||||||||
| 56 | + "kv_connector_extra_config": { | ||||||||||||||||||||||||
| 57 | + "prefill": { | ||||||||||||||||||||||||
| 58 | + "dp_size": 4, | ||||||||||||||||||||||||
| 59 | + "tp_size": 4 | ||||||||||||||||||||||||
| 60 | + }, | ||||||||||||||||||||||||
| 61 | + "decode": { | ||||||||||||||||||||||||
| 62 | + "dp_size": 16, | ||||||||||||||||||||||||
| 63 | + "tp_size": 1 | ||||||||||||||||||||||||
| 64 | + } | ||||||||||||||||||||||||
| 65 | + } | ||||||||||||||||||||||||
| 66 | + }' | ||||||||||||||||||||||||
| @@ -0,0 +1,1517 @@ | |||
| 1 | +# Copyright (c) Huawei Technologies Co., Ltd. 2025-2026. All rights reserved. | ||
| 2 | +# | ||
| 3 | +# 独立脚本：不依赖 motor 包，可直接拷贝到任意目录运行。 | ||
| 4 | +# 部署示例：只需拷贝本文件到 /mnt/share/g00955508/b081_0707/ 即可使用。 | ||
| 5 | +# | ||
| 6 | +# 用法: | ||
| 7 | +# PD 混部: 同目录放置 run_dp_template.sh 后执行 | ||
| 8 | +# python vllm_to_motor.py --deploy-scenario hybrid --hardware-type A3 | ||
| 9 | +# PD 分离: 同目录放置 run_dp_template_prefill.sh / run_dp_template_decode.sh 后执行 | ||
| 10 | +# python vllm_to_motor.py --deploy-scenario separate --hardware-type A3 | ||
| 11 | +# 可选: --weight-path <路径> --image-name <镜像> | ||
| 12 | +# 输出: output_config/user_config.json、output_config/env.json | ||
| 13 | +# | ||
| 14 | +# 并行度：脚本 kv extra 提供 world_size；--hardware-type 仅在 tp 超过单节点上限时重算 engine dp/tp。 | ||
| 15 | +# deploy Pod 切分：按 world_size 打包到节点（pod 数 = world/cards，每 pod 卡数 = cards 或 world）。 | ||
| 16 | +# env.json：仅转换脚本中字面量 export。部署/运行时在 shell 中展开的环境变量不写入 env.json： | ||
| 17 | +# - 显式跳过：HCCL_IF_IP、网卡名、LD_PRELOAD、ASCEND_RT_VISIBLE_DEVICES 等（见 SKIP_ENV_KEYS） | ||
| 18 | +# - 值含 $ 引用：如 LD_PRELOAD=...:$LD_PRELOAD、LD_LIBRARY_PATH=...:$LD_LIBRARY_PATH、$1/$nic_name 等 | ||
| 19 | + | ||
| 20 | +from __future__ import annotations | ||
| 21 | + | ||
| 22 | +import argparse | ||
| 23 | +import json | ||
| 24 | +import re | ||
| 25 | +import shlex | ||
| 26 | +import sys | ||
| 27 | +from pathlib import Path | ||
| 28 | +from typing import Any | ||
| 29 | + | ||
| 30 | +# --------------------------------------------------------------------------- | ||
| 31 | +# converter core (standalone copy, no motor import) | ||
| 32 | +# --------------------------------------------------------------------------- | ||
| 33 | + | ||
| 34 | +SKIP_RUNTIME_KEYS = frozenset( | ||
| 35 | + { | ||
| 36 | + "host", | ||
| 37 | + "port", | ||
| 38 | + "data_parallel_rank", | ||
| 39 | + "data_parallel_address", | ||
| 40 | + } | ||
| 41 | +) | ||
| 42 | + | ||
| 43 | +# 并行度以 kv_connector_extra_config 为准；CLI 中的以下字段忽略 | ||
| 44 | +SKIP_PARALLEL_CLI_KEYS = frozenset( | ||
| 45 | + { | ||
| 46 | + "data_parallel_size", | ||
| 47 | + "tensor_parallel_size", | ||
| 48 | + "data_parallel_rpc_port", | ||
| 49 | + } | ||
| 50 | +) | ||
| 51 | + | ||
| 52 | +DEFAULT_DP_RPC_PORT = 9000 | ||
| 53 | + | ||
| 54 | +# Motor 部署/运行时自行注入，不转换到 env.json | ||
| 55 | +SKIP_ENV_KEYS = frozenset( | ||
| 56 | + { | ||
| 57 | + "HCCL_IF_IP", | ||
| 58 | + "GLOO_SOCKET_IFNAME", | ||
| 59 | + "TP_SOCKET_IFNAME", | ||
| 60 | + "HCCL_SOCKET_IFNAME", | ||
| 61 | + "LD_PRELOAD", | ||
| 62 | + "ASCEND_RT_VISIBLE_DEVICES", | ||
| 63 | + } | ||
| 64 | +) | ||
| 65 | + | ||
| 66 | +# export 值中的 shell 变量引用（$VAR / ${VAR} / $1），表示运行时拼接，不转换 | ||
| 67 | +_SHELL_VAR_REF = re.compile(r"\$(?:\{[^}]+\}|[A-Za-z_][A-Za-z0-9_]*|\d+)") | ||
| 68 | + | ||
| 69 | +DEFAULT_ENV_COMMON = { | ||
| 70 | + "CANN_INSTALL_PATH": "/usr/local/Ascend", | ||
| 71 | +} | ||
| 72 | + | ||
| 73 | +DEFAULT_OUTPUT_DIR = "output_config" | ||
| 74 | +DEFAULT_USER_CONFIG_NAME = "user_config.json" | ||
| 75 | +DEFAULT_ENV_NAME = "env.json" | ||
| 76 | +DEPLOY_SCENARIO_HYBRID = "hybrid" | ||
| 77 | +DEPLOY_SCENARIO_SEPARATE = "separate" | ||
| 78 | + | ||
| 79 | +AUTO_HYBRID_SCRIPT = "run_dp_template_hybrid.sh" | ||
| 80 | +AUTO_PREFILL_SCRIPT = "run_dp_template_prefill.sh" | ||
| 81 | +AUTO_DECODE_SCRIPT = "run_dp_template_decode.sh" | ||
| 82 | +MANUAL_FILL_WEIGHT_MOUNT_PATH = "<请按实际情况填写模型权重文件的访问路径>" | ||
| 83 | +MANUAL_FILL_IMAGE_NAME = "<请按实际情况填写镜像名称>" | ||
| 84 | + | ||
| 85 | +_ANSI_BLUE = "\033[34m" | ||
| 86 | +_ANSI_RESET = "\033[0m" | ||
| 87 | + | ||
| 88 | +ENV_CONFIG_KEY_ORDER = ( | ||
| 89 | + "version", | ||
| 90 | + "motor_common_env", | ||
| 91 | + "motor_controller_env", | ||
| 92 | + "motor_coordinator_env", | ||
| 93 | + "motor_engine_prefill_env", | ||
| 94 | + "motor_engine_decode_env", | ||
| 95 | + "motor_kv_cache_pool_env", | ||
| 96 | +) | ||
| 97 | + | ||
| 98 | +HYBRID_ENV_CONFIG_KEY_ORDER = ( | ||
| 99 | + "version", | ||
| 100 | + "motor_common_env", | ||
| 101 | + "motor_controller_env", | ||
| 102 | + "motor_coordinator_env", | ||
| 103 | + "motor_engine_union_env", | ||
| 104 | + "motor_kv_cache_pool_env", | ||
| 105 | +) | ||
| 106 | + | ||
| 107 | +UNION_ENV_KEY_ORDER = ( | ||
| 108 | + "VLLM_RPC_TIMEOUT", | ||
| 109 | + "VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS", | ||
| 110 | + "HCCL_EXEC_TIMEOUT", | ||
| 111 | + "HCCL_CONNECT_TIMEOUT", | ||
| 112 | + "OMP_PROC_BIND", | ||
| 113 | + "OMP_NUM_THREADS", | ||
| 114 | + "PYTORCH_NPU_ALLOC_CONF", | ||
| 115 | + "HCCL_BUFFSIZE", | ||
| 116 | + "TASK_QUEUE_ENABLE", | ||
| 117 | + "HCCL_OP_EXPANSION_MODE", | ||
| 118 | + "VLLM_ASCEND_ENABLE_FLASHCOMM1", | ||
| 119 | + "VLLM_ASCEND_ENABLE_FUSED_MC2", | ||
| 120 | + "DYNAMIC_EPLB", | ||
| 121 | + "VLLM_TORCH_PROFILER_DIR", | ||
| 122 | + "VLLM_TORCH_PROFILER_WITH_STACK", | ||
| 123 | +) | ||
| 124 | + | ||
| 125 | +PREFILL_ENV_KEY_ORDER = ( | ||
| 126 | + "VLLM_RPC_TIMEOUT", | ||
| 127 | + "VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS", | ||
| 128 | + "HCCL_EXEC_TIMEOUT", | ||
| 129 | + "HCCL_CONNECT_TIMEOUT", | ||
| 130 | + "OMP_PROC_BIND", | ||
| 131 | + "OMP_NUM_THREADS", | ||
| 132 | + "PYTORCH_NPU_ALLOC_CONF", | ||
| 133 | + "HCCL_BUFFSIZE", | ||
| 134 | + "TASK_QUEUE_ENABLE", | ||
| 135 | + "HCCL_OP_EXPANSION_MODE", | ||
| 136 | + "VLLM_ASCEND_ENABLE_FLASHCOMM1", | ||
| 137 | +) | ||
| 138 | + | ||
| 139 | +DECODE_ENV_KEY_ORDER = ( | ||
| 140 | + "HCCL_OP_EXPANSION_MODE", | ||
| 141 | + "TASK_QUEUE_ENABLE", | ||
| 142 | + "VLLM_RPC_TIMEOUT", | ||
| 143 | + "VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS", | ||
| 144 | + "HCCL_EXEC_TIMEOUT", | ||
| 145 | + "HCCL_CONNECT_TIMEOUT", | ||
| 146 | + "VLLM_ASCEND_ENABLE_FUSED_MC2", | ||
| 147 | + "OMP_PROC_BIND", | ||
| 148 | + "OMP_NUM_THREADS", | ||
| 149 | + "PYTORCH_NPU_ALLOC_CONF", | ||
| 150 | + "HCCL_BUFFSIZE", | ||
| 151 | + "VLLM_ASCEND_ENABLE_FLASHCOMM1", | ||
| 152 | + "DYNAMIC_EPLB", | ||
| 153 | + "VLLM_TORCH_PROFILER_DIR", | ||
| 154 | + "VLLM_TORCH_PROFILER_WITH_STACK", | ||
| 155 | +) | ||
| 156 | + | ||
| 157 | +UNDERSCORE_KEYS = frozenset( | ||
| 158 | + { | ||
| 159 | + "data_parallel_size", | ||
| 160 | + "tensor_parallel_size", | ||
| 161 | + "pipeline_parallel_size", | ||
| 162 | + "data_parallel_rpc_port", | ||
| 163 | + "enable_expert_parallel", | ||
| 164 | + "gpu_memory_utilization", | ||
| 165 | + "no_disable_hybrid_kv_cache_manager", | ||
| 166 | + "served_model_name", | ||
| 167 | + "model", | ||
| 168 | + "kv_transfer_config", | ||
| 169 | + } | ||
| 170 | +) | ||
| 171 | + | ||
| 172 | +DEFAULT_PROFILER_CONFIG = { | ||
| 173 | + "profiler": "torch", | ||
| 174 | + "torch_profiler_dir": "./vllm_profile", | ||
| 175 | + "torch_profiler_with_stack": False, | ||
| 176 | +} | ||
| 177 | + | ||
| 178 | +DEFAULT_DEPLOY_CONFIG = { | ||
| 179 | + "p_instances_num": 1, | ||
| 180 | + "d_instances_num": 1, | ||
| 181 | + "single_p_instance_pod_num": 1, | ||
| 182 | + "single_d_instance_pod_num": 1, | ||
| 183 | + "p_pod_npu_num": 16, | ||
| 184 | + "d_pod_npu_num": 16, | ||
| 185 | + "image_name": "", | ||
| 186 | + "job_id": "mindie-motor", | ||
| 187 | + "hardware_type": "800I_A3", | ||
| 188 | + "weight_mount_path": "/mnt/weight/", | ||
| 189 | +} | ||
| 190 | + | ||
| 191 | +# 各硬件平台默认部署参数（其余字段可生成后手工修改） | ||
| 192 | +HARDWARE_PRESETS: dict[str, dict[str, Any]] = { | ||
| 193 | + "800I_A2": { | ||
| 194 | + "hardware_type": "800I_A2", | ||
| 195 | + "cards_per_node": 8, | ||
| 196 | + "image_name": "<请手动填写镜像名称，例如：mindie-motor-vllm:dev-26.1.0.B081-800I-A3-py311-Ubuntu24.04-lts-aarch64>", | ||
| 197 | + "weight_mount_path": "/data01/models/", | ||
| 198 | + "job_id": "mindie-motor", | ||
| 199 | + }, | ||
| 200 | + "800I_A3": { | ||
| 201 | + "hardware_type": "800I_A3", | ||
| 202 | + "cards_per_node": 16, | ||
| 203 | + "image_name": "<请手动填写镜像名称，例如：mindie-motor-vllm:dev-26.1.0.B081-800I-A3-py311-Ubuntu24.04-lts-aarch64>", | ||
| 204 | + "weight_mount_path": "/mnt/weight/", | ||
| 205 | + "job_id": "mindie-motor", | ||
| 206 | + }, | ||
| 207 | +} | ||
| 208 | + | ||
| 209 | +USER_CONFIG_KEY_ORDER = ( | ||
| 210 | + "version", | ||
| 211 | + "motor_deploy_config", | ||
| 212 | + "motor_controller_config", | ||
| 213 | + "motor_coordinator_config", | ||
| 214 | + "motor_engine_prefill_config", | ||
| 215 | + "motor_engine_decode_config", | ||
| 216 | +) | ||
| 217 | + | ||
| 218 | +HYBRID_USER_CONFIG_KEY_ORDER = ( | ||
| 219 | + "version", | ||
| 220 | + "motor_deploy_config", | ||
| 221 | + "motor_controller_config", | ||
| 222 | + "motor_coordinator_config", | ||
| 223 | + "motor_engine_union_config", | ||
| 224 | +) | ||
| 225 | + | ||
| 226 | +DEPLOY_CONFIG_KEY_ORDER = ( | ||
| 227 | + "p_instances_num", | ||
| 228 | + "d_instances_num", | ||
| 229 | + "single_p_instance_pod_num", | ||
| 230 | + "single_d_instance_pod_num", | ||
| 231 | + "p_pod_npu_num", | ||
| 232 | + "d_pod_npu_num", | ||
| 233 | + "image_name", | ||
| 234 | + "job_id", | ||
| 235 | + "hardware_type", | ||
| 236 | + "weight_mount_path", | ||
| 237 | +) | ||
| 238 | + | ||
| 239 | +HYBRID_DEPLOY_CONFIG_KEY_ORDER = ( | ||
| 240 | + "deploy_mode", | ||
| 241 | + "hybrid_instances_num", | ||
| 242 | + "single_hybrid_instance_pod_num", | ||
| 243 | + "hybrid_pod_npu_num", | ||
| 244 | + "image_name", | ||
| 245 | + "job_id", | ||
| 246 | + "hardware_type", | ||
| 247 | + "weight_mount_path", | ||
| 248 | +) | ||
| 249 | + | ||
| 250 | +ENGINE_ROLE_KEY_ORDER = ("engine_type", "engine_config") | ||
| 251 | + | ||
| 252 | +HYBRID_ENGINE_CONFIG_KEY_ORDER = ( | ||
| 253 | + "served_model_name", | ||
| 254 | + "model", | ||
| 255 | + "gpu_memory_utilization", | ||
| 256 | + "data_parallel_size", | ||
| 257 | + "tensor_parallel_size", | ||
| 258 | + "pipeline_parallel_size", | ||
| 259 | + "enable_expert_parallel", | ||
| 260 | + "data_parallel_rpc_port", | ||
| 261 | + "seed", | ||
| 262 | + "max-model-len", | ||
| 263 | + "max-num-batched-tokens", | ||
| 264 | + "max-num-seqs", | ||
| 265 | + "block-size", | ||
| 266 | + "enforce-eager", | ||
| 267 | + "async-scheduling", | ||
| 268 | + "enable-prefix-caching", | ||
| 269 | + "no-enable-prefix-caching", | ||
| 270 | + "trust-remote-code", | ||
| 271 | + "quantization", | ||
| 272 | + "safetensors-load-strategy", | ||
| 273 | + "model-loader-extra-config", | ||
| 274 | + "tokenizer-mode", | ||
| 275 | + "tool-call-parser", | ||
| 276 | + "enable-auto-tool-choice", | ||
| 277 | + "reasoning-parser", | ||
| 278 | + "speculative-config", | ||
| 279 | + "compilation-config", | ||
| 280 | + "profiler-config", | ||
| 281 | + "additional-config", | ||
| 282 | +) | ||
| 283 | + | ||
| 284 | +PREFILL_ENGINE_CONFIG_KEY_ORDER = ( | ||
| 285 | + "data_parallel_size", | ||
| 286 | + "tensor_parallel_size", | ||
| 287 | + "pipeline_parallel_size", | ||
| 288 | + "data_parallel_rpc_port", | ||
| 289 | + "served_model_name", | ||
| 290 | + "model", | ||
| 291 | + "seed", | ||
| 292 | + "enable_expert_parallel", | ||
| 293 | + "max-model-len", | ||
| 294 | + "max-num-batched-tokens", | ||
| 295 | + "max-num-seqs", | ||
| 296 | + "block-size", | ||
| 297 | + "enforce-eager", | ||
| 298 | + "async-scheduling", | ||
| 299 | + "no_disable_hybrid_kv_cache_manager", | ||
| 300 | + "enable-prefix-caching", | ||
| 301 | + "trust-remote-code", | ||
| 302 | + "gpu_memory_utilization", | ||
| 303 | + "quantization", | ||
| 304 | + "safetensors-load-strategy", | ||
| 305 | + "model-loader-extra-config", | ||
| 306 | + "tokenizer-mode", | ||
| 307 | + "tool-call-parser", | ||
| 308 | + "enable-auto-tool-choice", | ||
| 309 | + "reasoning-parser", | ||
| 310 | + "speculative-config", | ||
| 311 | + "profiler-config", | ||
| 312 | + "additional-config", | ||
| 313 | + "kv_transfer_config", | ||
| 314 | +) | ||
| 315 | + | ||
| 316 | +DECODE_ENGINE_CONFIG_KEY_ORDER = ( | ||
| 317 | + "data_parallel_size", | ||
| 318 | + "tensor_parallel_size", | ||
| 319 | + "pipeline_parallel_size", | ||
| 320 | + "data_parallel_rpc_port", | ||
| 321 | + "served_model_name", | ||
| 322 | + "model", | ||
| 323 | + "seed", | ||
| 324 | + "enable_expert_parallel", | ||
| 325 | + "max-model-len", | ||
| 326 | + "max-num-batched-tokens", | ||
| 327 | + "max-num-seqs", | ||
| 328 | + "async-scheduling", | ||
| 329 | + "block-size", | ||
| 330 | + "no-enable-prefix-caching", | ||
| 331 | + "no_disable_hybrid_kv_cache_manager", | ||
| 332 | + "safetensors-load-strategy", | ||
| 333 | + "model-loader-extra-config", | ||
| 334 | + "trust-remote-code", | ||
| 335 | + "tokenizer-mode", | ||
| 336 | + "tool-call-parser", | ||
| 337 | + "enable-auto-tool-choice", | ||
| 338 | + "reasoning-parser", | ||
| 339 | + "gpu_memory_utilization", | ||
| 340 | + "quantization", | ||
| 341 | + "compilation-config", | ||
| 342 | + "speculative-config", | ||
| 343 | + "profiler-config", | ||
| 344 | + "additional-config", | ||
| 345 | + "kv_transfer_config", | ||
| 346 | +) | ||
| 347 | + | ||
| 348 | +NESTED_KEY_ORDERS: dict[str, tuple[str, ...]] = { | ||
| 349 | + "profiler-config": ( | ||
| 350 | + "profiler", | ||
| 351 | + "torch_profiler_dir", | ||
| 352 | + "torch_profiler_with_stack", | ||
| 353 | + ), | ||
| 354 | + "kv_transfer_config": ( | ||
| 355 | + "kv_connector", | ||
| 356 | + "kv_role", | ||
| 357 | + "kv_port", | ||
| 358 | + "engine_id", | ||
| 359 | + ), | ||
| 360 | + "compilation-config": ("cudagraph_mode",), | ||
| 361 | + "ascend_compilation_config": ( | ||
| 362 | + "enable_npugraph_ex", | ||
| 363 | + "enable_static_kernel", | ||
| 364 | + ), | ||
| 365 | +} | ||
| 366 | + | ||
| 367 | + | ||
| 368 | +def reorder_dict( | ||
| 369 | + data: dict[str, Any], | ||
| 370 | + key_order: tuple[str, ...] | list[str], | ||
| 371 | + *, | ||
| 372 | + nested_orders: dict[str, tuple[str, ...]] | None = None, | ||
| 373 | +) -> dict[str, Any]: | ||
| 374 | + """Reorder dict keys; unknown keys are appended at the end.""" | ||
| 375 | + nested_orders = nested_orders or NESTED_KEY_ORDERS | ||
| 376 | + ordered: dict[str, Any] = {} | ||
| 377 | + for key in key_order: | ||
| 378 | + if key not in data: | ||
| 379 | + continue | ||
| 380 | + value = data[key] | ||
| 381 | + if isinstance(value, dict) and key in nested_orders: | ||
| 382 | + value = reorder_dict(value, nested_orders[key], nested_orders=nested_orders) | ||
| 383 | + elif isinstance(value, dict) and key == "additional-config": | ||
| 384 | + value = _reorder_additional_config(value) | ||
| 385 | + ordered[key] = value | ||
| 386 | + for key, value in data.items(): | ||
| 387 | + if key in ordered: | ||
| 388 | + continue | ||
| 389 | + if isinstance(value, dict) and key in nested_orders: | ||
| 390 | + value = reorder_dict(value, nested_orders[key], nested_orders=nested_orders) | ||
| 391 | + elif isinstance(value, dict) and key == "additional-config": | ||
| 392 | + value = _reorder_additional_config(value) | ||
| 393 | + ordered[key] = value | ||
| 394 | + return ordered | ||
| 395 | + | ||
| 396 | + | ||
| 397 | +def _reorder_additional_config(data: dict[str, Any]) -> dict[str, Any]: | ||
| 398 | + preferred = ( | ||
| 399 | + "enable_cpu_binding", | ||
| 400 | + "enable_shared_expert_dp", | ||
| 401 | + "enable_dsa_cp", | ||
| 402 | + "multistream_overlap_shared_expert", | ||
| 403 | + "recompute_scheduler_enable", | ||
| 404 | + "ascend_compilation_config", | ||
| 405 | + ) | ||
| 406 | + ordered = reorder_dict(data, preferred, nested_orders=NESTED_KEY_ORDERS) | ||
| 407 | + return ordered | ||
| 408 | + | ||
| 409 | + | ||
| 410 | +def format_engine_config(engine_config: dict[str, Any], *, role: str) -> dict[str, Any]: | ||
| 411 | + if role == "hybrid": | ||
| 412 | + key_order = HYBRID_ENGINE_CONFIG_KEY_ORDER | ||
| 413 | + elif role == "prefill": | ||
| 414 | + key_order = PREFILL_ENGINE_CONFIG_KEY_ORDER | ||
| 415 | + else: | ||
| 416 | + key_order = DECODE_ENGINE_CONFIG_KEY_ORDER | ||
| 417 | + return reorder_dict(engine_config, key_order) | ||
| 418 | + | ||
| 419 | + | ||
| 420 | +def format_engine_role_config(role_config: dict[str, Any], *, role: str) -> dict[str, Any]: | ||
| 421 | + formatted = reorder_dict(role_config, ENGINE_ROLE_KEY_ORDER) | ||
| 422 | + if "engine_config" in formatted: | ||
| 423 | + formatted["engine_config"] = format_engine_config(formatted["engine_config"], role=role) | ||
| 424 | + return formatted | ||
| 425 | + | ||
| 426 | + | ||
| 427 | +def format_user_config(config: dict[str, Any]) -> dict[str, Any]: | ||
| 428 | + """Apply canonical key order for user_config.json output.""" | ||
| 429 | + if "motor_engine_union_config" in config: | ||
| 430 | + ordered = reorder_dict(config, HYBRID_USER_CONFIG_KEY_ORDER) | ||
| 431 | + if "motor_deploy_config" in ordered: | ||
| 432 | + ordered["motor_deploy_config"] = reorder_dict( | ||
| 433 | + ordered["motor_deploy_config"], | ||
| 434 | + HYBRID_DEPLOY_CONFIG_KEY_ORDER, | ||
| 435 | + ) | ||
| 436 | + if "motor_engine_union_config" in ordered: | ||
| 437 | + ordered["motor_engine_union_config"] = format_engine_role_config( | ||
| 438 | + ordered["motor_engine_union_config"], | ||
| 439 | + role="hybrid", | ||
| 440 | + ) | ||
| 441 | + return ordered | ||
| 442 | + | ||
| 443 | + if "motor_engine_prefill_config" not in config and "version" not in config: | ||
| 444 | + role = "prefill" | ||
| 445 | + kv = config.get("kv_transfer_config") or {} | ||
| 446 | + if isinstance(kv, dict) and str(kv.get("kv_role", "")).lower() == "kv_consumer": | ||
| 447 | + role = "decode" | ||
| 448 | + return format_engine_config(config, role=role) | ||
| 449 | + | ||
| 450 | + ordered = reorder_dict(config, USER_CONFIG_KEY_ORDER) | ||
| 451 | + if "motor_deploy_config" in ordered: | ||
| 452 | + ordered["motor_deploy_config"] = reorder_dict( | ||
| 453 | + ordered["motor_deploy_config"], | ||
| 454 | + DEPLOY_CONFIG_KEY_ORDER, | ||
| 455 | + ) | ||
| 456 | + if "motor_engine_prefill_config" in ordered: | ||
| 457 | + ordered["motor_engine_prefill_config"] = format_engine_role_config( | ||
| 458 | + ordered["motor_engine_prefill_config"], | ||
| 459 | + role="prefill", | ||
| 460 | + ) | ||
| 461 | + if "motor_engine_decode_config" in ordered: | ||
| 462 | + ordered["motor_engine_decode_config"] = format_engine_role_config( | ||
| 463 | + ordered["motor_engine_decode_config"], | ||
| 464 | + role="decode", | ||
| 465 | + ) | ||
| 466 | + return ordered | ||
| 467 | + | ||
| 468 | + | ||
| 469 | +def normalize_hardware_type(value: str) -> str: | ||
| 470 | + """Normalize ``A2`` / ``800I-A2`` -> ``800I_A2``.""" | ||
| 471 | + text = value.strip().upper().replace("-", "_") | ||
| 472 | + if text in {"A2", "800I_A2", "910B"}: | ||
| 473 | + return "800I_A2" | ||
| 474 | + if text in {"A3", "800I_A3"}: | ||
| 475 | + return "800I_A3" | ||
| 476 | + if text in HARDWARE_PRESETS: | ||
| 477 | + return text | ||
| 478 | + raise ValueError(f"unsupported hardware type: {value!r}, use A2 or A3") | ||
| 479 | + | ||
| 480 | + | ||
| 481 | +def _get_kv_config(cli_args: dict[str, Any]) -> dict[str, Any] | None: | ||
| 482 | + for key in ("kv-transfer-config", "kv_transfer_config"): | ||
| 483 | + raw = cli_args.get(key) | ||
| 484 | + if raw is None: | ||
| 485 | + continue | ||
| 486 | + if isinstance(raw, str): | ||
| 487 | + return json.loads(raw) | ||
| 488 | + if isinstance(raw, dict): | ||
| 489 | + return raw | ||
| 490 | + return None | ||
| 491 | + | ||
| 492 | + | ||
| 493 | +def extract_parallel_from_cli_args(cli_args: dict[str, Any]) -> dict[str, int]: | ||
| 494 | + """Read P/D dp/tp from kv_connector_extra_config (preferred).""" | ||
| 495 | + kv_config = _get_kv_config(cli_args) | ||
| 496 | + if not kv_config: | ||
| 497 | + raise ValueError("脚本中缺少 kv-transfer-config / kv_connector_extra_config，无法推断 P/D 并行度。") | ||
| 498 | + | ||
| 499 | + extra = kv_config.get("kv_connector_extra_config") or {} | ||
| 500 | + prefill = extra.get("prefill") or {} | ||
| 501 | + decode = extra.get("decode") or {} | ||
| 502 | + | ||
| 503 | + def _as_int(value: Any) -> int | None: | ||
| 504 | + if value is None or value is True or value is False: | ||
| 505 | + return None | ||
| 506 | + try: | ||
| 507 | + return int(value) | ||
| 508 | + except (TypeError, ValueError): | ||
| 509 | + return None | ||
| 510 | + | ||
| 511 | + prefill_dp = _as_int(prefill.get("dp_size")) | ||
| 512 | + prefill_tp = _as_int(prefill.get("tp_size")) | ||
| 513 | + decode_dp = _as_int(decode.get("dp_size")) | ||
| 514 | + decode_tp = _as_int(decode.get("tp_size")) | ||
| 515 | + | ||
| 516 | + if not all([prefill_dp, prefill_tp, decode_dp, decode_tp]): | ||
| 517 | + raise ValueError("kv_connector_extra_config 中需包含完整的 prefill/decode dp_size 与 tp_size。") | ||
| 518 | + return { | ||
| 519 | + "prefill_dp": prefill_dp, | ||
| 520 | + "prefill_tp": prefill_tp, | ||
| 521 | + "decode_dp": decode_dp, | ||
| 522 | + "decode_tp": decode_tp, | ||
| 523 | + } | ||
| 524 | + | ||
| 525 | + | ||
| 526 | +def _as_positive_int(value: Any, *, field: str) -> int: | ||
| 527 | + if value is None or value is True or value is False: | ||
| 528 | + raise ValueError(f"脚本中缺少有效的 {field}。") | ||
| 529 | + try: | ||
| 530 | + parsed = int(value) | ||
| 531 | + except (TypeError, ValueError) as exc: | ||
| 532 | + raise ValueError(f"脚本中 {field} 无法解析为整数: {value!r}") from exc | ||
| 533 | + if parsed <= 0: | ||
| 534 | + raise ValueError(f"脚本中 {field} 必须为正整数，当前为 {parsed}。") | ||
| 535 | + return parsed | ||
| 536 | + | ||
| 537 | + | ||
| 538 | +def extract_hybrid_parallel_from_cli_args(cli_args: dict[str, Any]) -> tuple[int, int]: | ||
| 539 | + """Read dp/tp from hybrid script CLI flags; omitted values default to 1.""" | ||
| 540 | + dp_raw = cli_args.get("data-parallel-size", cli_args.get("data_parallel_size")) | ||
| 541 | + tp_raw = cli_args.get("tensor-parallel-size", cli_args.get("tensor_parallel_size")) | ||
| 542 | + dp = 1 if dp_raw is None else _as_positive_int(dp_raw, field="--data-parallel-size") | ||
| 543 | + tp = 1 if tp_raw is None else _as_positive_int(tp_raw, field="--tensor-parallel-size") | ||
| 544 | + return dp, tp | ||
| 545 | + | ||
| 546 | + | ||
| 547 | +def remap_hybrid_parallel_for_hardware(dp: int, tp: int, hardware_type: str) -> tuple[int, int]: | ||
| 548 | + """Remap hybrid dp/tp while preserving world_size (same rules as PD remap).""" | ||
| 549 | + cards = cards_per_node(hardware_type) | ||
| 550 | + world = dp * tp | ||
| 551 | + new_tp = min(tp, cards) | ||
| 552 | + if new_tp == tp: | ||
| 553 | + return dp, tp | ||
| 554 | + if world % new_tp != 0: | ||
| 555 | + raise ValueError( | ||
| 556 | + f"hybrid: world_size={world} 在 tp 上限 {cards} 下无法整除为 tp={new_tp}，" | ||
| 557 | + "请检查脚本中的 --data-parallel-size / --tensor-parallel-size。" | ||
| 558 | + ) | ||
| 559 | + return world // new_tp, new_tp | ||
| 560 | + | ||
| 561 | + | ||
| 562 | +def cards_per_node(hardware_type: str) -> int: | ||
| 563 | + return int(HARDWARE_PRESETS[normalize_hardware_type(hardware_type)]["cards_per_node"]) | ||
| 564 | + | ||
| 565 | + | ||
| 566 | +def remap_parallel_for_hardware( | ||
| 567 | + parallel: dict[str, int], | ||
| 568 | + hardware_type: str, | ||
| 569 | +) -> dict[str, int]: | ||
| 570 | + """Remap dp/tp while preserving world_size. | ||
| 571 | + | ||
| 572 | + Script tp is kept when already <= cards-per-node; only values above the | ||
| 573 | + hardware cap (8 for A2, 16 for A3) are reduced and dp is increased accordingly. | ||
| 574 | + """ | ||
| 575 | + cards = cards_per_node(hardware_type) | ||
| 576 | + | ||
| 577 | + def _remap_role(dp: int, tp: int, role: str) -> tuple[int, int]: | ||
| 578 | + world = dp * tp | ||
| 579 | + new_tp = min(tp, cards) | ||
| 580 | + if new_tp == tp: | ||
| 581 | + return dp, tp | ||
| 582 | + if world % new_tp != 0: | ||
| 583 | + raise ValueError( | ||
| 584 | + f"{role}: world_size={world} 在 tp 上限 {cards} 下无法整除为 tp={new_tp}，" | ||
| 585 | + "请检查 kv_connector_extra_config 中的 dp_size/tp_size。" | ||
| 586 | + ) | ||
| 587 | + return world // new_tp, new_tp | ||
| 588 | + | ||
| 589 | + p_dp, p_tp = _remap_role(parallel["prefill_dp"], parallel["prefill_tp"], "prefill") | ||
| 590 | + d_dp, d_tp = _remap_role(parallel["decode_dp"], parallel["decode_tp"], "decode") | ||
| 591 | + return { | ||
| 592 | + "prefill_dp": p_dp, | ||
| 593 | + "prefill_tp": p_tp, | ||
| 594 | + "decode_dp": d_dp, | ||
| 595 | + "decode_tp": d_tp, | ||
| 596 | + } | ||
| 597 | + | ||
| 598 | + | ||
| 599 | +def apply_engine_parallel( | ||
| 600 | + engine_config: dict[str, Any], | ||
| 601 | + *, | ||
| 602 | + role: str, | ||
| 603 | + parallel: dict[str, int], | ||
| 604 | +) -> None: | ||
| 605 | + """Write data_parallel_size / tensor_parallel_size from kv extra; rpc port fixed.""" | ||
| 606 | + if role == "prefill": | ||
| 607 | + engine_config["data_parallel_size"] = parallel["prefill_dp"] | ||
| 608 | + engine_config["tensor_parallel_size"] = parallel["prefill_tp"] | ||
| 609 | + elif role == "decode": | ||
| 610 | + engine_config["data_parallel_size"] = parallel["decode_dp"] | ||
| 611 | + engine_config["tensor_parallel_size"] = parallel["decode_tp"] | ||
| 612 | + else: | ||
| 613 | + raise ValueError(f"unsupported engine role: {role}") | ||
| 614 | + engine_config["data_parallel_rpc_port"] = DEFAULT_DP_RPC_PORT | ||
| 615 | + engine_config["pipeline_parallel_size"] = 1 | ||
| 616 | + | ||
| 617 | + | ||
| 618 | +def _infer_role_from_cli_args(cli_args: dict[str, Any]) -> str: | ||
| 619 | + kv_config = _get_kv_config(cli_args) or {} | ||
| 620 | + role = str(kv_config.get("kv_role", "")).lower() | ||
| 621 | + if role == "kv_producer": | ||
| 622 | + return "prefill" | ||
| 623 | + if role == "kv_consumer": | ||
| 624 | + return "decode" | ||
| 625 | + return "prefill" | ||
| 626 | + | ||
| 627 | + | ||
| 628 | +def _infer_pod_layout(dp: int, tp: int, cards: int, *, role: str) -> tuple[int, int]: | ||
| 629 | + """Pack dp×tp world_size onto nodes with ``cards`` NPUs each.""" | ||
| 630 | + world = dp * tp | ||
| 631 | + if tp > cards: | ||
| 632 | + raise ValueError(f"{role}: tp={tp} 超过单节点上限 {cards} 卡。") | ||
| 633 | + if world <= cards: | ||
| 634 | + return 1, world | ||
| 635 | + if world % cards != 0: | ||
| 636 | + raise ValueError(f"{role}: world_size={world} 无法按每节点 {cards} 卡整除切分 Pod。") | ||
| 637 | + return world // cards, cards | ||
| 638 | + | ||
| 639 | + | ||
| 640 | +def infer_motor_deploy_config( | ||
| 641 | + parallel: dict[str, int], | ||
| 642 | + hardware_type: str, | ||
| 643 | + *, | ||
| 644 | + overrides: dict[str, Any] | None = None, | ||
| 645 | +) -> dict[str, Any]: | ||
| 646 | + """Infer motor_deploy_config by packing remapped dp×tp onto nodes.""" | ||
| 647 | + hw = normalize_hardware_type(hardware_type) | ||
| 648 | + preset = dict(HARDWARE_PRESETS[hw]) | ||
| 649 | + cards = int(preset.pop("cards_per_node")) | ||
| 650 | + | ||
| 651 | + p_pods, p_npu = _infer_pod_layout( | ||
| 652 | + parallel["prefill_dp"], | ||
| 653 | + parallel["prefill_tp"], | ||
| 654 | + cards, | ||
| 655 | + role="prefill", | ||
| 656 | + ) | ||
| 657 | + d_pods, d_npu = _infer_pod_layout( | ||
| 658 | + parallel["decode_dp"], | ||
| 659 | + parallel["decode_tp"], | ||
| 660 | + cards, | ||
| 661 | + role="decode", | ||
| 662 | + ) | ||
| 663 | + | ||
| 664 | + deploy = { | ||
| 665 | + "p_instances_num": 1, | ||
| 666 | + "d_instances_num": 1, | ||
| 667 | + "single_p_instance_pod_num": p_pods, | ||
| 668 | + "single_d_instance_pod_num": d_pods, | ||
| 669 | + "p_pod_npu_num": p_npu, | ||
| 670 | + "d_pod_npu_num": d_npu, | ||
| 671 | + } | ||
| 672 | + | ||
| 673 | + deploy.update(preset) | ||
| 674 | + if overrides: | ||
| 675 | + deploy.update(overrides) | ||
| 676 | + return deploy | ||
| 677 | + | ||
| 678 | + | ||
| 679 | +def infer_hybrid_motor_deploy_config( | ||
| 680 | + dp: int, | ||
| 681 | + tp: int, | ||
| 682 | + hardware_type: str, | ||
| 683 | + *, | ||
| 684 | + overrides: dict[str, Any] | None = None, | ||
| 685 | +) -> dict[str, Any]: | ||
| 686 | + """Infer hybrid motor_deploy_config by packing dp×tp onto nodes.""" | ||
| 687 | + hw = normalize_hardware_type(hardware_type) | ||
| 688 | + preset = dict(HARDWARE_PRESETS[hw]) | ||
| 689 | + cards = int(preset.pop("cards_per_node")) | ||
| 690 | + pods, npu = _infer_pod_layout(dp, tp, cards, role="hybrid") | ||
| 691 | + deploy = { | ||
| 692 | + "deploy_mode": "infer_service_set", | ||
| 693 | + "hybrid_instances_num": 1, | ||
| 694 | + "single_hybrid_instance_pod_num": pods, | ||
| 695 | + "hybrid_pod_npu_num": npu, | ||
| 696 | + } | ||
| 697 | + deploy.update(preset) | ||
| 698 | + if overrides: | ||
| 699 | + deploy.update(overrides) | ||
| 700 | + return deploy | ||
| 701 | + | ||
| 702 | + | ||
| 703 | +def resolve_hybrid_script(directory: Path) -> Path: | ||
| 704 | + """Read hybrid template script from *directory*.""" | ||
| 705 | + script_path = directory / AUTO_HYBRID_SCRIPT | ||
| 706 | + if not script_path.is_file(): | ||
| 707 | + raise FileNotFoundError( | ||
| 708 | + f"在目录 {directory} 未找到: {AUTO_HYBRID_SCRIPT}。\n" | ||
| 709 | + f"PD 混部场景请使用 --deploy-scenario hybrid;" | ||
| 710 | + f"PD 分离场景请使用 --deploy-scenario separate。" | ||
| 711 | + ) | ||
| 712 | + return script_path | ||
| 713 | + | ||
| 714 | + | ||
| 715 | +def resolve_pd_scripts(directory: Path) -> tuple[Path, Path]: | ||
| 716 | + """Read P/D template scripts from *directory*.""" | ||
| 717 | + prefill_path = directory / AUTO_PREFILL_SCRIPT | ||
| 718 | + decode_path = directory / AUTO_DECODE_SCRIPT | ||
| 719 | + missing = [p.name for p in (prefill_path, decode_path) if not p.is_file()] | ||
| 720 | + if missing: | ||
| 721 | + raise FileNotFoundError( | ||
| 722 | + f"在目录 {directory} 未找到: {', '.join(missing)}。\n请放置 {AUTO_PREFILL_SCRIPT} 与 {AUTO_DECODE_SCRIPT}。" | ||
| 723 | + ) | ||
| 724 | + return prefill_path, decode_path | ||
| 725 | + | ||
| 726 | + | ||
| 727 | +def apply_manual_fill_placeholders( | ||
| 728 | + user_config: dict[str, Any], | ||
| 729 | + *, | ||
| 730 | + hybrid: bool = False, | ||
| 731 | +) -> None: | ||
| 732 | + """Mark deploy/model paths that must be edited manually after generation.""" | ||
| 733 | + user_config["motor_deploy_config"]["weight_mount_path"] = MANUAL_FILL_WEIGHT_MOUNT_PATH | ||
| 734 | + if hybrid: | ||
| 735 | + engine_config = user_config["motor_engine_union_config"]["engine_config"] | ||
| 736 | + if "model" in engine_config: | ||
| 737 | + engine_config["model"] = MANUAL_FILL_WEIGHT_MOUNT_PATH | ||
| 738 | + return | ||
| 739 | + for role_key in ("motor_engine_prefill_config", "motor_engine_decode_config"): | ||
| 740 | + engine_config = user_config[role_key]["engine_config"] | ||
| 741 | + if "model" in engine_config: | ||
| 742 | + engine_config["model"] = MANUAL_FILL_WEIGHT_MOUNT_PATH | ||
| 743 | + | ||
| 744 | + | ||
| 745 | +def _try_parse_json_text(raw: str) -> Any | None: | ||
| 746 | + """Parse CLI JSON blob; tolerate leading spaces and shlex escape artifacts.""" | ||
| 747 | + if not isinstance(raw, str): | ||
| 748 | + return None | ||
| 749 | + text = raw.strip() | ||
| 750 | + if not text or text[0] not in "{[": | ||
| 751 | + return None | ||
| 752 | + if '\\"' in text: | ||
| 753 | + text = text.replace('\\"', '"') | ||
| 754 | + try: | ||
| 755 | + return json.loads(text) | ||
| 756 | + except json.JSONDecodeError: | ||
| 757 | + return None | ||
| 758 | + | ||
| 759 | + | ||
| 760 | +def _coerce(raw: str) -> Any: | ||
| 761 | + lowered = raw.lower() | ||
| 762 | + if lowered == "true": | ||
| 763 | + return True | ||
| 764 | + if lowered == "false": | ||
| 765 | + return False | ||
| 766 | + parsed_json = _try_parse_json_text(raw) | ||
| 767 | + if parsed_json is not None: | ||
| 768 | + return parsed_json | ||
| 769 | + for caster in (int, float): | ||
| 770 | + try: | ||
| 771 | + return caster(raw) | ||
| 772 | + except ValueError: | ||
| 773 | + pass | ||
| 774 | + return raw | ||
| 775 | + | ||
| 776 | + | ||
| 777 | +def _set_config(config: dict[str, Any], key: str, value: Any) -> None: | ||
| 778 | + if "." not in key: | ||
| 779 | + config[key] = value | ||
| 780 | + return | ||
| 781 | + node = config | ||
| 782 | + for part in key.split(".")[:-1]: | ||
| 783 | + name = part.replace("-", "_") | ||
| 784 | + if not isinstance(node.get(name), dict): | ||
| 785 | + node[name] = {} | ||
| 786 | + node = node[name] | ||
| 787 | + node[key.rsplit(".", 1)[-1].replace("-", "_")] = value | ||
| 788 | + | ||
| 789 | + | ||
| 790 | +def cli_to_config(tokens: list[str]) -> dict[str, Any]: | ||
| 791 | + config: dict[str, Any] = {} | ||
| 792 | + rest = list(tokens) | ||
| 793 | + while rest: | ||
| 794 | + head, *rest = rest | ||
| 795 | + if not head.startswith("--"): | ||
| 796 | + config.setdefault("model", head) | ||
| 797 | + continue | ||
| 798 | + key = head[2:] | ||
| 799 | + if rest and not rest[0].startswith("--"): | ||
| 800 | + value = rest[0] | ||
| 801 | + _set_config(config, key, _coerce(value) if isinstance(value, str) else value) | ||
| 802 | + rest = rest[1:] | ||
| 803 | + else: | ||
| 804 | + _set_config(config, key, True) | ||
| 805 | + return config | ||
| 806 | + | ||
| 807 | + | ||
| 808 | +def _unquote_shell_value(raw: str) -> str: | ||
| 809 | + if len(raw) >= 2 and raw[0] == raw[-1] and raw[0] in "\"'": | ||
| 810 | + return raw[1:-1] | ||
| 811 | + return raw | ||
| 812 | + | ||
| 813 | + | ||
| 814 | +def _coerce_env_value(raw: str) -> Any: | ||
| 815 | + """Coerce export values for env.json (bools stay lowercase strings).""" | ||
| 816 | + lowered = raw.lower() | ||
| 817 | + if lowered in ("true", "false"): | ||
| 818 | + return lowered | ||
| 819 | + if raw.isdigit() or (raw.startswith("-") and raw[1:].isdigit()): | ||
| 820 | + return int(raw) | ||
| 821 | + return raw | ||
| 822 | + | ||
| 823 | + | ||
| 824 | +def _value_has_shell_var_ref(raw: str) -> bool: | ||
| 825 | + """True when export value depends on shell expansion (lib path append, $1, etc.).""" | ||
| 826 | + return _SHELL_VAR_REF.search(raw) is not None | ||
| 827 | + | ||
| 828 | + | ||
| 829 | +def parse_script_exports(text: str) -> dict[str, Any]: | ||
| 830 | + """Parse ``export KEY=VALUE`` lines before ``vllm serve``. | ||
| 831 | + | ||
| 832 | + Skips ``SKIP_ENV_KEYS`` and any value containing ``$`` shell references. | ||
| 833 | + Export 段不做 shell 变量展开。 | ||
| 834 | + """ | ||
| 835 | + exports: dict[str, Any] = {} | ||
| 836 | + for line in text.splitlines(): | ||
| 837 | + stripped = line.split("#", 1)[0].strip() | ||
| 838 | + if not stripped: | ||
| 839 | + continue | ||
| 840 | + if re.search(r"\bvllm\s+serve\b", stripped): | ||
| 841 | + break | ||
| 842 | + if re.match(r"^(nic_name|local_ip)=", stripped, re.IGNORECASE): | ||
| 843 | + continue | ||
| 844 | + match = re.match(r"^export\s+([A-Za-z_][A-Za-z0-9_]*)=(.*)$", stripped) | ||
| 845 | + if not match: | ||
| 846 | + continue | ||
| 847 | + key, raw_value = match.group(1), match.group(2).strip() | ||
| 848 | + if key in SKIP_ENV_KEYS or _value_has_shell_var_ref(raw_value): | ||
| 849 | + continue | ||
| 850 | + exports[key] = _coerce_env_value(_unquote_shell_value(raw_value)) | ||
| 851 | + return exports | ||
| 852 | + | ||
| 853 | + | ||
| 854 | +def extract_vllm_command_text(text: str) -> str: | ||
| 855 | + """Keep only the ``vllm serve ...`` portion of a shell script.""" | ||
| 856 | + chunks: list[str] = [] | ||
| 857 | + started = False | ||
| 858 | + for line in text.splitlines(): | ||
| 859 | + stripped = line.split("#", 1)[0].strip() | ||
| 860 | + if not stripped: | ||
| 861 | + continue | ||
| 862 | + if re.search(r"\bvllm\s+serve\b", stripped): | ||
| 863 | + started = True | ||
| 864 | + if started: | ||
| 865 | + chunks.append(stripped) | ||
| 866 | + if not chunks: | ||
| 867 | + return text | ||
| 868 | + return "\n".join(chunks) | ||
| 869 | + | ||
| 870 | + | ||
| 871 | +def format_role_env(env: dict[str, Any], *, role: str) -> dict[str, Any]: | ||
| 872 | + if role == "hybrid": | ||
| 873 | + key_order = UNION_ENV_KEY_ORDER | ||
| 874 | + elif role == "prefill": | ||
| 875 | + key_order = PREFILL_ENV_KEY_ORDER | ||
| 876 | + else: | ||
| 877 | + key_order = DECODE_ENV_KEY_ORDER | ||
| 878 | + return reorder_dict(env, key_order) | ||
| 879 | + | ||
| 880 | + | ||
| 881 | +def build_env_config( | ||
| 882 | + prefill_env: dict[str, Any], | ||
| 883 | + decode_env: dict[str, Any], | ||
| 884 | +) -> dict[str, Any]: | ||
| 885 | + return reorder_dict( | ||
| 886 | + { | ||
| 887 | + "version": "2.0.0", | ||
| 888 | + "motor_common_env": dict(DEFAULT_ENV_COMMON), | ||
| 889 | + "motor_controller_env": {}, | ||
| 890 | + "motor_coordinator_env": {}, | ||
| 891 | + "motor_engine_prefill_env": format_role_env(prefill_env, role="prefill"), | ||
| 892 | + "motor_engine_decode_env": format_role_env(decode_env, role="decode"), | ||
| 893 | + "motor_kv_cache_pool_env": {}, | ||
| 894 | + }, | ||
| 895 | + ENV_CONFIG_KEY_ORDER, | ||
| 896 | + ) | ||
| 897 | + | ||
| 898 | + | ||
| 899 | +def build_hybrid_env_config(script_env: dict[str, Any]) -> dict[str, Any]: | ||
| 900 | + return reorder_dict( | ||
| 901 | + { | ||
| 902 | + "version": "2.0.0", | ||
| 903 | + "motor_common_env": dict(DEFAULT_ENV_COMMON), | ||
| 904 | + "motor_controller_env": {}, | ||
| 905 | + "motor_coordinator_env": {}, | ||
| 906 | + "motor_engine_union_env": format_role_env(script_env, role="hybrid"), | ||
| 907 | + "motor_kv_cache_pool_env": {}, | ||
| 908 | + }, | ||
| 909 | + HYBRID_ENV_CONFIG_KEY_ORDER, | ||
| 910 | + ) | ||
| 911 | + | ||
| 912 | + | ||
| 913 | +def _normalize_script_text(text: str) -> str: | ||
| 914 | + text = re.sub(r"\\\s*\r?\n", " ", text) | ||
| 915 | + text = re.sub(r"\s*\\\s*$", "", text, flags=re.MULTILINE) | ||
| 916 | + lines = [] | ||
| 917 | + for line in text.splitlines(): | ||
| 918 | + stripped = line.split("#", 1)[0].strip() | ||
| 919 | + if stripped: | ||
| 920 | + lines.append(stripped) | ||
| 921 | + return " ".join(lines) | ||
| 922 | + | ||
| 923 | + | ||
| 924 | +def _fold_multiline_quoted_json_flags(text: str) -> str: | ||
| 925 | + """Collapse multiline single-quoted JSON values (e.g. --kv-transfer-config) to one line.""" | ||
| 926 | + | ||
| 927 | + def _collapse(match: re.Match[str]) -> str: | ||
| 928 | + flag = match.group(1) | ||
| 929 | + body = match.group(2) | ||
| 930 | + compact = " ".join(body.split()) | ||
| 931 | + return f"{flag} '{compact}'" | ||
| 932 | + | ||
| 933 | + return re.sub( | ||
| 934 | + r"(--[\w-]+)\s*'\s*(\{.*?\})\s*'", | ||
| 935 | + _collapse, | ||
| 936 | + text, | ||
| 937 | + flags=re.DOTALL, | ||
| 938 | + ) | ||
| 939 | + | ||
| 940 | + | ||
| 941 | +def _neutralize_shell_vars_for_parse(text: str) -> str: | ||
| 942 | + """Replace non-numeric shell vars so shlex can tokenize tutorial scripts.""" | ||
| 943 | + | ||
| 944 | + def _replace_braced(match: re.Match[str]) -> str: | ||
| 945 | + name = match.group(1) | ||
| 946 | + if name.isdigit(): | ||
| 947 | + return match.group(0) | ||
| 948 | + if name == "MODEL_PATH": | ||
| 949 | + return "/placeholder/model" | ||
| 950 | + return "placeholder" | ||
| 951 | + | ||
| 952 | + def _replace_bare(match: re.Match[str]) -> str: | ||
| 953 | + name = match.group(1) | ||
| 954 | + if name.isdigit(): | ||
| 955 | + return match.group(0) | ||
| 956 | + return "placeholder" | ||
| 957 | + | ||
| 958 | + text = re.sub(r"\$\{([A-Za-z_][A-Za-z0-9_]*)\}", _replace_braced, text) | ||
| 959 | + text = re.sub(r"\$([A-Za-z_][A-Za-z0-9_]*)", _replace_bare, text) | ||
| 960 | + return text | ||
| 961 | + | ||
| 962 | + | ||
| 963 | +def _prepare_command_text_for_parse(text: str) -> str: | ||
| 964 | + normalized = _normalize_script_text(text) | ||
| 965 | + normalized = _fold_multiline_quoted_json_flags(normalized) | ||
| 966 | + return _neutralize_shell_vars_for_parse(normalized) | ||
| 967 | + | ||
| 968 | + | ||
| 969 | +def _substitute_shell_vars(text: str, variables: dict[str, str] | None = None) -> str: | ||
| 970 | + variables = variables or {} | ||
| 971 | + | ||
| 972 | + def replacer(match: re.Match[str]) -> str: | ||
| 973 | + name = match.group(1) or match.group(2) | ||
| 974 | + return str(variables[name]) if name in variables else match.group(0) | ||
| 975 | + | ||
| 976 | + return re.sub(r"\$(\d+)|\$\{(\d+)\}", replacer, text) | ||
| 977 | + | ||
| 978 | + | ||
| 979 | +def _expand_inline_assignments(tokens: list[str]) -> list[str]: | ||
| 980 | + expanded: list[str] = [] | ||
| 981 | + for token in tokens: | ||
| 982 | + if token.startswith("--") and "=" in token[2:]: | ||
| 983 | + key, value = token[2:].split("=", 1) | ||
| 984 | + expanded.extend([f"--{key}", value]) | ||
| 985 | + else: | ||
| 986 | + expanded.append(token) | ||
| 987 | + return expanded | ||
| 988 | + | ||
| 989 | + | ||
| 990 | +def _substitute_tokens(tokens: list[str], variables: dict[str, str] | None) -> list[str]: | ||
| 991 | + if not variables: | ||
| 992 | + return tokens | ||
| 993 | + joined = _substitute_shell_vars(" ".join(shlex.quote(t) for t in tokens), variables) | ||
| 994 | + return shlex.split(joined, posix=True) | ||
| 995 | + | ||
| 996 | + | ||
| 997 | +def script_to_cli_tokens(text: str, *, variables: dict[str, str] | None = None) -> list[str]: | ||
| 998 | + command_text = extract_vllm_command_text(text) | ||
| 999 | + normalized = _prepare_command_text_for_parse(command_text) | ||
| 1000 | + normalized = _substitute_shell_vars(normalized, variables) | ||
| 1001 | + tokens = shlex.split(normalized, posix=True) | ||
| 1002 | + if tokens and tokens[0] == "vllm": | ||
| 1003 | + tokens = tokens[1:] | ||
| 1004 | + if tokens and tokens[0] == "serve": | ||
| 1005 | + tokens = tokens[1:] | ||
| 1006 | + return _expand_inline_assignments(tokens) | ||
| 1007 | + | ||
| 1008 | + | ||
| 1009 | +def parse_vllm_serve_command(text: str, *, variables: dict[str, str] | None = None) -> dict[str, Any]: | ||
| 1010 | + tokens = script_to_cli_tokens(text, variables=variables) | ||
| 1011 | + if not tokens: | ||
| 1012 | + raise ValueError("empty vLLM command") | ||
| 1013 | + return cli_to_config(tokens) | ||
| 1014 | + | ||
| 1015 | + | ||
| 1016 | +def _motor_config_key(cli_key: str) -> str: | ||
| 1017 | + underscored = cli_key.replace("-", "_") | ||
| 1018 | + return underscored if underscored in UNDERSCORE_KEYS else cli_key | ||
| 1019 | + | ||
| 1020 | + | ||
| 1021 | +def _should_skip_key(cli_key: str, *, include_parallel: bool = False) -> bool: | ||
| 1022 | + normalized = cli_key.replace("-", "_") | ||
| 1023 | + if normalized in SKIP_RUNTIME_KEYS: | ||
| 1024 | + return True | ||
| 1025 | + if not include_parallel and normalized in SKIP_PARALLEL_CLI_KEYS: | ||
| 1026 | + return True | ||
| 1027 | + return False | ||
| 1028 | + | ||
| 1029 | + | ||
| 1030 | +def _convert_kv_transfer_config(raw_value: Any) -> dict[str, Any]: | ||
| 1031 | + if isinstance(raw_value, str): | ||
| 1032 | + kv_config = json.loads(raw_value) | ||
| 1033 | + elif isinstance(raw_value, dict): | ||
| 1034 | + kv_config = dict(raw_value) | ||
| 1035 | + else: | ||
| 1036 | + raise ValueError("kv_transfer_config must be a JSON object") | ||
| 1037 | + motor_kv = { | ||
| 1038 | + "kv_connector": kv_config.get("kv_connector"), | ||
| 1039 | + "kv_role": kv_config.get("kv_role"), | ||
| 1040 | + "kv_port": str(kv_config.get("kv_port", "")), | ||
| 1041 | + "engine_id": str(kv_config.get("engine_id", "0")), | ||
| 1042 | + } | ||
| 1043 | + return {k: v for k, v in motor_kv.items() if v is not None} | ||
| 1044 | + | ||
| 1045 | + | ||
| 1046 | +_JSON_CONFIG_SUFFIXES = ("-config", "_config") | ||
| 1047 | + | ||
| 1048 | + | ||
| 1049 | +def _normalize_json_config_fields(engine_config: dict[str, Any]) -> None: | ||
| 1050 | + """Coerce stringified JSON blobs (e.g. multiline --additional-config) to objects.""" | ||
| 1051 | + for key, value in list(engine_config.items()): | ||
| 1052 | + if not isinstance(value, str): | ||
| 1053 | + continue | ||
| 1054 | + if not any(key.endswith(suffix) for suffix in _JSON_CONFIG_SUFFIXES): | ||
| 1055 | + continue | ||
| 1056 | + parsed = _try_parse_json_text(value) | ||
| 1057 | + if parsed is not None: | ||
| 1058 | + engine_config[key] = parsed | ||
| 1059 | + | ||
| 1060 | + | ||
| 1061 | +def cli_args_to_engine_config( | ||
| 1062 | + cli_args: dict[str, Any], | ||
| 1063 | + *, | ||
| 1064 | + weight_mount_path: str | None = None, | ||
| 1065 | + overrides: dict[str, Any] | None = None, | ||
| 1066 | + add_profiler_config: bool = True, | ||
| 1067 | + strip_kv_extra_config: bool = True, | ||
| 1068 | + role: str | None = None, | ||
| 1069 | + parallel: dict[str, int] | None = None, | ||
| 1070 | + include_parallel_cli: bool = False, | ||
| 1071 | + skip_kv_transfer: bool = False, | ||
| 1072 | +) -> dict[str, Any]: | ||
| 1073 | + engine_config: dict[str, Any] = {} | ||
| 1074 | + for cli_key, value in cli_args.items(): | ||
| 1075 | + if _should_skip_key(cli_key, include_parallel=include_parallel_cli): | ||
| 1076 | + continue | ||
| 1077 | + config_key = _motor_config_key(cli_key) | ||
| 1078 | + if config_key == "kv_transfer_config": | ||
| 1079 | + if skip_kv_transfer: | ||
| 1080 | + continue | ||
| 1081 | + if strip_kv_extra_config: | ||
| 1082 | + engine_config[config_key] = _convert_kv_transfer_config(value) | ||
| 1083 | + if role == "prefill": | ||
| 1084 | + engine_config[config_key]["kv_role"] = "kv_producer" | ||
| 1085 | + elif role == "decode": | ||
| 1086 | + engine_config[config_key]["kv_role"] = "kv_consumer" | ||
| 1087 | + continue | ||
| 1088 | + engine_config[config_key] = value | ||
| 1089 | + | ||
| 1090 | + _normalize_json_config_fields(engine_config) | ||
| 1091 | + | ||
| 1092 | + if weight_mount_path: | ||
| 1093 | + engine_config["model"] = weight_mount_path | ||
| 1094 | + elif "model" in engine_config: | ||
| 1095 | + engine_config["model"] = str(engine_config["model"]) | ||
| 1096 | + | ||
| 1097 | + if parallel is None: | ||
| 1098 | + try: | ||
| 1099 | + parallel = extract_parallel_from_cli_args(cli_args) | ||
| 1100 | + except ValueError: | ||
| 1101 | + parallel = None | ||
| 1102 | + | ||
| 1103 | + if parallel is not None: | ||
| 1104 | + engine_role = role or _infer_role_from_cli_args(cli_args) | ||
| 1105 | + apply_engine_parallel(engine_config, role=engine_role, parallel=parallel) | ||
| 1106 | + else: | ||
| 1107 | + engine_config.setdefault("pipeline_parallel_size", 1) | ||
| 1108 | + engine_config.setdefault("data_parallel_rpc_port", DEFAULT_DP_RPC_PORT) | ||
| 1109 | + | ||
| 1110 | + if add_profiler_config and "profiler-config" not in engine_config: | ||
| 1111 | + engine_config["profiler-config"] = dict(DEFAULT_PROFILER_CONFIG) | ||
| 1112 | + if overrides: | ||
| 1113 | + engine_config.update(overrides) | ||
| 1114 | + return engine_config | ||
| 1115 | + | ||
| 1116 | + | ||
| 1117 | +def cli_tokens_to_engine_config( | ||
| 1118 | + tokens: list[str], | ||
| 1119 | + *, | ||
| 1120 | + variables: dict[str, str] | None = None, | ||
| 1121 | + weight_mount_path: str | None = None, | ||
| 1122 | + overrides: dict[str, Any] | None = None, | ||
| 1123 | +) -> dict[str, Any]: | ||
| 1124 | + if tokens[:1] == ["--"]: | ||
| 1125 | + tokens = tokens[1:] | ||
| 1126 | + tokens = _expand_inline_assignments(_substitute_tokens(tokens, variables)) | ||
| 1127 | + return cli_args_to_engine_config( | ||
| 1128 | + cli_to_config(tokens), | ||
| 1129 | + weight_mount_path=weight_mount_path, | ||
| 1130 | + overrides=overrides, | ||
| 1131 | + ) | ||
| 1132 | + | ||
| 1133 | + | ||
| 1134 | +def build_engine_role_config( | ||
| 1135 | + engine_config: dict[str, Any], | ||
| 1136 | + *, | ||
| 1137 | + engine_type: str = "vllm", | ||
| 1138 | + minimal: bool = False, | ||
| 1139 | +) -> dict[str, Any]: | ||
| 1140 | + role_cfg: dict[str, Any] = { | ||
| 1141 | + "engine_type": engine_type, | ||
| 1142 | + "engine_config": engine_config, | ||
| 1143 | + } | ||
| 1144 | + if not minimal: | ||
| 1145 | + role_cfg["motor_nodemanger_config"] = {} | ||
| 1146 | + return role_cfg | ||
| 1147 | + | ||
| 1148 | + | ||
| 1149 | +def _coerce_deploy_value(value: str) -> Any: | ||
| 1150 | + if value.isdigit(): | ||
| 1151 | + return int(value) | ||
| 1152 | + try: | ||
| 1153 | + if "." in value: | ||
| 1154 | + return float(value) | ||
| 1155 | + except ValueError: | ||
| 1156 | + pass | ||
| 1157 | + if value.lower() in {"true", "false"}: | ||
| 1158 | + return value.lower() == "true" | ||
| 1159 | + return value | ||
| 1160 | + | ||
| 1161 | + | ||
| 1162 | +def build_user_config( | ||
| 1163 | + prefill_engine_config: dict[str, Any], | ||
| 1164 | + decode_engine_config: dict[str, Any], | ||
| 1165 | + *, | ||
| 1166 | + deploy_config: dict[str, Any] | None = None, | ||
| 1167 | + minimal_template: bool = False, | ||
| 1168 | +) -> dict[str, Any]: | ||
| 1169 | + deploy = dict(DEFAULT_DEPLOY_CONFIG) | ||
| 1170 | + if deploy_config: | ||
| 1171 | + for key, value in deploy_config.items(): | ||
| 1172 | + deploy[key] = _coerce_deploy_value(value) if isinstance(value, str) else value | ||
| 1173 | + model_path = ( | ||
| 1174 | + deploy.get("weight_mount_path") | ||
| 1175 | + if deploy.get("weight_mount_path") not in (None, "", "/mnt/weight/", "/data01/models/") | ||
| 1176 | + else prefill_engine_config.get("model") | ||
| 1177 | + or decode_engine_config.get("model") | ||
| 1178 | + or deploy.get("weight_mount_path") | ||
| 1179 | + or "/mnt/weight/" | ||
| 1180 | + ) | ||
| 1181 | + deploy["weight_mount_path"] = model_path | ||
| 1182 | + return format_user_config( | ||
| 1183 | + { | ||
| 1184 | + "version": "v2.0", | ||
| 1185 | + "motor_deploy_config": deploy, | ||
| 1186 | + "motor_controller_config": {}, | ||
| 1187 | + "motor_coordinator_config": {}, | ||
| 1188 | + "motor_engine_prefill_config": build_engine_role_config( | ||
| 1189 | + prefill_engine_config, | ||
| 1190 | + minimal=minimal_template, | ||
| 1191 | + ), | ||
| 1192 | + "motor_engine_decode_config": build_engine_role_config( | ||
| 1193 | + decode_engine_config, | ||
| 1194 | + minimal=minimal_template, | ||
| 1195 | + ), | ||
| 1196 | + } | ||
| 1197 | + ) | ||
| 1198 | + | ||
| 1199 | + | ||
| 1200 | +def build_hybrid_user_config( | ||
| 1201 | + engine_config: dict[str, Any], | ||
| 1202 | + *, | ||
| 1203 | + deploy_config: dict[str, Any] | None = None, | ||
| 1204 | + minimal_template: bool = False, | ||
| 1205 | +) -> dict[str, Any]: | ||
| 1206 | + deploy = { | ||
| 1207 | + "deploy_mode": "infer_service_set", | ||
| 1208 | + "hybrid_instances_num": 1, | ||
| 1209 | + "single_hybrid_instance_pod_num": 1, | ||
| 1210 | + "hybrid_pod_npu_num": 1, | ||
| 1211 | + "image_name": "", | ||
| 1212 | + "job_id": "mindie-motor", | ||
| 1213 | + "hardware_type": "800I_A3", | ||
| 1214 | + "weight_mount_path": "/mnt/weight/", | ||
| 1215 | + } | ||
| 1216 | + if deploy_config: | ||
| 1217 | + for key, value in deploy_config.items(): | ||
| 1218 | + deploy[key] = _coerce_deploy_value(value) if isinstance(value, str) else value | ||
| 1219 | + model_path = ( | ||
| 1220 | + deploy.get("weight_mount_path") | ||
| 1221 | + if deploy.get("weight_mount_path") not in (None, "", "/mnt/weight/", "/data01/models/") | ||
| 1222 | + else engine_config.get("model") or deploy.get("weight_mount_path") or "/mnt/weight/" | ||
| 1223 | + ) | ||
| 1224 | + deploy["weight_mount_path"] = model_path | ||
| 1225 | + return format_user_config( | ||
| 1226 | + { | ||
| 1227 | + "version": "v2.0", | ||
| 1228 | + "motor_deploy_config": deploy, | ||
| 1229 | + "motor_controller_config": {}, | ||
| 1230 | + "motor_coordinator_config": { | ||
| 1231 | + "scheduler_config": { | ||
| 1232 | + "deploy_mode": "single_node", | ||
| 1233 | + }, | ||
| 1234 | + }, | ||
| 1235 | + "motor_engine_union_config": build_engine_role_config( | ||
| 1236 | + engine_config, | ||
| 1237 | + minimal=minimal_template, | ||
| 1238 | + ), | ||
| 1239 | + } | ||
| 1240 | + ) | ||
| 1241 | + | ||
| 1242 | + | ||
| 1243 | +def convert_vllm_hybrid_script_to_user_config( | ||
| 1244 | + script: str, | ||
| 1245 | + *, | ||
| 1246 | + variables: dict[str, str] | None = None, | ||
| 1247 | + weight_mount_path: str | None = None, | ||
| 1248 | + hardware_type: str | None = None, | ||
| 1249 | + deploy_config: dict[str, Any] | None = None, | ||
| 1250 | + engine_overrides: dict[str, Any] | None = None, | ||
| 1251 | + minimal_template: bool = True, | ||
| 1252 | +) -> tuple[dict[str, Any], dict[str, Any]]: | ||
| 1253 | + cli = parse_vllm_serve_command(script, variables=variables) | ||
| 1254 | + dp, tp = extract_hybrid_parallel_from_cli_args(cli) | ||
| 1255 | + if hardware_type: | ||
| 1256 | + dp, tp = remap_hybrid_parallel_for_hardware(dp, tp, hardware_type) | ||
| 1257 | + | ||
| 1258 | + env_config = build_hybrid_env_config(parse_script_exports(script)) | ||
| 1259 | + engine_config = cli_args_to_engine_config( | ||
| 1260 | + cli, | ||
| 1261 | + weight_mount_path=weight_mount_path, | ||
| 1262 | + overrides=engine_overrides, | ||
| 1263 | + include_parallel_cli=True, | ||
| 1264 | + skip_kv_transfer=True, | ||
| 1265 | + add_profiler_config=False, | ||
| 1266 | + ) | ||
| 1267 | + engine_config["data_parallel_size"] = dp | ||
| 1268 | + engine_config["tensor_parallel_size"] = tp | ||
| 1269 | + engine_config.setdefault("pipeline_parallel_size", 1) | ||
| 1270 | + engine_config.setdefault("data_parallel_rpc_port", DEFAULT_DP_RPC_PORT) | ||
| 1271 | + | ||
| 1272 | + merged_deploy = dict(deploy_config or {}) | ||
| 1273 | + if weight_mount_path: | ||
| 1274 | + merged_deploy["weight_mount_path"] = weight_mount_path | ||
| 1275 | + if hardware_type: | ||
| 1276 | + merged_deploy = infer_hybrid_motor_deploy_config( | ||
| 1277 | + dp, | ||
| 1278 | + tp, | ||
| 1279 | + hardware_type, | ||
| 1280 | + overrides=merged_deploy or None, | ||
| 1281 | + ) | ||
| 1282 | + elif merged_deploy: | ||
| 1283 | + merged_deploy = { | ||
| 1284 | + "deploy_mode": "infer_service_set", | ||
| 1285 | + "hybrid_instances_num": 1, | ||
| 1286 | + "single_hybrid_instance_pod_num": 1, | ||
| 1287 | + "hybrid_pod_npu_num": dp * tp, | ||
| 1288 | + **merged_deploy, | ||
| 1289 | + } | ||
| 1290 | + | ||
| 1291 | + user_config = build_hybrid_user_config( | ||
| 1292 | + engine_config, | ||
| 1293 | + deploy_config=merged_deploy or None, | ||
| 1294 | + minimal_template=minimal_template, | ||
| 1295 | + ) | ||
| 1296 | + if not weight_mount_path: | ||
| 1297 | + apply_manual_fill_placeholders(user_config, hybrid=True) | ||
| 1298 | + return user_config, env_config | ||
| 1299 | + | ||
| 1300 | + | ||
| 1301 | +def convert_vllm_scripts_to_user_config( | ||
| 1302 | + prefill_script: str, | ||
| 1303 | + decode_script: str, | ||
| 1304 | + *, | ||
| 1305 | + variables: dict[str, str] | None = None, | ||
| 1306 | + weight_mount_path: str | None = None, | ||
| 1307 | + hardware_type: str | None = None, | ||
| 1308 | + deploy_config: dict[str, Any] | None = None, | ||
| 1309 | + prefill_overrides: dict[str, Any] | None = None, | ||
| 1310 | + decode_overrides: dict[str, Any] | None = None, | ||
| 1311 | + minimal_template: bool = True, | ||
| 1312 | +) -> tuple[dict[str, Any], dict[str, Any]]: | ||
| 1313 | + prefill_cli = parse_vllm_serve_command(prefill_script, variables=variables) | ||
| 1314 | + decode_cli = parse_vllm_serve_command(decode_script, variables=variables) | ||
| 1315 | + script_parallel = extract_parallel_from_cli_args(prefill_cli) | ||
| 1316 | + parallel = remap_parallel_for_hardware(script_parallel, hardware_type) if hardware_type else script_parallel | ||
| 1317 | + env_config = build_env_config( | ||
| 1318 | + parse_script_exports(prefill_script), | ||
| 1319 | + parse_script_exports(decode_script), | ||
| 1320 | + ) | ||
| 1321 | + | ||
| 1322 | + prefill_engine = cli_args_to_engine_config( | ||
| 1323 | + prefill_cli, | ||
| 1324 | + weight_mount_path=weight_mount_path, | ||
| 1325 | + overrides=prefill_overrides, | ||
| 1326 | + role="prefill", | ||
| 1327 | + parallel=parallel, | ||
| 1328 | + ) | ||
| 1329 | + decode_engine = cli_args_to_engine_config( | ||
| 1330 | + decode_cli, | ||
| 1331 | + weight_mount_path=weight_mount_path, | ||
| 1332 | + overrides=decode_overrides, | ||
| 1333 | + role="decode", | ||
| 1334 | + parallel=parallel, | ||
| 1335 | + ) | ||
| 1336 | + | ||
| 1337 | + merged_deploy = dict(deploy_config or {}) | ||
| 1338 | + if weight_mount_path: | ||
| 1339 | + merged_deploy["weight_mount_path"] = weight_mount_path | ||
| 1340 | + | ||
| 1341 | + if hardware_type: | ||
| 1342 | + inferred = infer_motor_deploy_config(parallel, hardware_type, overrides=merged_deploy or None) | ||
| 1343 | + merged_deploy = inferred | ||
| 1344 | + elif merged_deploy: | ||
| 1345 | + merged_deploy = {**DEFAULT_DEPLOY_CONFIG, **merged_deploy} | ||
| 1346 | + | ||
| 1347 | + user_config = build_user_config( | ||
| 1348 | + prefill_engine, | ||
| 1349 | + decode_engine, | ||
| 1350 | + deploy_config=merged_deploy or None, | ||
| 1351 | + minimal_template=minimal_template, | ||
| 1352 | + ) | ||
| 1353 | + if not weight_mount_path: | ||
| 1354 | + apply_manual_fill_placeholders(user_config) | ||
| 1355 | + return user_config, env_config | ||
| 1356 | + | ||
| 1357 | + | ||
| 1358 | +# --------------------------------------------------------------------------- | ||
| 1359 | +# CLI | ||
| 1360 | +# --------------------------------------------------------------------------- | ||
| 1361 | + | ||
| 1362 | + | ||
| 1363 | +def _default_output_paths() -> tuple[Path, Path]: | ||
| 1364 | + user_path = Path(DEFAULT_OUTPUT_DIR) / DEFAULT_USER_CONFIG_NAME | ||
| 1365 | + env_path = Path(DEFAULT_OUTPUT_DIR) / DEFAULT_ENV_NAME | ||
| 1366 | + return user_path, env_path | ||
| 1367 | + | ||
| 1368 | + | ||
| 1369 | +def _blue_text(text: str, *, stream: Any = sys.stderr) -> str: | ||
| 1370 | + """Wrap *text* in blue ANSI codes when writing to a TTY.""" | ||
| 1371 | + if hasattr(stream, "isatty") and stream.isatty(): | ||
| 1372 | + return f"{_ANSI_BLUE}{text}{_ANSI_RESET}" | ||
| 1373 | + return text | ||
| 1374 | + | ||
| 1375 | + | ||
| 1376 | +def _print_optional_arg_reminders( | ||
| 1377 | + *, | ||
| 1378 | + weight_path: str | None, | ||
| 1379 | + image_name: str | None, | ||
| 1380 | + deploy_scenario: str, | ||
| 1381 | + hardware_type: str, | ||
| 1382 | +) -> None: | ||
| 1383 | + if weight_path and image_name: | ||
| 1384 | + return | ||
| 1385 | + | ||
| 1386 | + print( | ||
| 1387 | + f"\n{_blue_text('[提示]')} 配置文件生成成功，请基于 user_config.json 内的提示补充两项参数（env.json文件无需修改），完成后可以正常使用。", | ||
| 1388 | + file=sys.stderr, | ||
| 1389 | + ) | ||
| 1390 | + print(file=sys.stderr) | ||
| 1391 | + print( | ||
| 1392 | + f"{_blue_text('[推荐]')} 如果您不希望二次修改 user_config.json，可以执行以下全量生成命令：", | ||
| 1393 | + file=sys.stderr, | ||
| 1394 | + ) | ||
| 1395 | + print(file=sys.stderr) | ||
| 1396 | + cmd = ( | ||
| 1397 | + f"python3 deploy.py --mode general_config --deploy-scenario {deploy_scenario} " | ||
| 1398 | + f"--hardware-type {hardware_type} " | ||
| 1399 | + f"--weight-path <权重路径> --image-name <镜像名称>" | ||
| 1400 | + ) | ||
| 1401 | + print(cmd, file=sys.stderr) | ||
| 1402 | + | ||
| 1403 | + example_weight = "/home/weights/DeepSeek-V4-Flash-w8a8-mtp" | ||
| 1404 | + hw = normalize_hardware_type(hardware_type) | ||
| 1405 | + example_images = { | ||
| 1406 | + "800I_A2": "mindie-motor-vllm:r0.17.0rc1-800I-A2-py311-lts-aarch64", | ||
| 1407 | + "800I_A3": "mindie-motor-vllm:dev-26.1.0.B081-800I-A3-py311-Ubuntu24.04-lts-aarch64", | ||
| 1408 | + } | ||
| 1409 | + example_image = example_images.get(hw, example_images["800I_A3"]) | ||
| 1410 | + example_cmd = ( | ||
| 1411 | + f"python3 deploy.py --mode general_config --deploy-scenario {deploy_scenario} " | ||
| 1412 | + f"--hardware-type {hardware_type} " | ||
| 1413 | + f"--weight-path {example_weight} " | ||
| 1414 | + f"--image-name {example_image}" | ||
| 1415 | + ) | ||
| 1416 | + print(file=sys.stderr) | ||
| 1417 | + print(f"例如：\n{example_cmd}", file=sys.stderr) | ||
| 1418 | + | ||
| 1419 | + | ||
| 1420 | +def main(argv: list[str] | None = None) -> int: | ||
| 1421 | + parser = argparse.ArgumentParser( | ||
| 1422 | + description="从 vLLM 启动脚本生成 Motor 配置。", | ||
| 1423 | + formatter_class=argparse.RawDescriptionHelpFormatter, | ||
| 1424 | + epilog=( | ||
| 1425 | + f"PD 混部输入: {AUTO_HYBRID_SCRIPT} (--deploy-scenario hybrid)\n" | ||
| 1426 | + f"PD 分离输入: {AUTO_PREFILL_SCRIPT}, {AUTO_DECODE_SCRIPT} " | ||
| 1427 | + f"(--deploy-scenario separate)\n" | ||
| 1428 | + f"输出: {DEFAULT_OUTPUT_DIR}/{DEFAULT_USER_CONFIG_NAME}, " | ||
| 1429 | + f"{DEFAULT_OUTPUT_DIR}/{DEFAULT_ENV_NAME}" | ||
| 1430 | + ), | ||
| 1431 | + ) | ||
| 1432 | + parser.add_argument( | ||
| 1433 | + "--deploy-scenario", | ||
| 1434 | + required=True, | ||
| 1435 | + choices=[DEPLOY_SCENARIO_HYBRID, DEPLOY_SCENARIO_SEPARATE], | ||
| 1436 | + help="部署场景: hybrid=PD混部(单脚本); separate=PD分离(prefill+decode双脚本)", | ||
| 1437 | + ) | ||
| 1438 | + parser.add_argument( | ||
| 1439 | + "--hardware-type", | ||
| 1440 | + required=True, | ||
| 1441 | + help="硬件类型: A2 或 A3", | ||
| 1442 | + ) | ||
| 1443 | + parser.add_argument( | ||
| 1444 | + "--weight-path", | ||
| 1445 | + default=None, | ||
| 1446 | + help="模型权重挂载路径（可选，未指定时写入占位说明）", | ||
| 1447 | + ) | ||
| 1448 | + parser.add_argument( | ||
| 1449 | + "--image-name", | ||
| 1450 | + default=None, | ||
| 1451 | + help="容器镜像名称（可选，未指定时写入占位说明）", | ||
| 1452 | + ) | ||
| 1453 | + args = parser.parse_args(argv) | ||
| 1454 | + | ||
| 1455 | + deploy_overrides: dict[str, str] = {} | ||
| 1456 | + if args.image_name: | ||
| 1457 | + deploy_overrides["image_name"] = args.image_name | ||
| 1458 | + | ||
| 1459 | + workdir = Path.cwd() | ||
| 1460 | + user_path, env_path = _default_output_paths() | ||
| 1461 | + try: | ||
| 1462 | + if args.deploy_scenario == DEPLOY_SCENARIO_HYBRID: | ||
| 1463 | + script_path = resolve_hybrid_script(workdir) | ||
| 1464 | + user_config, env_config = convert_vllm_hybrid_script_to_user_config( | ||
| 1465 | + script_path.read_text(encoding="utf-8"), | ||
| 1466 | + hardware_type=args.hardware_type, | ||
| 1467 | + weight_mount_path=args.weight_path, | ||
| 1468 | + deploy_config=deploy_overrides or None, | ||
| 1469 | + ) | ||
| 1470 | + print( | ||
| 1471 | + f"已读取: {script_path.name} (scenario=hybrid, hardware={normalize_hardware_type(args.hardware_type)})", | ||
| 1472 | + file=sys.stderr, | ||
| 1473 | + ) | ||
| 1474 | + else: | ||
| 1475 | + prefill_path, decode_path = resolve_pd_scripts(workdir) | ||
| 1476 | + user_config, env_config = convert_vllm_scripts_to_user_config( | ||
| 1477 | + prefill_path.read_text(encoding="utf-8"), | ||
| 1478 | + decode_path.read_text(encoding="utf-8"), | ||
| 1479 | + hardware_type=args.hardware_type, | ||
| 1480 | + weight_mount_path=args.weight_path, | ||
| 1481 | + deploy_config=deploy_overrides or None, | ||
| 1482 | + ) | ||
| 1483 | + print( | ||
| 1484 | + f"已读取: {prefill_path.name}, {decode_path.name} " | ||
| 1485 | + f"(scenario=separate, hardware={normalize_hardware_type(args.hardware_type)})", | ||
| 1486 | + file=sys.stderr, | ||
| 1487 | + ) | ||
| 1488 | + except FileNotFoundError as exc: | ||
| 1489 | + raise SystemExit(str(exc)) from exc | ||
| 1490 | + | ||
| 1491 | + if not args.image_name: | ||
| 1492 | + user_config["motor_deploy_config"]["image_name"] = MANUAL_FILL_IMAGE_NAME | ||
| 1493 | + | ||
| 1494 | + env_path.parent.mkdir(parents=True, exist_ok=True) | ||
| 1495 | + env_path.write_text( | ||
| 1496 | + json.dumps(env_config, indent=2, ensure_ascii=False) + "\n", | ||
| 1497 | + encoding="utf-8", | ||
| 1498 | + ) | ||
| 1499 | + print(f"已生成: {env_path}", file=sys.stderr) | ||
| 1500 | + | ||
| 1501 | + user_path.parent.mkdir(parents=True, exist_ok=True) | ||
| 1502 | + user_path.write_text( | ||
| 1503 | + json.dumps(format_user_config(user_config), indent=2, ensure_ascii=False) + "\n", | ||
| 1504 | + encoding="utf-8", | ||
| 1505 | + ) | ||
| 1506 | + print(f"已生成: {user_path}") | ||
| 1507 | + _print_optional_arg_reminders( | ||
| 1508 | + weight_path=args.weight_path, | ||
| 1509 | + image_name=args.image_name, | ||
| 1510 | + deploy_scenario=args.deploy_scenario, | ||
| 1511 | + hardware_type=args.hardware_type, | ||
| 1512 | + ) | ||
| 1513 | + return 0 | ||
| 1514 | + | ||
| 1515 | + | ||
| 1516 | +if __name__ == "__main__": | ||
| 1517 | + raise SystemExit(main()) | ||
| @@ -256,8 +256,40 @@ def _start_log_collection(deploy_config): | |||
| 256 | logger.info("Log collection started via show_log.sh") | 256 | logger.info("Log collection started via show_log.sh") |
| 257 | 257 | ||
| 258 | 258 | ||
| 259 | +def handle_general_config(args): | ||
| 260 | + config_tool_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), "config_tool") | ||
| 261 | + script = os.path.join(config_tool_dir, "vllm_to_motor.py") | ||
| 262 | + cmd = [ | ||
| 263 | + sys.executable, | ||
| 264 | + script, | ||
| 265 | + "--deploy-scenario", | ||
| 266 | + args.deploy_scenario, | ||
| 267 | + "--hardware-type", | ||
| 268 | + args.hardware_type, | ||
| 269 | + ] | ||
| 270 | + if args.weight_path: | ||
| 271 | + cmd.extend(["--weight-path", args.weight_path]) | ||
| 272 | + if args.image_name: | ||
| 273 | + cmd.extend(["--image-name", args.image_name]) | ||
| 274 | + subprocess.run(cmd, cwd=config_tool_dir, check=True) | ||
| 275 | + | ||
| 276 | + | ||
| 259 | def parse_arguments(): | 277 | def parse_arguments(): |
| 260 | parser = argparse.ArgumentParser() | 278 | parser = argparse.ArgumentParser() |
| 279 | + parser.add_argument( | ||
| 280 | + "--mode", | ||
| 281 | + choices=["deploy", "general_config"], | ||
| 282 | + default="deploy", | ||
| 283 | + help="deploy: deploy service; general_config: generate user_config.json/env.json from config_tool", | ||
| 284 | + ) | ||
| 285 | + parser.add_argument( | ||
| 286 | + "--deploy-scenario", | ||
| 287 | + choices=["hybrid", "separate"], | ||
| 288 | + help="Required for general_config mode", | ||
| 289 | + ) | ||
| 290 | + parser.add_argument("--hardware-type", type=str, help="Required for general_config mode: A2 or A3") | ||
| 291 | + parser.add_argument("--weight-path", type=str, help="Optional for general_config mode: weight mount path") | ||
| 292 | + parser.add_argument("--image-name", type=str, help="Optional for general_config mode: container image name") | ||
| 261 | parser.add_argument( | 293 | parser.add_argument( |
| 262 | "--config_dir", | 294 | "--config_dir", |
| 263 | "--dir", | 295 | "--dir", |
| @@ -355,6 +387,13 @@ def start_monitoring(user_config): | |||
| 355 | def main(): | 387 | def main(): |
| 356 | args = parse_arguments() | 388 | args = parse_arguments() |
| 357 | 389 | ||
| 390 | + if args.mode == "general_config": | ||
| 391 | + if not args.deploy_scenario or not args.hardware_type: | ||
| 392 | + logger.error("In general_config mode, the --deploy-scenario and --hardware-type parameters are required.") | ||
| 393 | + sys.exit(2) | ||
| 394 | + handle_general_config(args) | ||
| 395 | + return | ||
| 396 | + | ||
| 358 | # No configuration at all → launch TUI directly (undeployed mode) | 397 | # No configuration at all → launch TUI directly (undeployed mode) |
| 359 | no_config = not (args.config_dir or args.user_config_path or args.env_config_path) | 398 | no_config = not (args.config_dir or args.user_config_path or args.env_config_path) |
| 360 | if no_config: | 399 | if no_config: |
| @@ -1,5 +0,0 @@ | |||
| 1 | -# 说明 | ||
| 2 | - | ||
| 3 | -| 文件 | vllm版本 | PyMotor版本 | 权重 | 执行时间 | 说明 | | ||
| 4 | -| --- | --- | --- | --- | --- | --- | | ||
| 5 | -| user_config.json | 0.17.0 | 主线版本 | GLM-5-w8a8 | 2026.3.27 | | | ||
| @@ -1,46 +0,0 @@ | |||
| 1 | -{ | ||
| 2 | - "version": "2.0.0", | ||
| 3 | - "motor_common_env": { | ||
| 4 | - "CANN_INSTALL_PATH": "/usr/local/Ascend", | ||
| 5 | - "MOTOR_LOG_ROOT_PATH": "/root/ascend/log" | ||
| 6 | - }, | ||
| 7 | - "motor_controller_env": {}, | ||
| 8 | - "motor_coordinator_env": { | ||
| 9 | - "OPENAI_STANDARD": "GLM" | ||
| 10 | - }, | ||
| 11 | - "motor_engine_prefill_env": { | ||
| 12 | - "VLLM_ASCEND_ENABLE_MLAPO": 1, | ||
| 13 | - "VLLM_ASCEND_ENABLE_NZ": 1, | ||
| 14 | - "HCCL_OP_EXPANSION_MODE": "AIV", | ||
| 15 | - "OMP_PROC_BIND": "false", | ||
| 16 | - "OMP_NUM_THREADS": 10, | ||
| 17 | - "VLLM_USE_V1": 1, | ||
| 18 | - "HCCL_BUFFSIZE": 200, | ||
| 19 | - "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", | ||
| 20 | - "TASK_QUEUE_ENABLE": 1, | ||
| 21 | - "CPU_AFFINITY_CONF": 1, | ||
| 22 | - "VLLM_ASCEND_ENABLE_FLASHCOMM1": 1, | ||
| 23 | - "ASCEND_AGGREGATE_ENABLE": 1, | ||
| 24 | - "ASCEND_TRANSPORT_PRINT": 1, | ||
| 25 | - "ACL_OP_INIT_MODE": 1, | ||
| 26 | - "VLLM_NIXL_ABORT_REQUEST_TIMEOUT": 300000 | ||
| 27 | - }, | ||
| 28 | - "motor_engine_decode_env": { | ||
| 29 | - "VLLM_ASCEND_ENABLE_MLAPO": 1, | ||
| 30 | - "VLLM_ASCEND_ENABLE_NZ": 1, | ||
| 31 | - "HCCL_OP_EXPANSION_MODE": "AIV", | ||
| 32 | - "OMP_PROC_BIND": "false", | ||
| 33 | - "OMP_NUM_THREADS": 10, | ||
| 34 | - "VLLM_USE_V1": 1, | ||
| 35 | - "HCCL_BUFFSIZE": 200, | ||
| 36 | - "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", | ||
| 37 | - "TASK_QUEUE_ENABLE": 1, | ||
| 38 | - "CPU_AFFINITY_CONF": 1, | ||
| 39 | - "ASCEND_AGGREGATE_ENABLE": 1, | ||
| 40 | - "ASCEND_TRANSPORT_PRINT": 1, | ||
| 41 | - "ACL_OP_INIT_MODE": 1, | ||
| 42 | - "VLLM_NIXL_ABORT_REQUEST_TIMEOUT": 300000 | ||
| 43 | - }, | ||
| 44 | - "motor_kv_cache_pool_env": {}, | ||
| 45 | - "motor_kv_conductor_env": {} | ||
| 46 | -} | ||
| @@ -1,138 +0,0 @@ | |||
| 1 | -{ | ||
| 2 | - "version": "v2.0", | ||
| 3 | - "motor_deploy_config": { | ||
| 4 | - "p_instances_num": 1, | ||
| 5 | - "d_instances_num": 1, | ||
| 6 | - "single_p_instance_pod_num": 4, | ||
| 7 | - "single_d_instance_pod_num": 4, | ||
| 8 | - "p_pod_npu_num": 8, | ||
| 9 | - "d_pod_npu_num": 8, | ||
| 10 | - "image_name": "mindie-motor-vllm:r0.17.0rc1-800I-A2-py311-lts-aarch64", | ||
| 11 | - "job_id": "mindie-motor", | ||
| 12 | - "hardware_type": "800I_A2", | ||
| 13 | - "env_path": "./conf/env.json", | ||
| 14 | - "weight_mount_path": "/data01/models/", | ||
| 15 | - "deploy_mode": "multi_deployment" | ||
| 16 | - }, | ||
| 17 | - "motor_controller_config": {}, | ||
| 18 | - "motor_coordinator_config": {}, | ||
| 19 | - "motor_nodemanger_config": {}, | ||
| 20 | - "motor_engine_prefill_config": { | ||
| 21 | - "engine_type": "vllm", | ||
| 22 | - "engine_config": { | ||
| 23 | - "served_model_name": "GLM-5", | ||
| 24 | - "model": "/data01/models/GLM-5-w4a8-mtp-QuaRot", | ||
| 25 | - "gpu_memory_utilization": 0.95, | ||
| 26 | - "data_parallel_size": 4, | ||
| 27 | - "tensor_parallel_size": 8, | ||
| 28 | - "pipeline_parallel_size": 1, | ||
| 29 | - "data_parallel_rpc_port": 9000, | ||
| 30 | - "enable_expert_parallel": true, | ||
| 31 | - "enable-chunked-prefill": true, | ||
| 32 | - "seed": 1024, | ||
| 33 | - "max-model-len": 65536, | ||
| 34 | - "max-num-batched-tokens": 4096, | ||
| 35 | - "trust-remote-code": true, | ||
| 36 | - "max-num-seqs": 48, | ||
| 37 | - "quantization": "ascend", | ||
| 38 | - "async-scheduling": true, | ||
| 39 | - "enforce-eager": true, | ||
| 40 | - "enable-auto-tool-choice": true, | ||
| 41 | - "tool-call-parser": "glm47", | ||
| 42 | - "reasoning-parser": "glm45", | ||
| 43 | - "speculative-config": { | ||
| 44 | - "num_speculative_tokens": 3, | ||
| 45 | - "method": "deepseek_mtp" | ||
| 46 | - }, | ||
| 47 | - "additional-config": { | ||
| 48 | - "recompute_scheduler_enable": true, | ||
| 49 | - "multistream_overlap_shared_expert": true, | ||
| 50 | - "fuse_qknorm_rope": false, | ||
| 51 | - "fuse_muls_add": true, | ||
| 52 | - "enable_npugraph_ex": true | ||
| 53 | - }, | ||
| 54 | - "kv_transfer_config": { | ||
| 55 | - "kv_connector": "MooncakeConnectorV1", | ||
| 56 | - "kv_buffer_device": "npu", | ||
| 57 | - "kv_role": "kv_producer", | ||
| 58 | - "kv_parallel_size": 1, | ||
| 59 | - "kv_port": "21001", | ||
| 60 | - "engine_id": "0", | ||
| 61 | - "kv_rank": 0, | ||
| 62 | - "kv_connector_extra_config": { | ||
| 63 | - "use_ascend_direct": true | ||
| 64 | - } | ||
| 65 | - } | ||
| 66 | - }, | ||
| 67 | - "health_check_config": { | ||
| 68 | - "enable_virtual_inference": false, | ||
| 69 | - "npu_usage_threshold": 3, | ||
| 70 | - "max_failure_count": 6 | ||
| 71 | - } | ||
| 72 | - }, | ||
| 73 | - "motor_engine_decode_config": { | ||
| 74 | - "engine_type": "vllm", | ||
| 75 | - "engine_config": { | ||
| 76 | - "served_model_name": "GLM-5", | ||
| 77 | - "model": "/data01/models/GLM-5-w4a8-mtp-QuaRot", | ||
| 78 | - "gpu_memory_utilization": 0.95, | ||
| 79 | - "data_parallel_size": 8, | ||
| 80 | - "tensor_parallel_size": 4, | ||
| 81 | - "pipeline_parallel_size": 1, | ||
| 82 | - "data_parallel_rpc_port": 9000, | ||
| 83 | - "enable_expert_parallel": true, | ||
| 84 | - "enable-chunked-prefill": true, | ||
| 85 | - "seed": 1024, | ||
| 86 | - "max-model-len": 65536, | ||
| 87 | - "max-num-batched-tokens": 64, | ||
| 88 | - "trust-remote-code": true, | ||
| 89 | - "max-num-seqs": 48, | ||
| 90 | - "async-scheduling": true, | ||
| 91 | - "quantization": "ascend", | ||
| 92 | - "enable-auto-tool-choice": true, | ||
| 93 | - "tool-call-parser": "glm47", | ||
| 94 | - "reasoning-parser": "glm45", | ||
| 95 | - "compilation_config": { | ||
| 96 | - "cudagraph_capture_sizes": [ | ||
| 97 | - 1, 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 48, 56, 64, 80, 96 | ||
| 98 | - ], | ||
| 99 | - "cudagraph_mode": "FULL_DECODE_ONLY" | ||
| 100 | - }, | ||
| 101 | - "speculative-config": { | ||
| 102 | - "num_speculative_tokens": 3, | ||
| 103 | - "method": "deepseek_mtp" | ||
| 104 | - }, | ||
| 105 | - "additional-config": { | ||
| 106 | - "recompute_scheduler_enable": true, | ||
| 107 | - "multistream_overlap_shared_expert": true, | ||
| 108 | - "fuse_qknorm_rope": false, | ||
| 109 | - "fuse_muls_add": true, | ||
| 110 | - "enable_npugraph_ex": true | ||
| 111 | - }, | ||
| 112 | - "kv_transfer_config": { | ||
| 113 | - "kv_connector": "MooncakeConnectorV1", | ||
| 114 | - "kv_buffer_device": "npu", | ||
| 115 | - "kv_role": "kv_consumer", | ||
| 116 | - "kv_parallel_size": 1, | ||
| 117 | - "kv_port": "21001", | ||
| 118 | - "engine_id": "0", | ||
| 119 | - "kv_rank": 0, | ||
| 120 | - "kv_connector_extra_config": { | ||
| 121 | - "use_ascend_direct": true | ||
| 122 | - } | ||
| 123 | - } | ||
| 124 | - }, | ||
| 125 | - "health_check_config": { | ||
| 126 | - "enable_virtual_inference": false, | ||
| 127 | - "npu_usage_threshold": 3, | ||
| 128 | - "max_failure_count": 6 | ||
| 129 | - } | ||
| 130 | - }, | ||
| 131 | - "kv_cache_pool_config": { | ||
| 132 | - "metadata_server": "P2PHANDSHAKE", | ||
| 133 | - "protocol": "ascend", | ||
| 134 | - "device_name": "", | ||
| 135 | - "alloc_in_same_node": true, | ||
| 136 | - "global_segment_size": "1GB" | ||
| 137 | - } | ||
| 138 | -} | ||
| @@ -0,0 +1,162 @@ | |||
| 1 | +# MindIE Motor配置自动生成指导 | ||
| 2 | + | ||
| 3 | +MindIE Motor的一键部署工具可以实现“将vllm-ascend社区的部署脚本转换为Motor部署配置”，以降低维护成本并保证与下游推理引擎配置一致。 | ||
| 4 | + | ||
| 5 | +--- | ||
| 6 | + | ||
| 7 | +## 目录简介 | ||
| 8 | + | ||
| 9 | +配置生成脚本存放于[examples/deployer/config_tool/](../../../deployer/config_tool/)目录下，各文件功能如下。 | ||
| 10 | + | ||
| 11 | +```bash | ||
| 12 | +examples/deployer/config_tool/ | ||
| 13 | +├── vllm_to_motor.py # 配置转换脚本 | ||
| 14 | +├── run_dp_template_hybrid.sh # 用户粘贴：vLLM-ascend 混部启动脚本 | ||
| 15 | +├── run_dp_template_prefill.sh # 用户粘贴：vLLM-ascend P 实例启动脚本 | ||
| 16 | +├── run_dp_template_decode.sh # 用户粘贴：vLLM-ascend D 实例启动脚本 | ||
| 17 | +└── output_config/ # 生成的Motor配置内容 | ||
| 18 | + ├── user_config.json | ||
| 19 | + └── env.json | ||
| 20 | +``` | ||
| 21 | + | ||
| 22 | +--- | ||
| 23 | + | ||
| 24 | +## 注意事项 | ||
| 25 | + | ||
| 26 | +1. 在执行使用方法的第2步[在vllm-ascend社区查找模型部署脚本]时，请确保部署镜像中的vllm-ascend版本和社区版本一致，**避免新配置应用于旧代码的情况**。 | ||
| 27 | +2. 生成的Motor配置仅给出一种可行的模型切分示例，**用户可根据集群服务器数量，调整服务占用的服务器数量和模型划分策略，调整模型切分策略时关注以下参数即可**。 | ||
| 28 | + | ||
| 29 | + | 配置项 | 取值类型 | 取值范围 | 配置说明 | | ||
| 30 | + | --- | --- | --- | --- | | ||
| 31 | + | p_instances_num | int | ≥1 | Prefill 实例数量。 | | ||
| 32 | + | d_instances_num | int | ≥1| Decode 实例数量。 | | ||
| 33 | + | single_p_instance_pod_num | int | ≥1 | 1 个 P 实例拆成几个 Pod。| | ||
| 34 | + | single_d_instance_pod_num | int | ≥1 | 1 个 D 实例拆成几个 Pod。 | | ||
| 35 | + | p_pod_npu_num | int | ≥1，单 Pod 最大 16 卡 | 每个 P Pod 使用的 NPU 卡数。 | | ||
| 36 | + | d_pod_npu_num | int | ≥1，单 Pod 最大 16 卡 | 每个 D Pod 使用的 NPU 卡数。 | | ||
| 37 | + | data_parallel_size | int | ≥1 | 数据并行（DP）数 | | ||
| 38 | + | tensor_parallel_size | int | ≥1 | 张量并行（TP）数 | | ||
| 39 | + | ||
| 40 | + 1个P实例占用的NPU卡数 = single_p_instance_pod_num（占用几个pod，跨几机） × p_pod_npu_num （每个pod占用的NPU数）= data_parallel_size × tensor_parallel_size | ||
| 41 | + | ||
| 42 | +3. 生成的Motor配置仅支撑基础推理服务成功部署，**Motor特性调整（例如：主备倒换、KV 亲和性调度、服务限流）需要用户手动修改配置**。 | ||
| 43 | + | ||
| 44 | +--- | ||
| 45 | + | ||
| 46 | +## 使用方法 | ||
| 47 | + | ||
| 48 | +1. 进入配置脚本主目录。 | ||
| 49 | + | ||
| 50 | + ```bash | ||
| 51 | + cd examples/deployer/ | ||
| 52 | + ``` | ||
| 53 | + | ||
| 54 | +2. 在vllm-ascend社区查找部署脚本。 | ||
| 55 | + | ||
| 56 | + 进入[vllm-ascend模型部署指导网址](https://github.com/vllm-project/vllm-ascend/tree/main/docs/source/tutorials/models)，基于模型选择对应文档，在文档的`Online Service Deployment`小节（通常为第5小节）找到模型部署脚本（通常命名为`run_dp_template.sh`），重点关注：**pd混部部署脚本**（小标题名称为`Single-Node Online Deployment`）和**PD分离部署脚本**（小标题名称为`Multi-Node PD Separation Deployment`）。 | ||
| 57 | + | ||
| 58 | + **举例**： | ||
| 59 | + | ||
| 60 | + [dsv4 flash部署指导](https://github.com/vllm-project/vllm-ascend/blob/main/docs/source/tutorials/models/DeepSeek-V4-Flash.md#51-single-node-online-deployment)下。 | ||
| 61 | + | ||
| 62 | + - 5.1小节的"A3 series"小标题下，即为**PD混部部署脚本**，内容如下： | ||
| 63 | + | ||
| 64 | + ```bash | ||
| 65 | + export OMP_PROC_BIND=false | ||
| 66 | + export OMP_NUM_THREADS=10 | ||
| 67 | + ... | ||
| 68 | + | ||
| 69 | + vllm serve /root/.cache/modelscope/hub/models/vllm-ascend/DeepSeek-V4-Flash-w8a8-mtp \ | ||
| 70 | + --max-model-len 1048576 \ | ||
| 71 | + --max-num-batched-tokens 10240 \ | ||
| 72 | + ... | ||
| 73 | + ``` | ||
| 74 | + | ||
| 75 | + - 5.2.1小节的"run_dp_template.sh"的脚本内容，即为**PD分离部署脚本**，以P实例脚本为例： | ||
| 76 | + | ||
| 77 | + ```bash | ||
| 78 | + nic_name="xxxx" # change to your own nic name | ||
| 79 | + local_ip=xx.xx.xx.1 # change to your own ip | ||
| 80 | + | ||
| 81 | + export HCCL_IF_IP=$local_ip | ||
| 82 | + export GLOO_SOCKET_IFNAME=$nic_name | ||
| 83 | + ... | ||
| 84 | + | ||
| 85 | + vllm serve /root/.cache/modelscope/hub/models/vllm-ascend/DeepSeek-V4-Flash-w8a8-mtp \ | ||
| 86 | + --host 0.0.0.0 \ | ||
| 87 | + --port $2 \ | ||
| 88 | + ... | ||
| 89 | + ``` | ||
| 90 | + | ||
| 91 | + >[!NOTE]说明 | ||
| 92 | + >一个实例占用多个服务器的场景下，vllm-ascend社区可能为一个实例的部署提供多份脚本（分别对应多台服务器），这些部署脚本的配置没有明显差异，仅需要关注其中一份脚本。 | ||
| 93 | + > | ||
| 94 | + >例如：[qwen3-235B模型部署指导](https://github.com/vllm-project/vllm-ascend/blob/main/docs/source/tutorials/models/Qwen3-235B-A22B.md#52-multi-node-pd-separation-deployment)的5.2小节中，同时存在Decode node 0和Decode node 1，在使用时选取任意一份作为D实例部署脚本即可。 | ||
| 95 | + | ||
| 96 | +3. 拷贝vllm-ascend模型部署脚本至examples/deployer/config_tool/目录。 | ||
| 97 | + | ||
| 98 | + run_dp_template_prefill.sh、run_dp_template_decode.sh和run_dp_template_hybrid.sh文件用于保存vllm-ascnd部署脚本，这些文件均保存于examples/deployer/config_tool/目录下。 | ||
| 99 | + | ||
| 100 | + - **场景一**：通过Motor部署PD分离服务。 | ||
| 101 | + | ||
| 102 | + 无需额外修改，将上述网址中PD分离部署脚本直接拷贝至run_dp_template_prefill.sh(P节点)和run_dp_template_decode.sh(D节点)文件内。 | ||
| 103 | + | ||
| 104 | + - **场景二**：通过Motor部署PD混部服务。 | ||
| 105 | + | ||
| 106 | + 无需额外修改，将上述网址中的PD混部部署脚本直接拷贝至 run_dp_template_hybrid.sh文件内。 | ||
| 107 | + | ||
| 108 | +4. 生成Motor配置。 | ||
| 109 | + | ||
| 110 | + 根据场景，执行以下命令直接生成Motor配置： | ||
| 111 | + | ||
| 112 | + ```bash | ||
| 113 | + # PD分离、Atlas 800I A3 超节点服务器 | ||
| 114 | + python3 deploy.py --mode general_config --deploy-scenario separate --hardware-type A3 | ||
| 115 | + # PD混部、Atlas 800I A3 超节点服务器 | ||
| 116 | + python3 deploy.py --mode general_config --deploy-scenario hybrid --hardware-type A3 | ||
| 117 | + # PD分离、Atlas 800I A2 推理服务器 | ||
| 118 | + python3 deploy.py --mode general_config --deploy-scenario separate --hardware-type A2 | ||
| 119 | + # PD混部、Atlas 800I A2 推理服务器 | ||
| 120 | + python3 deploy.py --mode general_config --deploy-scenario hybrid --hardware-type A2 | ||
| 121 | + ``` | ||
| 122 | + | ||
| 123 | +5. 查看结果并微调。 | ||
| 124 | + | ||
| 125 | + 进入output_config目录，可以查看生成的user_config.json和env.json文件。 | ||
| 126 | + | ||
| 127 | + ```bash | ||
| 128 | + cd examples/deployer/config_tool/output_config && ls | ||
| 129 | + ``` | ||
| 130 | + | ||
| 131 | + user_config.json文件的以下内容需要用户根据实际情况手动填写： | ||
| 132 | + | ||
| 133 | + ```bash | ||
| 134 | + { | ||
| 135 | + "version": "v2.0", | ||
| 136 | + "motor_deploy_config": { | ||
| 137 | + ... | ||
| 138 | + "image_name": "<请手动填写镜像名称>", | ||
| 139 | + ... | ||
| 140 | + "weight_mount_path": "<请按实际情况填写模型权重文件的访问路径>" | ||
| 141 | + }, | ||
| 142 | + ... | ||
| 143 | + "motor_engine_prefill_config": { | ||
| 144 | + ... | ||
| 145 | + "engine_config": { | ||
| 146 | + ... | ||
| 147 | + "model": "<请按实际情况填写模型权重文件的访问路径>", | ||
| 148 | + ... | ||
| 149 | + } | ||
| 150 | + }, | ||
| 151 | + "motor_engine_decode_config": { | ||
| 152 | + ... | ||
| 153 | + "engine_config": { | ||
| 154 | + ... | ||
| 155 | + "model": "<请按实际情况填写模型权重文件的访问路径>", | ||
| 156 | + ... | ||
| 157 | + } | ||
| 158 | + } | ||
| 159 | + } | ||
| 160 | + ``` | ||
| 161 | + | ||
| 162 | + env.json文件无需修改，至此，配置文件生成完成。 | ||
| @@ -1,3 +0,0 @@ | |||
| 1 | -# 说明 | ||
| 2 | - | ||
| 3 | -该目录配置主要针对`800I_A3` | ||
| @@ -1,37 +0,0 @@ | |||
| 1 | -{ | ||
| 2 | - "version": "2.0.0", | ||
| 3 | - "motor_common_env": { | ||
| 4 | - "CANN_INSTALL_PATH": "/usr/local/Ascend", | ||
| 5 | - "MOTOR_LOG_ROOT_PATH": "/root/ascend/log" | ||
| 6 | - }, | ||
| 7 | - "motor_engine_prefill_env": { | ||
| 8 | - "VLLM_RPC_TIMEOUT": 3600000, | ||
| 9 | - "VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS": 30000, | ||
| 10 | - "HCCL_EXEC_TIMEOUT": 360, | ||
| 11 | - "HCCL_CONNECT_TIMEOUT": 600, | ||
| 12 | - "OMP_PROC_BIND": false, | ||
| 13 | - "OMP_NUM_THREADS": 10, | ||
| 14 | - "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", | ||
| 15 | - "VLLM_ASCEND_ENABLE_MLAPO": 1, | ||
| 16 | - "HCCL_BUFFSIZE": 256, | ||
| 17 | - "TASK_QUEUE_ENABLE": 1, | ||
| 18 | - "HCCL_OP_EXPANSION_MODE": "AIV", | ||
| 19 | - "VLLM_USE_V1": 1, | ||
| 20 | - "ASCEND_BUFFER_POOL": "0:0" | ||
| 21 | - }, | ||
| 22 | - "motor_engine_decode_env": { | ||
| 23 | - "VLLM_RPC_TIMEOUT": 3600000, | ||
| 24 | - "VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS": 30000, | ||
| 25 | - "HCCL_EXEC_TIMEOUT": 3600, | ||
| 26 | - "HCCL_CONNECT_TIMEOUT": 3600, | ||
| 27 | - "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", | ||
| 28 | - "VLLM_ASCEND_ENABLE_MLAPO": 1, | ||
| 29 | - "HCCL_BUFFSIZE": 1200, | ||
| 30 | - "TASK_QUEUE_ENABLE": 1, | ||
| 31 | - "HCCL_OP_EXPANSION_MODE": "AIV", | ||
| 32 | - "OMP_PROC_BIND": "false", | ||
| 33 | - "OMP_NUM_THREADS": 10, | ||
| 34 | - "ASCEND_BUFFER_POOL": "0:0", | ||
| 35 | - "VLLM_USE_V1": 1 | ||
| 36 | - } | ||
| 37 | -} | ||
| @@ -1,103 +0,0 @@ | |||
| 1 | -{ | ||
| 2 | - "version": "v2.0", | ||
| 3 | - "motor_deploy_config": { | ||
| 4 | - "p_instances_num": 2, | ||
| 5 | - "d_instances_num": 1, | ||
| 6 | - "single_p_instance_pod_num": 1, | ||
| 7 | - "single_d_instance_pod_num": 2, | ||
| 8 | - "p_pod_npu_num": 16, | ||
| 9 | - "d_pod_npu_num": 16, | ||
| 10 | - "image_name": "mindie-motor-vllm:dev-2.3.RC1.B132-800I-A3-py311-Ubuntu24.04-lts-aarch64", | ||
| 11 | - "job_id": "mindie-motor", | ||
| 12 | - "hardware_type": "800I_A3", | ||
| 13 | - "weight_mount_path": "/mnt/weight/" | ||
| 14 | - }, | ||
| 15 | - "motor_controller_config": {}, | ||
| 16 | - "motor_coordinator_config": {}, | ||
| 17 | - "motor_engine_prefill_config": { | ||
| 18 | - "engine_type": "vllm", | ||
| 19 | - "motor_nodemanger_config": {}, | ||
| 20 | - "engine_config": { | ||
| 21 | - "served_model_name": "dsv3_1", | ||
| 22 | - "model": "/mnt/weights/DeepSeek-V3.1_w8a8mix_mtp/", | ||
| 23 | - "gpu_memory_utilization": 0.9, | ||
| 24 | - "data_parallel_size": 2, | ||
| 25 | - "tensor_parallel_size": 8, | ||
| 26 | - "pipeline_parallel_size": 1, | ||
| 27 | - "data_parallel_rpc_port": 9000, | ||
| 28 | - "enable_expert_parallel": true, | ||
| 29 | - "enforce-eager": true, | ||
| 30 | - "trust-remote-code": true, | ||
| 31 | - "max_model_len": 73000, | ||
| 32 | - "max-num-batched-tokens": 16384, | ||
| 33 | - "max-num-seqs": 16, | ||
| 34 | - "quantization": "ascend", | ||
| 35 | - "seed": 1024, | ||
| 36 | - "distributed-executor-backend": "mp", | ||
| 37 | - "speculative-config": { | ||
| 38 | - "num_speculative_tokens": 1, | ||
| 39 | - "method": "deepseek_mtp" | ||
| 40 | - }, | ||
| 41 | - "additional-config": { | ||
| 42 | - "recompute_scheduler_enable": true | ||
| 43 | - }, | ||
| 44 | - "kv_transfer_config": { | ||
| 45 | - "kv_connector": "MooncakeLayerwiseConnector", | ||
| 46 | - "kv_buffer_device": "npu", | ||
| 47 | - "kv_role": "kv_producer", | ||
| 48 | - "kv_parallel_size": 1, | ||
| 49 | - "kv_port": "30001", | ||
| 50 | - "engine_id": "0", | ||
| 51 | - "kv_rank": 0, | ||
| 52 | - "kv_connector_extra_config": { | ||
| 53 | - "use_ascend_direct": true | ||
| 54 | - } | ||
| 55 | - } | ||
| 56 | - } | ||
| 57 | - }, | ||
| 58 | - "motor_engine_decode_config": { | ||
| 59 | - "engine_type": "vllm", | ||
| 60 | - "motor_nodemanger_config": {}, | ||
| 61 | - "engine_config": { | ||
| 62 | - "served_model_name": "dsv3_1", | ||
| 63 | - "model": "/mnt/weights/DeepSeek-V3.1_w8a8mix_mtp/", | ||
| 64 | - "gpu_memory_utilization": 0.9, | ||
| 65 | - "data_parallel_size": 32, | ||
| 66 | - "tensor_parallel_size": 1, | ||
| 67 | - "pipeline_parallel_size": 1, | ||
| 68 | - "data_parallel_rpc_port": 9000, | ||
| 69 | - "enable_expert_parallel": true, | ||
| 70 | - "trust-remote-code": true, | ||
| 71 | - "max_model_len": 73000, | ||
| 72 | - "max-num-batched-tokens": 256, | ||
| 73 | - "max-num-seqs": 24, | ||
| 74 | - "quantization": "ascend", | ||
| 75 | - "seed": 1024, | ||
| 76 | - "compilation_config": { | ||
| 77 | - "cudagraph_capture_sizes": [4, 8, 16, 32, 48, 64, 80, 96], | ||
| 78 | - "cudagraph_mode": "FULL_DECODE_ONLY" | ||
| 79 | - }, | ||
| 80 | - "distributed-executor-backend": "mp", | ||
| 81 | - "speculative-config": { | ||
| 82 | - "num_speculative_tokens": 3, | ||
| 83 | - "method": "deepseek_mtp" | ||
| 84 | - }, | ||
| 85 | - "additional-config": { | ||
| 86 | - "recompute_scheduler_enable": true, | ||
| 87 | - "lm_head_tensor_parallel_size": 16 | ||
| 88 | - }, | ||
| 89 | - "kv_transfer_config": { | ||
| 90 | - "kv_connector": "MooncakeLayerwiseConnector", | ||
| 91 | - "kv_buffer_device": "npu", | ||
| 92 | - "kv_role": "kv_consumer", | ||
| 93 | - "kv_parallel_size": 1, | ||
| 94 | - "kv_port": "30001", | ||
| 95 | - "engine_id": "0", | ||
| 96 | - "kv_rank": 0, | ||
| 97 | - "kv_connector_extra_config": { | ||
| 98 | - "use_ascend_direct": true | ||
| 99 | - } | ||
| 100 | - } | ||
| 101 | - } | ||
| 102 | - } | ||
| 103 | -} | ||
| @@ -1,5 +0,0 @@ | |||
| 1 | -# 说明 | ||
| 2 | - | ||
| 3 | -| 文件 | vllm版本 | PyMotor版本 | 权重 | 执行时间 | 说明 | | ||
| 4 | -| --- | --- | --- | --- | --- | --- | | ||
| 5 | -| user_config.json | 0.17.0 | 主线版本 | DeepSeek-V3.2-w8a8-mtp-QuaRot | 2026.3.27 | | | ||
| @@ -1,44 +0,0 @@ | |||
| 1 | -{ | ||
| 2 | - "version": "2.0.0", | ||
| 3 | - "motor_common_env": { | ||
| 4 | - "CANN_INSTALL_PATH": "/usr/local/Ascend", | ||
| 5 | - "MOTOR_LOG_ROOT_PATH": "/root/ascend/log" | ||
| 6 | - }, | ||
| 7 | - "motor_controller_env": {}, | ||
| 8 | - "motor_coordinator_env": {}, | ||
| 9 | - "motor_engine_prefill_env": { | ||
| 10 | - "VLLM_ASCEND_ENABLE_MLAPO": 1, | ||
| 11 | - "VLLM_ASCEND_ENABLE_NZ": 1, | ||
| 12 | - "HCCL_OP_EXPANSION_MODE": "AIV", | ||
| 13 | - "OMP_PROC_BIND": "false", | ||
| 14 | - "OMP_NUM_THREADS": 10, | ||
| 15 | - "VLLM_USE_V1": 1, | ||
| 16 | - "HCCL_BUFFSIZE": 200, | ||
| 17 | - "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", | ||
| 18 | - "TASK_QUEUE_ENABLE": 1, | ||
| 19 | - "CPU_AFFINITY_CONF": 1, | ||
| 20 | - "VLLM_ASCEND_ENABLE_FLASHCOMM1": 1, | ||
| 21 | - "ASCEND_AGGREGATE_ENABLE": 1, | ||
| 22 | - "ASCEND_TRANSPORT_PRINT": 1, | ||
| 23 | - "ACL_OP_INIT_MODE": 1, | ||
| 24 | - "VLLM_NIXL_ABORT_REQUEST_TIMEOUT": 300000 | ||
| 25 | - }, | ||
| 26 | - "motor_engine_decode_env": { | ||
| 27 | - "VLLM_ASCEND_ENABLE_MLAPO": 1, | ||
| 28 | - "VLLM_ASCEND_ENABLE_NZ": 1, | ||
| 29 | - "HCCL_OP_EXPANSION_MODE": "AIV", | ||
| 30 | - "OMP_PROC_BIND": "false", | ||
| 31 | - "OMP_NUM_THREADS": 10, | ||
| 32 | - "VLLM_USE_V1": 1, | ||
| 33 | - "HCCL_BUFFSIZE": 200, | ||
| 34 | - "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", | ||
| 35 | - "TASK_QUEUE_ENABLE": 1, | ||
| 36 | - "CPU_AFFINITY_CONF": 1, | ||
| 37 | - "ASCEND_AGGREGATE_ENABLE": 1, | ||
| 38 | - "ASCEND_TRANSPORT_PRINT": 1, | ||
| 39 | - "ACL_OP_INIT_MODE": 1, | ||
| 40 | - "VLLM_NIXL_ABORT_REQUEST_TIMEOUT": 300000 | ||
| 41 | - }, | ||
| 42 | - "motor_kv_cache_pool_env": {}, | ||
| 43 | - "motor_kv_conductor_env": {} | ||
| 44 | -} | ||
| @@ -1,131 +0,0 @@ | |||
| 1 | -{ | ||
| 2 | - "version": "v2.0", | ||
| 3 | - "motor_deploy_config": { | ||
| 4 | - "p_instances_num": 1, | ||
| 5 | - "d_instances_num": 1, | ||
| 6 | - "single_p_instance_pod_num": 4, | ||
| 7 | - "single_d_instance_pod_num": 4, | ||
| 8 | - "p_pod_npu_num": 8, | ||
| 9 | - "d_pod_npu_num": 8, | ||
| 10 | - "image_name": "mindie-motor-vllm:kv_and_tracing-r0.17.0rc1-800I-A2-py311-lts-aarch64-traton", | ||
| 11 | - "job_id": "mindie-motor", | ||
| 12 | - "hardware_type": "800I_A2", | ||
| 13 | - "env_path": "./conf/env.json", | ||
| 14 | - "weight_mount_path": "/data01/models/", | ||
| 15 | - "deploy_mode": "multi_deployment" | ||
| 16 | - }, | ||
| 17 | - "motor_controller_config": {}, | ||
| 18 | - "motor_coordinator_config": {}, | ||
| 19 | - "motor_engine_prefill_config": { | ||
| 20 | - "engine_type": "vllm", | ||
| 21 | - "motor_nodemanger_config": {}, | ||
| 22 | - "engine_config": { | ||
| 23 | - "served_model_name": "DeepSeek-V3.2", | ||
| 24 | - "model": "/data01/models/DeepSeek-V3.2-w8a8-mtp-QuaRot", | ||
| 25 | - "gpu_memory_utilization": 0.95, | ||
| 26 | - "data_parallel_size": 4, | ||
| 27 | - "tensor_parallel_size": 8, | ||
| 28 | - "pipeline_parallel_size": 1, | ||
| 29 | - "data_parallel_rpc_port": 9000, | ||
| 30 | - "enable_expert_parallel": true, | ||
| 31 | - "enable-chunked-prefill": true, | ||
| 32 | - "seed": 1024, | ||
| 33 | - "max-model-len": 65536, | ||
| 34 | - "max-num-batched-tokens": 4096, | ||
| 35 | - "trust-remote-code": true, | ||
| 36 | - "max-num-seqs": 48, | ||
| 37 | - "quantization": "ascend", | ||
| 38 | - "async-scheduling": true, | ||
| 39 | - "enforce-eager": true, | ||
| 40 | - "tokenizer-mode": "deepseek_v32", | ||
| 41 | - "enable-auto-tool-choice": true, | ||
| 42 | - "tool-call-parser": "deepseek_v32", | ||
| 43 | - "reasoning-parser": "deepseek_v3", | ||
| 44 | - "speculative-config": { | ||
| 45 | - "num_speculative_tokens": 3, | ||
| 46 | - "method": "deepseek_mtp" | ||
| 47 | - }, | ||
| 48 | - "additional-config": { | ||
| 49 | - "recompute_scheduler_enable": true, | ||
| 50 | - "multistream_overlap_shared_expert": true, | ||
| 51 | - "fuse_qknorm_rope": false, | ||
| 52 | - "fuse_muls_add": true, | ||
| 53 | - "enable_npugraph_ex": true | ||
| 54 | - }, | ||
| 55 | - "kv_transfer_config": { | ||
| 56 | - "kv_connector": "MooncakeConnectorV1", | ||
| 57 | - "kv_buffer_device": "npu", | ||
| 58 | - "kv_role": "kv_producer", | ||
| 59 | - "kv_parallel_size": 1, | ||
| 60 | - "kv_port": "21001", | ||
| 61 | - "engine_id": "0", | ||
| 62 | - "kv_rank": 0, | ||
| 63 | - "kv_connector_extra_config": { | ||
| 64 | - "use_ascend_direct": true | ||
| 65 | - } | ||
| 66 | - } | ||
| 67 | - } | ||
| 68 | - }, | ||
| 69 | - "motor_engine_decode_config": { | ||
| 70 | - "engine_type": "vllm", | ||
| 71 | - "motor_nodemanger_config": {}, | ||
| 72 | - "engine_config": { | ||
| 73 | - "served_model_name": "DeepSeek-V3.2", | ||
| 74 | - "model": "/data01/models/DeepSeek-V3.2-w8a8-mtp-QuaRot", | ||
| 75 | - "gpu_memory_utilization": 0.95, | ||
| 76 | - "data_parallel_size": 8, | ||
| 77 | - "tensor_parallel_size": 4, | ||
| 78 | - "pipeline_parallel_size": 1, | ||
| 79 | - "data_parallel_rpc_port": 9000, | ||
| 80 | - "enable_expert_parallel": true, | ||
| 81 | - "enable-chunked-prefill": true, | ||
| 82 | - "seed": 1024, | ||
| 83 | - "max-model-len": 65536, | ||
| 84 | - "max-num-batched-tokens": 64, | ||
| 85 | - "trust-remote-code": true, | ||
| 86 | - "max-num-seqs": 48, | ||
| 87 | - "async-scheduling": true, | ||
| 88 | - "quantization": "ascend", | ||
| 89 | - "tokenizer-mode": "deepseek_v32", | ||
| 90 | - "enable-auto-tool-choice": true, | ||
| 91 | - "tool-call-parser": "deepseek_v32", | ||
| 92 | - "reasoning-parser": "deepseek_v3", | ||
| 93 | - "compilation_config": { | ||
| 94 | - "cudagraph_capture_sizes": [ | ||
| 95 | - 1, 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 48, 56, 64, 80, 96 | ||
| 96 | - ], | ||
| 97 | - "cudagraph_mode": "FULL_DECODE_ONLY" | ||
| 98 | - }, | ||
| 99 | - "speculative-config": { | ||
| 100 | - "num_speculative_tokens": 3, | ||
| 101 | - "method": "deepseek_mtp" | ||
| 102 | - }, | ||
| 103 | - "additional-config": { | ||
| 104 | - "recompute_scheduler_enable": true, | ||
| 105 | - "multistream_overlap_shared_expert": true, | ||
| 106 | - "fuse_qknorm_rope": false, | ||
| 107 | - "fuse_muls_add": true, | ||
| 108 | - "enable_npugraph_ex": true | ||
| 109 | - }, | ||
| 110 | - "kv_transfer_config": { | ||
| 111 | - "kv_connector": "MooncakeConnectorV1", | ||
| 112 | - "kv_buffer_device": "npu", | ||
| 113 | - "kv_role": "kv_consumer", | ||
| 114 | - "kv_parallel_size": 1, | ||
| 115 | - "kv_port": "21001", | ||
| 116 | - "engine_id": "0", | ||
| 117 | - "kv_rank": 0, | ||
| 118 | - "kv_connector_extra_config": { | ||
| 119 | - "use_ascend_direct": true | ||
| 120 | - } | ||
| 121 | - } | ||
| 122 | - } | ||
| 123 | - }, | ||
| 124 | - "kv_cache_pool_config": { | ||
| 125 | - "metadata_server": "P2PHANDSHAKE", | ||
| 126 | - "protocol": "ascend", | ||
| 127 | - "device_name": "", | ||
| 128 | - "alloc_in_same_node": true, | ||
| 129 | - "global_segment_size": "1GB" | ||
| 130 | - } | ||
| 131 | -} | ||
| @@ -1,5 +0,0 @@ | |||
| 1 | -# 说明 | ||
| 2 | - | ||
| 3 | -| 文件 | vllm版本 | PyMotor版本 | 权重 | 执行时间 | 说明 | | ||
| 4 | -| --- | --- | --- | --- | --- | --- | | ||
| 5 | -| user_config.json | 0.18.0 | 主线版本 | DeepSeek-V4-Flash-w8a8-mtp | 2026.5.5 | | | ||
| @@ -1,46 +0,0 @@ | |||
| 1 | -{ | ||
| 2 | - "version": "2.0.0", | ||
| 3 | - "motor_common_env": { | ||
| 4 | - "CANN_INSTALL_PATH": "/usr/local/Ascend" | ||
| 5 | - }, | ||
| 6 | - "motor_controller_env": { | ||
| 7 | - }, | ||
| 8 | - "motor_coordinator_env": { | ||
| 9 | - }, | ||
| 10 | - "motor_engine_prefill_env": { | ||
| 11 | - "VLLM_RPC_TIMEOUT": 3600000, | ||
| 12 | - "VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS": 30000, | ||
| 13 | - "HCCL_EXEC_TIMEOUT": 204, | ||
| 14 | - "HCCL_CONNECT_TIMEOUT": 1200, | ||
| 15 | - "OMP_PROC_BIND": "false", | ||
| 16 | - "OMP_NUM_THREADS": 10, | ||
| 17 | - "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", | ||
| 18 | - "HCCL_BUFFSIZE": 2560, | ||
| 19 | - "TASK_QUEUE_ENABLE": 1, | ||
| 20 | - "HCCL_OP_EXPANSION_MODE": "AIV", | ||
| 21 | - "ASCEND_BUFFER_POOL": "4:8", | ||
| 22 | - "USE_MULTI_GROUPS_KV_CACHE": 1, | ||
| 23 | - "USE_MULTI_BLOCK_POOL": 1 | ||
| 24 | - }, | ||
| 25 | - "motor_engine_decode_env": { | ||
| 26 | - "HCCL_OP_EXPANSION_MODE": "AIV", | ||
| 27 | - "TASK_QUEUE_ENABLE": 1, | ||
| 28 | - "VLLM_RPC_TIMEOUT": 3600000, | ||
| 29 | - "VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS": 30000, | ||
| 30 | - "HCCL_EXEC_TIMEOUT": 204, | ||
| 31 | - "HCCL_CONNECT_TIMEOUT": 1200, | ||
| 32 | - "VLLM_ASCEND_ENABLE_FUSED_MC2": 1, | ||
| 33 | - "OMP_PROC_BIND": "false", | ||
| 34 | - "OMP_NUM_THREADS": 10, | ||
| 35 | - "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", | ||
| 36 | - "HCCL_BUFFSIZE": 1024, | ||
| 37 | - "ASCEND_BUFFER_POOL": "4:8", | ||
| 38 | - "DYNAMIC_EPLB": "true", | ||
| 39 | - "VLLM_TORCH_PROFILER_DIR": "./vllm_profile", | ||
| 40 | - "VLLM_TORCH_PROFILER_WITH_STACK": 0, | ||
| 41 | - "USE_MULTI_GROUPS_KV_CACHE": 1, | ||
| 42 | - "USE_MULTI_BLOCK_POOL": 1 | ||
| 43 | - }, | ||
| 44 | - "motor_kv_cache_pool_env": { | ||
| 45 | - } | ||
| 46 | -} | ||
| @@ -1,5 +0,0 @@ | |||
| 1 | -# 说明 | ||
| 2 | - | ||
| 3 | -| 文件 | vllm版本 | PyMotor版本 | 权重 | 执行时间 | 说明 | | ||
| 4 | -| --- | --- | --- | --- | --- | --- | | ||
| 5 | -| user_config.json | 0.18.0 | 主线版本 | DeepSeek-V4-Pro-w4a8-mtp | 2026.5.12 | | | ||
| @@ -1,115 +0,0 @@ | |||
| 1 | -{ | ||
| 2 | - "version": "v2.0", | ||
| 3 | - "motor_deploy_config": { | ||
| 4 | - "p_instances_num": 1, | ||
| 5 | - "d_instances_num": 1, | ||
| 6 | - "single_p_instance_pod_num": 2, | ||
| 7 | - "single_d_instance_pod_num": 2, | ||
| 8 | - "p_pod_npu_num": 16, | ||
| 9 | - "d_pod_npu_num": 16, | ||
| 10 | - "image_name": "mindie-motor-vllm:3.0.0b1-800I-A3-py3.11-Ubuntu24.04-lts-aarch64", | ||
| 11 | - "job_id": "mindie-motor", | ||
| 12 | - "hardware_type": "800I_A3", | ||
| 13 | - "env_path": "./conf/env.json", | ||
| 14 | - "weight_mount_path": "/data1/DeepSeek-V4-Pro-w4a8-mtp", | ||
| 15 | - "deploy_mode": "multi_deployment" | ||
| 16 | - }, | ||
| 17 | - "motor_controller_config": {}, | ||
| 18 | - "motor_coordinator_config": {}, | ||
| 19 | - "motor_engine_prefill_config": { | ||
| 20 | - "engine_type": "vllm", | ||
| 21 | - "motor_nodemanger_config": {}, | ||
| 22 | - "engine_config": { | ||
| 23 | - "served_model_name": "auto", | ||
| 24 | - "model": "/data1/DeepSeek-V4-Pro-w4a8-mtp", | ||
| 25 | - "gpu_memory_utilization": 0.9, | ||
| 26 | - "data_parallel_size": 2, | ||
| 27 | - "tensor_parallel_size": 16, | ||
| 28 | - "pipeline_parallel_size": 1, | ||
| 29 | - "enable_expert_parallel": true, | ||
| 30 | - "data_parallel_rpc_port": 9000, | ||
| 31 | - "seed": 1024, | ||
| 32 | - "max-model-len": 135000, | ||
| 33 | - "max-num-batched-tokens": 8192, | ||
| 34 | - "max-num-seqs": 4, | ||
| 35 | - "no_disable_hybrid_kv_cache_manager": true, | ||
| 36 | - "safetensors-load-strategy": "prefetch", | ||
| 37 | - "trust-remote-code": true, | ||
| 38 | - "tokenizer-mode": "deepseek_v4", | ||
| 39 | - "tool-call-parser": "deepseek_v4", | ||
| 40 | - "enable-auto-tool-choice": true, | ||
| 41 | - "reasoning-parser": "deepseek_v4", | ||
| 42 | - "quantization": "ascend", | ||
| 43 | - "block-size": "128", | ||
| 44 | - "profiler-config": { | ||
| 45 | - "profiler": "torch", | ||
| 46 | - "torch_profiler_dir": "./vllm_profile", | ||
| 47 | - "torch_profiler_with_stack": false | ||
| 48 | - }, | ||
| 49 | - "enforce-eager": true, | ||
| 50 | - "speculative-config": {"num_speculative_tokens": 1, "method":"deepseek_mtp"}, | ||
| 51 | - "additional-config": { | ||
| 52 | - "enable_cpu_binding": true | ||
| 53 | - }, | ||
| 54 | - "kv_transfer_config": { | ||
| 55 | - "kv_connector": "MooncakeHybridConnector", | ||
| 56 | - "kv_role": "kv_producer", | ||
| 57 | - "kv_port": "30001", | ||
| 58 | - "engine_id": "0" | ||
| 59 | - } | ||
| 60 | - } | ||
| 61 | - }, | ||
| 62 | - "motor_engine_decode_config": { | ||
| 63 | - "engine_type": "vllm", | ||
| 64 | - "motor_nodemanger_config": {}, | ||
| 65 | - "engine_config": { | ||
| 66 | - "served_model_name": "auto", | ||
| 67 | - "model": "/data1/DeepSeek-V4-Pro-w4a8-mtp/", | ||
| 68 | - "gpu_memory_utilization": 0.9, | ||
| 69 | - "data_parallel_size": 16, | ||
| 70 | - "tensor_parallel_size": 2, | ||
| 71 | - "pipeline_parallel_size": 1, | ||
| 72 | - "enable_expert_parallel": true, | ||
| 73 | - "data_parallel_rpc_port": 9000, | ||
| 74 | - "seed": 1024, | ||
| 75 | - "max-model-len": 135000, | ||
| 76 | - "max-num-batched-tokens": 120, | ||
| 77 | - "max-num-seqs": 60, | ||
| 78 | - "async-scheduling": true, | ||
| 79 | - "block-size": "128", | ||
| 80 | - "no_disable_hybrid_kv_cache_manager": true, | ||
| 81 | - "safetensors-load-strategy": "prefetch", | ||
| 82 | - "trust-remote-code": true, | ||
| 83 | - "tokenizer-mode": "deepseek_v4", | ||
| 84 | - "tool-call-parser": "deepseek_v4", | ||
| 85 | - "enable-auto-tool-choice": true, | ||
| 86 | - "reasoning-parser": "deepseek_v4", | ||
| 87 | - "quantization": "ascend", | ||
| 88 | - "profiler-config": { | ||
| 89 | - "profiler": "torch", | ||
| 90 | - "torch_profiler_dir": "./vllm_profile", | ||
| 91 | - "torch_profiler_with_stack": false | ||
| 92 | - }, | ||
| 93 | - "speculative-config": {"num_speculative_tokens": 1, "method":"deepseek_mtp"}, | ||
| 94 | - "compilation_config": { | ||
| 95 | - "cudagraph_mode": "FULL_DECODE_ONLY" | ||
| 96 | - }, | ||
| 97 | - "kv_transfer_config": { | ||
| 98 | - "kv_connector": "MooncakeHybridConnector", | ||
| 99 | - "kv_role": "kv_consumer", | ||
| 100 | - "kv_port": "30001", | ||
| 101 | - "engine_id": "0" | ||
| 102 | - }, | ||
| 103 | - "additional-config": { | ||
| 104 | - "ascend_compilation_config": { | ||
| 105 | - "enable_npugraph_ex": true, | ||
| 106 | - "enable_static_kernel": false | ||
| 107 | - }, | ||
| 108 | - "enable_cpu_binding": true, | ||
| 109 | - "multistream_dsa_preprocess": false, | ||
| 110 | - "multistream_overlap_shared_expert":false, | ||
| 111 | - "recompute_scheduler_enable":true | ||
| 112 | - } | ||
| 113 | - } | ||
| 114 | - } | ||
| 115 | -} | ||
| @@ -1,41 +0,0 @@ | |||
| 1 | -{ | ||
| 2 | - "version": "2.0.0", | ||
| 3 | - "motor_common_env": { | ||
| 4 | - "CANN_INSTALL_PATH": "/usr/local/Ascend", | ||
| 5 | - "MOTOR_LOG_ROOT_PATH": "/root/ascend/log" | ||
| 6 | - }, | ||
| 7 | - "motor_controller_env": { | ||
| 8 | - }, | ||
| 9 | - "motor_coordinator_env": { | ||
| 10 | - }, | ||
| 11 | - "motor_engine_prefill_env": { | ||
| 12 | - "VLLM_RPC_TIMEOUT": 3600000, | ||
| 13 | - "VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS": 30000, | ||
| 14 | - "HCCL_EXEC_TIMEOUT": 360, | ||
| 15 | - "HCCL_CONNECT_TIMEOUT": 600, | ||
| 16 | - "OMP_PROC_BIND": false, | ||
| 17 | - "OMP_NUM_THREADS": 10, | ||
| 18 | - "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", | ||
| 19 | - "VLLM_ASCEND_ENABLE_MLAPO": 1, | ||
| 20 | - "HCCL_BUFFSIZE": 256, | ||
| 21 | - "TASK_QUEUE_ENABLE": 1, | ||
| 22 | - "HCCL_OP_EXPANSION_MODE": "AIV", | ||
| 23 | - "VLLM_USE_V1": 1, | ||
| 24 | - "ASCEND_BUFFER_POOL": "0:0" | ||
| 25 | - }, | ||
| 26 | - "motor_engine_decode_env": { | ||
| 27 | - "VLLM_RPC_TIMEOUT": 3600000, | ||
| 28 | - "VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS": 30000, | ||
| 29 | - "HCCL_EXEC_TIMEOUT": 3600, | ||
| 30 | - "HCCL_CONNECT_TIMEOUT": 3600, | ||
| 31 | - "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", | ||
| 32 | - "VLLM_ASCEND_ENABLE_MLAPO": 1, | ||
| 33 | - "HCCL_BUFFSIZE": 1024, | ||
| 34 | - "TASK_QUEUE_ENABLE": 1, | ||
| 35 | - "HCCL_OP_EXPANSION_MODE": "AIV", | ||
| 36 | - "OMP_PROC_BIND": "false", | ||
| 37 | - "OMP_NUM_THREADS": 10, | ||
| 38 | - "ASCEND_BUFFER_POOL": "0:0", | ||
| 39 | - "VLLM_USE_V1": 1 | ||
| 40 | - } | ||
| 41 | -} | ||
🟡 Medium Priority
建议：在脚本第1行之前添加
#!/bin/bash，第2行添加set -euo pipefail。