已合并
Revert "update vllm" #896
fh_188创建于 1月15日
Revert "update vllm" #896
已合并
共 11 个文件变更+46-142
| @@ -1,16 +1,16 @@ | |||
| 1 | # 安装指南 | 1 | # 安装指南 |
| 2 | 2 | ||
| 3 | -## 版本配套表 | 3 | +## 版本配套表 |
| 4 | 4 | ||
| 5 | MindSpeed RL支持Atlas 800T A2等昇腾训练硬件形态。软件版本配套表如下： | 5 | MindSpeed RL支持Atlas 800T A2等昇腾训练硬件形态。软件版本配套表如下： |
| 6 | 6 | ||
| 7 | -| MindSpeed RL版本 | Megatron版本 | PyTorch版本 | torch_npu版本 | CANN版本 | Python版本 | | 7 | +| MindSpeed RL版本 | Megatron版本 | PyTorch版本 | torch_npu版本 | CANN版本 | Python版本 | |
| 8 | -| ---------------- | ------------ | ----------- | ------------- | -------- | ---------- | | 8 | +| ---------------- | ------------ |-----------|-------------|---------| ---------- | |
| 9 | -| master（主线） | Core 0.12.0 | 2.7.1 | 2.7.1 | 8.5.0 | Python3.10 | | 9 | +| master（主线） | Core 0.12.0 | 2.7.1 | 2.7.1 | 8.5.0 | Python3.10 | |
| 10 | -| 2.3.0（商分） | Core 0.12.0 | 2.5.1 | 2.5.1 | 8.5.0 | Python3.10 | | 10 | +| 2.3.0（商分） | Core 0.12.0 | 2.5.1 | 2.5.1 | 8.5.0 | Python3.10 | |
| 11 | -| 2.2.0（商分） | Core 0.8.0 | 2.5.1 | 2.5.1 | 8.3.RC1 | Python3.10 | | 11 | +| 2.2.0（商分） | Core 0.8.0 | 2.5.1 | 2.5.1 | 8.3.RC1 | Python3.10 | |
| 12 | -| 2.1.0（商分） | Core 0.8.0 | 2.5.1 | 2.5.1 | 8.2.RC1 | Python3.10 | | 12 | +| 2.1.0（商分） | Core 0.8.0 | 2.5.1 | 2.5.1 | 8.2.RC1 | Python3.10 | |
| 13 | -| 2.0.0（预览） | Core 0.8.0 | 2.5.1 | 2.5.1 | 8.1.RC1 | Python3.10 | | 13 | +| 2.0.0（预览） | Core 0.8.0 | 2.5.1 | 2.5.1 | 8.1.RC1 | Python3.10 | |
| 14 | 14 | ||
| 15 | [昇腾辅助软件](https://gitcode.com/Ascend/pytorch/tree/master#%E6%98%87%E8%85%BE%E8%BE%85%E5%8A%A9%E8%BD%AF%E4%BB%B6)中有更多关于PyTorch和CANN的版本信息。 | 15 | [昇腾辅助软件](https://gitcode.com/Ascend/pytorch/tree/master#%E6%98%87%E8%85%BE%E8%BE%85%E5%8A%A9%E8%BD%AF%E4%BB%B6)中有更多关于PyTorch和CANN的版本信息。 |
| 16 | 16 | ||
| @@ -77,9 +77,7 @@ bash Ascend-hdk-*-npu-driver_*.run --full | |||
| 77 | ``` | 77 | ``` |
| 78 | 78 | ||
| 79 | ### CANN安装 | 79 | ### CANN安装 |
| 80 | - | ||
| 81 | （注：下载参考：[商用稳定版CANN](https://www.hiascend.com/developer/download/commercial/result?module=cann)和[8.5.0 社区体验版CANN](https://www.hiascend.com/developer/download/community/result?module=cann)） | 80 | （注：下载参考：[商用稳定版CANN](https://www.hiascend.com/developer/download/commercial/result?module=cann)和[8.5.0 社区体验版CANN](https://www.hiascend.com/developer/download/community/result?module=cann)） |
| 82 | - | ||
| 83 | ```shell | 81 | ```shell |
| 84 | bash Ascend-cann-toolkit_*_linux-aarch64.run --install | 82 | bash Ascend-cann-toolkit_*_linux-aarch64.run --install |
| 85 | bash Ascend-cann-A3-ops_*_linux-aarch64.run --install | 83 | bash Ascend-cann-A3-ops_*_linux-aarch64.run --install |
| @@ -89,27 +87,23 @@ source /usr/local/Ascend/nnal/atb/set_env.sh | |||
| 89 | ``` | 87 | ``` |
| 90 | 88 | ||
| 91 | ### vllm及相关依赖安装： | 89 | ### vllm及相关依赖安装： |
| 92 | - | ||
| 93 | （注：环境中需要安装git，因为vllm的安装过程依赖git） | 90 | （注：环境中需要安装git，因为vllm的安装过程依赖git） |
| 94 | - | ||
| 95 | ```shell | 91 | ```shell |
| 96 | # pydantic高版本包会产生冲突，指定版本安装 | 92 | # pydantic高版本包会产生冲突，指定版本安装 |
| 97 | pip install pydantic==2.12.0 | 93 | pip install pydantic==2.12.0 |
| 98 | -git clone https://github.com/vllm-project/vllm.git | 94 | +git clone -b releases/v0.11.0 https://github.com/vllm-project/vllm.git |
| 99 | cd vllm | 95 | cd vllm |
| 100 | -git checkout ad32e3e19ccf0526cb6744a5fed09a138a5fb2f9 | 96 | +git checkout b8b302c |
| 101 | VLLM_TARGET_DEVICE=empty pip install . | 97 | VLLM_TARGET_DEVICE=empty pip install . |
| 102 | cd .. | 98 | cd .. |
| 103 | ``` | 99 | ``` |
| 104 | 100 | ||
| 105 | ### vllm_ascend安装 | 101 | ### vllm_ascend安装 |
| 106 | - | ||
| 107 | （注：若机器为x86架构，需要先进行[PyTorch框架安装](#PyTorch框架安装)中的torch和torch_npu安装，安装vllm_ascend时，注释requirements.txt和pyproject.toml中torch和torch_npu的安装） | 102 | （注：若机器为x86架构，需要先进行[PyTorch框架安装](#PyTorch框架安装)中的torch和torch_npu安装，安装vllm_ascend时，注释requirements.txt和pyproject.toml中torch和torch_npu的安装） |
| 108 | - | ||
| 109 | ```shell | 103 | ```shell |
| 110 | -git clone https://github.com/vllm-project/vllm-ascend.git | 104 | +git clone -b v0.11.0-dev https://github.com/vllm-project/vllm-ascend.git |
| 111 | cd vllm-ascend | 105 | cd vllm-ascend |
| 112 | -git checkout 0983c5510aa49c7310b79db72657d8a0f92918ec | 106 | +git checkout 1b16c01 |
| 113 | pip install -r requirements.txt | 107 | pip install -r requirements.txt |
| 114 | export COMPILE_CUSTOM_KERNELS=1 | 108 | export COMPILE_CUSTOM_KERNELS=1 |
| 115 | python setup.py install | 109 | python setup.py install |
| @@ -120,7 +114,6 @@ python setup.py install | |||
| 120 | ```shell | 114 | ```shell |
| 121 | pip install ray==2.42.1 | 115 | pip install ray==2.42.1 |
| 122 | ``` | 116 | ``` |
| 123 | - | ||
| 124 | ```shell | 117 | ```shell |
| 125 | # ray 生成的日志文件夹权限修改 | 118 | # ray 生成的日志文件夹权限修改 |
| 126 | # 此处针对 ray==2.42.1 实现 | 119 | # 此处针对 ray==2.42.1 实现 |
| @@ -130,10 +123,8 @@ sed -i 's/os.chmod(\(.*\), 0o0777)/os.chmod(\1, 0o0750)/g' "$UTILS_PATH" | |||
| 130 | ``` | 123 | ``` |
| 131 | 124 | ||
| 132 | ### PyTorch框架安装 | 125 | ### PyTorch框架安装 |
| 133 | - | ||
| 134 | （注：[PyTorch框架和torch_npu插件安装教程](https://www.hiascend.com/document/detail/zh/Pytorch/710/configandinstg/instg/insg_0004.html)） | 126 | （注：[PyTorch框架和torch_npu插件安装教程](https://www.hiascend.com/document/detail/zh/Pytorch/710/configandinstg/instg/insg_0004.html)） |
| 135 | 可从[PyTorch-Ascend官方代码仓](https://gitcode.com/Ascend/pytorch/releases)获取PyTorch各个版本对应的torch_npu的whl包，也可以通过本仓库对应版本的[vllm-ascend官方代码仓的requirements](https://github.com/vllm-project/vllm-ascend/blob/v0.11.0-dev/requirements.txt)中的安装方式安装torch和torch_npu。 | 127 | 可从[PyTorch-Ascend官方代码仓](https://gitcode.com/Ascend/pytorch/releases)获取PyTorch各个版本对应的torch_npu的whl包，也可以通过本仓库对应版本的[vllm-ascend官方代码仓的requirements](https://github.com/vllm-project/vllm-ascend/blob/v0.11.0-dev/requirements.txt)中的安装方式安装torch和torch_npu。 |
| 136 | - | ||
| 137 | ```shell | 128 | ```shell |
| 138 | # 安装torch和torch_npu | 129 | # 安装torch和torch_npu |
| 139 | pip install torch-2.7.1-cp310-cp310-*.whl | 130 | pip install torch-2.7.1-cp310-cp310-*.whl |
| @@ -144,19 +135,13 @@ pip install apex-0.1.dev*.whl | |||
| 144 | ``` | 135 | ``` |
| 145 | 136 | ||
| 146 | ### 高性能内存库 jemalloc 安装 | 137 | ### 高性能内存库 jemalloc 安装 |
| 147 | - | ||
| 148 | 为了确保 Ray 进程能够正常回收内存，需要安装并使能 jemalloc 库进行内存管理。 | 138 | 为了确保 Ray 进程能够正常回收内存，需要安装并使能 jemalloc 库进行内存管理。 |
| 149 | - | ||
| 150 | #### Ubuntu 操作系统 | 139 | #### Ubuntu 操作系统 |
| 151 | - | ||
| 152 | 通过操作系统源安装jemalloc（注意： 要求ubuntu版本>=20.04）： | 140 | 通过操作系统源安装jemalloc（注意： 要求ubuntu版本>=20.04）： |
| 153 | - | ||
| 154 | ```shell | 141 | ```shell |
| 155 | sudo apt install libjemalloc2 | 142 | sudo apt install libjemalloc2 |
| 156 | ``` | 143 | ``` |
| 157 | - | ||
| 158 | 在启动任务前执行如下命令通过环境变量导入jemalloc，需先通过 **find /usr -name libjemalloc.so.2** 确认文件是否存在 ： | 144 | 在启动任务前执行如下命令通过环境变量导入jemalloc，需先通过 **find /usr -name libjemalloc.so.2** 确认文件是否存在 ： |
| 159 | - | ||
| 160 | ```shell | 145 | ```shell |
| 161 | # arm64架构 | 146 | # arm64架构 |
| 162 | export LD_PRELOAD=/usr/lib/aarch64-linux-gnu/libjemalloc.so.2 | 147 | export LD_PRELOAD=/usr/lib/aarch64-linux-gnu/libjemalloc.so.2 |
| @@ -167,14 +152,11 @@ export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so.2 | |||
| 167 | #### OpenEuler 操作系统 | 152 | #### OpenEuler 操作系统 |
| 168 | 153 | ||
| 169 | 执行如下命令重操作系统源安装jemalloc | 154 | 执行如下命令重操作系统源安装jemalloc |
| 170 | - | ||
| 171 | ```shell | 155 | ```shell |
| 172 | yum install jemalloc | 156 | yum install jemalloc |
| 173 | ``` | 157 | ``` |
| 174 | - | ||
| 175 | 如果上述方法无法正常安装，可以通过源码编译安装 | 158 | 如果上述方法无法正常安装，可以通过源码编译安装 |
| 176 | 前往jemalloc官网下载最新稳定版本，官网地址:https://github.com/jemalloc/jemalloc/releases/ | 159 | 前往jemalloc官网下载最新稳定版本，官网地址:https://github.com/jemalloc/jemalloc/releases/ |
| 177 | - | ||
| 178 | ```shell | 160 | ```shell |
| 179 | tar -xvf jemalloc-{version}.tar.bz2 | 161 | tar -xvf jemalloc-{version}.tar.bz2 |
| 180 | cd jemalloc-{version} | 162 | cd jemalloc-{version} |
| @@ -182,9 +164,7 @@ cd jemalloc-{version} | |||
| 182 | make | 164 | make |
| 183 | make install | 165 | make install |
| 184 | ``` | 166 | ``` |
| 185 | - | ||
| 186 | 在启动任务前执行如下命令通过环境变量导入jemalloc： | 167 | 在启动任务前执行如下命令通过环境变量导入jemalloc： |
| 187 | - | ||
| 188 | ```shell | 168 | ```shell |
| 189 | #根据实际安装路径设置环境变量，例如安装路径为:/usr/local/lib/libjemalloc.so.2,可通过以下命令来设置环境变量(可通过 find /usr -name libjemalloc.so.2 确认文件是否存在) | 169 | #根据实际安装路径设置环境变量，例如安装路径为:/usr/local/lib/libjemalloc.so.2,可通过以下命令来设置环境变量(可通过 find /usr -name libjemalloc.so.2 确认文件是否存在) |
| 190 | export LD_PRELOAD=/usr/lib/aarch64-linux-gnu/libjemalloc.so.2 | 170 | export LD_PRELOAD=/usr/lib/aarch64-linux-gnu/libjemalloc.so.2 |
| @@ -193,7 +173,6 @@ export LD_PRELOAD=/usr/lib/aarch64-linux-gnu/libjemalloc.so.2 | |||
| 193 | > 如以上安装过程出现错误，可以通过提出issue获得更多解决建议。 | 173 | > 如以上安装过程出现错误，可以通过提出issue获得更多解决建议。 |
| 194 | 174 | ||
| 195 | ## 准备源码 | 175 | ## 准备源码 |
| 196 | - | ||
| 197 | ```shell | 176 | ```shell |
| 198 | git clone https://gitcode.com/Ascend/MindSpeed-RL.git | 177 | git clone https://gitcode.com/Ascend/MindSpeed-RL.git |
| 199 | 178 | ||
| @@ -223,7 +202,6 @@ pip install antlr4-python3-runtime==4.9.3 --no-deps | |||
| 223 | ``` | 202 | ``` |
| 224 | 203 | ||
| 225 | ## FAQ | 204 | ## FAQ |
| 226 | - | ||
| 227 | Q：安装过程中出现的软件依赖冲突，是否会影响正常运行？ | 205 | Q：安装过程中出现的软件依赖冲突，是否会影响正常运行？ |
| 228 | 206 | ||
| 229 | -A：建议软件安装顺序严格按照安装指南从上到下进行，此时遇到依赖冲突不会影响正常运行 | 207 | +A：建议软件安装顺序严格按照安装指南从上到下进行，此时遇到依赖冲突不会影响正常运行 |
| @@ -21,8 +21,6 @@ class GenerateConfig(BaseConfig): | |||
| 21 | infer_tensor_parallel_size: Tensor parallel size during inference. Default is 8. | 21 | infer_tensor_parallel_size: Tensor parallel size during inference. Default is 8. |
| 22 | infer_pipeline_parallel_size: Pipeline parallel size during inference. Default is 1. | 22 | infer_pipeline_parallel_size: Pipeline parallel size during inference. Default is 1. |
| 23 | infer_expert_parallel_size: Expert parallel size during inference. Default is 1. | 23 | infer_expert_parallel_size: Expert parallel size during inference. Default is 1. |
| 24 | - infer_prefill_context_parallel_size: Prefill context parallel size during inference. Default is 1. | ||
| 25 | - infer_decode_context_parallel_size: Decode context parallel size during inference. Default is 1. | ||
| 26 | 24 | ||
| 27 | max_num_seqs: Maximum number of sequences to process simultaneously. Default is 256. | 25 | max_num_seqs: Maximum number of sequences to process simultaneously. Default is 256. |
| 28 | max_model_len: Maximum model length (in tokens). Default is 2048. | 26 | max_model_len: Maximum model length (in tokens). Default is 2048. |
| @@ -75,12 +73,6 @@ class GenerateConfig(BaseConfig): | |||
| 75 | # 推理时的流水线并行大小，默认为 1 | 73 | # 推理时的流水线并行大小，默认为 1 |
| 76 | self.infer_pipeline_parallel_size = 1 | 74 | self.infer_pipeline_parallel_size = 1 |
| 77 | 75 | ||
| 78 | - # 推理时的Prefill上下文并行并行大小，默认为 1 | ||
| 79 | - self.infer_prefill_context_parallel_size = 1 | ||
| 80 | - | ||
| 81 | - # 推理时的Decode上下文并行并行大小，默认为 1 | ||
| 82 | - self.infer_decode_context_parallel_size = 1 | ||
| 83 | - | ||
| 84 | # 推理时的专家并行大小，默认为 1 | 76 | # 推理时的专家并行大小，默认为 1 |
| 85 | self.infer_expert_parallel_size = 1 | 77 | self.infer_expert_parallel_size = 1 |
| 86 | 78 | ||
| @@ -169,14 +169,6 @@ def validate_rl_args( | |||
| 169 | if generate_config.infer_pipeline_parallel_size > 1: | 169 | if generate_config.infer_pipeline_parallel_size > 1: |
| 170 | raise ValueError( | 170 | raise ValueError( |
| 171 | "pipeline_parallel for vllm is not supported yet ! ") | 171 | "pipeline_parallel for vllm is not supported yet ! ") |
| 172 | - | ||
| 173 | - if generate_config.infer_prefill_context_parallel_size > 1: | ||
| 174 | - raise ValueError( | ||
| 175 | - "prefill_context_parallel for vllm is not supported yet ! ") | ||
| 176 | - | ||
| 177 | - if generate_config.infer_decode_context_parallel_size > 1: | ||
| 178 | - raise ValueError( | ||
| 179 | - "decode_context_parallel for vllm is not supported yet ! ") | ||
| 180 | 172 | ||
| 181 | if ref_config: | 173 | if ref_config: |
| 182 | ref_data_parallel_size = rl_config.reference_resource.num_npus // ( | 174 | ref_data_parallel_size = rl_config.reference_resource.num_npus // ( |
| @@ -21,8 +21,6 @@ class BaseInferEngine(ABC): | |||
| 21 | infer_tensor_parallel_size: int = 8, | 21 | infer_tensor_parallel_size: int = 8, |
| 22 | infer_pipeline_parallel_size: int = 1, | 22 | infer_pipeline_parallel_size: int = 1, |
| 23 | infer_expert_parallel_size: int = 1, | 23 | infer_expert_parallel_size: int = 1, |
| 24 | - infer_prefill_context_parallel_size: int = 1, | ||
| 25 | - infer_decode_context_parallel_size: int = 1, | ||
| 26 | max_num_seqs: int = 1, # Default value set to 1 | 24 | max_num_seqs: int = 1, # Default value set to 1 |
| 27 | max_model_len: int = 2048, # Default value set to 2048 | 25 | max_model_len: int = 2048, # Default value set to 2048 |
| 28 | dtype: str = "bfloat16", # Default value set to "bfloat16" | 26 | dtype: str = "bfloat16", # Default value set to "bfloat16" |
| @@ -42,8 +40,6 @@ class BaseInferEngine(ABC): | |||
| 42 | infer_tensor_parallel_size (int): Tensor parallel size during inference. | 40 | infer_tensor_parallel_size (int): Tensor parallel size during inference. |
| 43 | infer_pipeline_parallel_size (int): Pipeline parallel size during inference. | 41 | infer_pipeline_parallel_size (int): Pipeline parallel size during inference. |
| 44 | infer_expert_parallel_size (int): Expert parallel size during inference. | 42 | infer_expert_parallel_size (int): Expert parallel size during inference. |
| 45 | - infer_prefill_context_parallel_size (int): Prefill context parallel size during inference. | ||
| 46 | - infer_decode_context_parallel_size (int): Decode context parallel size during inference. | ||
| 47 | max_num_seqs (int): Maximum number of sequences to process simultaneously. Default is 1. | 43 | max_num_seqs (int): Maximum number of sequences to process simultaneously. Default is 1. |
| 48 | max_model_len (int): Maximum model length (in tokens). Default is 2048. | 44 | max_model_len (int): Maximum model length (in tokens). Default is 2048. |
| 49 | dtype (str): Data type for model weights. Default is "bfloat16". | 45 | dtype (str): Data type for model weights. Default is "bfloat16". |
| @@ -61,8 +57,6 @@ class BaseInferEngine(ABC): | |||
| 61 | self.infer_tensor_parallel_size = infer_tensor_parallel_size | 57 | self.infer_tensor_parallel_size = infer_tensor_parallel_size |
| 62 | self.infer_pipeline_parallel_size = infer_pipeline_parallel_size | 58 | self.infer_pipeline_parallel_size = infer_pipeline_parallel_size |
| 63 | self.infer_expert_parallel_size = infer_expert_parallel_size | 59 | self.infer_expert_parallel_size = infer_expert_parallel_size |
| 64 | - self.infer_prefill_context_parallel_size = infer_prefill_context_parallel_size | ||
| 65 | - self.infer_decode_context_parallel_size = infer_decode_context_parallel_size | ||
| 66 | self.max_num_seqs = max_num_seqs | 60 | self.max_num_seqs = max_num_seqs |
| 67 | self.max_model_len = max_model_len | 61 | self.max_model_len = max_model_len |
| 68 | self.dtype = dtype | 62 | self.dtype = dtype |
| @@ -92,7 +92,14 @@ def initialize_kv_cache(self, kv_cache_config: KVCacheConfig) -> None: | |||
| 92 | 92 | ||
| 93 | self.may_reinitialize_input_batch(kv_cache_config) | 93 | self.may_reinitialize_input_batch(kv_cache_config) |
| 94 | 94 | ||
| 95 | - kv_caches = self.initialize_kv_cache_tensors(kv_cache_config) | 95 | + if self.use_sparse: |
| 96 | + kv_caches = self.initialize_kv_cache_tensors_deepseek_sfa( | ||
| 97 | + kv_cache_config) | ||
| 98 | + elif self.model_config.is_deepseek_mla: | ||
| 99 | + kv_caches = self.initialize_kv_cache_tensors_deepseek_mla( | ||
| 100 | + kv_cache_config) | ||
| 101 | + else: | ||
| 102 | + kv_caches = self.initialize_kv_cache_tensors(kv_cache_config) | ||
| 96 | if has_kv_transfer_group(): | 103 | if has_kv_transfer_group(): |
| 97 | get_kv_transfer_group().register_kv_caches(kv_caches) | 104 | get_kv_transfer_group().register_kv_caches(kv_caches) |
| 98 | 105 | ||
| @@ -24,13 +24,13 @@ from vllm.model_executor.layers.fused_moe.layer import (FusedMoE, UnquantizedFus | |||
| 24 | 24 | ||
| 25 | import vllm_ascend.envs as envs_ascend | 25 | import vllm_ascend.envs as envs_ascend |
| 26 | from vllm_ascend.ascend_config import get_ascend_config | 26 | from vllm_ascend.ascend_config import get_ascend_config |
| 27 | -from vllm_ascend.ascend_forward_context import MoECommType | 27 | +from vllm_ascend.ascend_forward_context import FusedMoEState |
| 28 | from vllm_ascend.distributed.parallel_state import get_mc2_group | 28 | from vllm_ascend.distributed.parallel_state import get_mc2_group |
| 29 | from vllm_ascend.ops.expert_load_balancer import ExpertLoadBalancer | 29 | from vllm_ascend.ops.expert_load_balancer import ExpertLoadBalancer |
| 30 | -from vllm_ascend.ops.fused_moe.experts_selector import select_experts | 30 | +from vllm_ascend.ops.moe.experts_selector import select_experts |
| 31 | -from vllm_ascend.ops.fused_moe.fused_moe import AscendFusedMoE | 31 | +from vllm_ascend.ops.common_fused_moe import AscendFusedMoE |
| 32 | -from vllm_ascend.utils import (AscendDeviceType, dispose_tensor, get_ascend_device_type, npu_stream_switch) | 32 | +from vllm_ascend.utils import (AscendSocVersion, dispose_tensor, get_ascend_soc_version, npu_stream_switch) |
| 33 | - | 33 | +from vllm_ascend.torchair.utils import npu_wait_tensor, super_kernel |
| 34 | import vllm_ascend | 34 | import vllm_ascend |
| 35 | 35 | ||
| 36 | 36 | ||
| @@ -85,11 +85,11 @@ def fused_experts_with_mc2( | |||
| 85 | ep_world_size) | 85 | ep_world_size) |
| 86 | 86 | ||
| 87 | # Currently, when in A3 or in torchair graph, we need to pass in some extra param into dispatch & combine | 87 | # Currently, when in A3 or in torchair graph, we need to pass in some extra param into dispatch & combine |
| 88 | - need_extra_args = get_ascend_device_type( | 88 | + need_extra_args = get_ascend_soc_version( |
| 89 | - ) == AscendDeviceType._910_93 or is_torchair | 89 | + ) == AscendSocVersion.A3 or is_torchair |
| 90 | 90 | ||
| 91 | # Currently, when in A3, we need to pass in some extra param into dispatch & combine | 91 | # Currently, when in A3, we need to pass in some extra param into dispatch & combine |
| 92 | - a3_need_extra_args = get_ascend_device_type() == AscendDeviceType._910_93 | 92 | + a3_need_extra_args = get_ascend_soc_version() == AscendSocVersion.A3 |
| 93 | 93 | ||
| 94 | enable_dispatch_v2 = hasattr(torch_npu, "npu_moe_distribute_dispatch_v2") | 94 | enable_dispatch_v2 = hasattr(torch_npu, "npu_moe_distribute_dispatch_v2") |
| 95 | 95 | ||
| @@ -137,10 +137,10 @@ def fused_experts_with_mc2( | |||
| 137 | 137 | ||
| 138 | if shared_experts is not None: | 138 | if shared_experts is not None: |
| 139 | with npu_stream_switch("moe_secondary", 0): | 139 | with npu_stream_switch("moe_secondary", 0): |
| 140 | - torch.npu.current_stream().synchronize() | 140 | + npu_wait_tensor(hidden_states_for_share, topk_weights) |
| 141 | shared_gate_up, _ = shared_experts.gate_up_proj( | 141 | shared_gate_up, _ = shared_experts.gate_up_proj( |
| 142 | hidden_states_for_share) | 142 | hidden_states_for_share) |
| 143 | - torch.npu.current_stream().synchronize() | 143 | + npu_wait_tensor(shared_gate_up, expand_x) |
| 144 | shared_act = shared_experts.act_fn(shared_gate_up) | 144 | shared_act = shared_experts.act_fn(shared_gate_up) |
| 145 | 145 | ||
| 146 | w1 = w1.transpose(1, 2) | 146 | w1 = w1.transpose(1, 2) |
| @@ -220,7 +220,7 @@ def fused_experts_with_mc2( | |||
| 220 | return hidden_states, expert_token_nums, group_list_type | 220 | return hidden_states, expert_token_nums, group_list_type |
| 221 | else: | 221 | else: |
| 222 | with npu_stream_switch("moe_secondary", 0): | 222 | with npu_stream_switch("moe_secondary", 0): |
| 223 | - torch.npu.current_stream().synchronize() | 223 | + npu_wait_tensor(shared_act, down_out_list) |
| 224 | shared_hidden_states, _ = shared_experts.down_proj(shared_act) | 224 | shared_hidden_states, _ = shared_experts.down_proj(shared_act) |
| 225 | return hidden_states, shared_hidden_states, expert_token_nums, group_list_type | 225 | return hidden_states, shared_hidden_states, expert_token_nums, group_list_type |
| 226 | 226 | ||
| @@ -43,10 +43,6 @@ _EP = None | |||
| 43 | _MC2 = None | 43 | _MC2 = None |
| 44 | # Data model parallel group that the current rank belongs to. | 44 | # Data model parallel group that the current rank belongs to. |
| 45 | _DP = None | 45 | _DP = None |
| 46 | -# Prefill context model parallel group that the current rank belongs to. | ||
| 47 | -_PCP = None | ||
| 48 | -# Decode context model parallel group that the current rank belongs to. | ||
| 49 | -_DCP = None | ||
| 50 | 46 | ||
| 51 | # Tensor model parallel group | 47 | # Tensor model parallel group |
| 52 | _TP_GROUP_RANKS = None | 48 | _TP_GROUP_RANKS = None |
| @@ -68,8 +64,6 @@ def initialize_parallel_state( | |||
| 68 | train_expert_model_parallel_size: int = 1, | 64 | train_expert_model_parallel_size: int = 1, |
| 69 | infer_expert_model_parallel_size: int = 1, | 65 | infer_expert_model_parallel_size: int = 1, |
| 70 | train_context_model_parallel_size: int = 1, | 66 | train_context_model_parallel_size: int = 1, |
| 71 | - infer_prefill_context_model_parallel_size: int = 1, | ||
| 72 | - infer_decode_context_model_parallel_size: int = 1, | ||
| 73 | ): | 67 | ): |
| 74 | os.environ["TORCH_NCCL_AVOID_RECORD_STREAMS"] = "1" | 68 | os.environ["TORCH_NCCL_AVOID_RECORD_STREAMS"] = "1" |
| 75 | 69 | ||
| @@ -93,9 +87,7 @@ def initialize_parallel_state( | |||
| 93 | infer_expert_tensor_parallel_size=infer_expert_tensor_parallel_size, | 87 | infer_expert_tensor_parallel_size=infer_expert_tensor_parallel_size, |
| 94 | train_expert_model_parallel_size=train_expert_model_parallel_size, | 88 | train_expert_model_parallel_size=train_expert_model_parallel_size, |
| 95 | infer_expert_model_parallel_size=infer_expert_model_parallel_size, | 89 | infer_expert_model_parallel_size=infer_expert_model_parallel_size, |
| 96 | - train_context_model_parallel_size=train_context_model_parallel_size, | 90 | + train_context_model_parallel_size=train_context_model_parallel_size |
| 97 | - infer_prefill_context_model_parallel_size=infer_prefill_context_model_parallel_size, | ||
| 98 | - infer_decode_context_model_parallel_size=infer_decode_context_model_parallel_size | ||
| 99 | ) | 91 | ) |
| 100 | else: | 92 | else: |
| 101 | initialize_model_parallel(infer_tensor_model_parallel_size, infer_pipeline_model_parallel_size, backend) | 93 | initialize_model_parallel(infer_tensor_model_parallel_size, infer_pipeline_model_parallel_size, backend) |
| @@ -110,8 +102,6 @@ def initialize_model_parallel_for_vllm( | |||
| 110 | train_expert_model_parallel_size: int = 1, | 102 | train_expert_model_parallel_size: int = 1, |
| 111 | infer_expert_model_parallel_size: int = 1, | 103 | infer_expert_model_parallel_size: int = 1, |
| 112 | train_context_model_parallel_size: int = 1, | 104 | train_context_model_parallel_size: int = 1, |
| 113 | - infer_prefill_context_model_parallel_size: int = 1, | ||
| 114 | - infer_decode_context_model_parallel_size: int = 1, | ||
| 115 | rebulid_EP_group: bool = False | 105 | rebulid_EP_group: bool = False |
| 116 | ) -> None: | 106 | ) -> None: |
| 117 | 107 | ||
| @@ -127,7 +117,7 @@ def initialize_model_parallel_for_vllm( | |||
| 127 | raise ValueError("tensor model parallel group is already initialized") | 117 | raise ValueError("tensor model parallel group is already initialized") |
| 128 | 118 | ||
| 129 | global _TP | 119 | global _TP |
| 130 | - | 120 | + |
| 131 | world_size: int = torch.distributed.get_world_size() | 121 | world_size: int = torch.distributed.get_world_size() |
| 132 | 122 | ||
| 133 | backend = torch.distributed.get_backend() | 123 | backend = torch.distributed.get_backend() |
| @@ -207,41 +197,7 @@ def initialize_model_parallel_for_vllm( | |||
| 207 | backend=backend, | 197 | backend=backend, |
| 208 | use_message_queue_broadcaster=True, | 198 | use_message_queue_broadcaster=True, |
| 209 | ) | 199 | ) |
| 210 | - | ||
| 211 | ps._TP = _TP | 200 | ps._TP = _TP |
| 212 | - | ||
| 213 | - global _PCP | ||
| 214 | - pcp_group_ranks = [] | ||
| 215 | - num_pcp_model_parallel_groups: int = world_size // infer_prefill_context_model_parallel_size | ||
| 216 | - for i in range(num_pcp_model_parallel_groups): | ||
| 217 | - ranks = list(range(i, world_size, num_pcp_model_parallel_groups)) | ||
| 218 | - pcp_group_ranks.append(ranks) | ||
| 219 | - logger.info(f"PCP rank: {pcp_group_ranks}") | ||
| 220 | - _PCP = init_model_parallel_group( | ||
| 221 | - group_ranks=pcp_group_ranks, | ||
| 222 | - local_rank=get_world_group().local_rank, | ||
| 223 | - backend=backend, | ||
| 224 | - use_message_queue_broadcaster=True, | ||
| 225 | - ) | ||
| 226 | - ps._PCP = _PCP | ||
| 227 | - | ||
| 228 | - global _DCP | ||
| 229 | - dcp_group_ranks = [] | ||
| 230 | - num_dcp_model_parallel_groups: int = world_size // infer_decode_context_model_parallel_size | ||
| 231 | - for i in range(num_dcp_model_parallel_groups): | ||
| 232 | - ranks = list(range(i, world_size, num_dcp_model_parallel_groups)) | ||
| 233 | - dcp_group_ranks.append(ranks) | ||
| 234 | - logger.info(f"DCP rank: {dcp_group_ranks}") | ||
| 235 | - | ||
| 236 | - _DCP = init_model_parallel_group( | ||
| 237 | - group_ranks=dcp_group_ranks, | ||
| 238 | - local_rank=get_world_group().local_rank, | ||
| 239 | - backend=backend, | ||
| 240 | - use_message_queue_broadcaster=True, | ||
| 241 | - ) | ||
| 242 | - ps._DCP = _DCP | ||
| 243 | - | ||
| 244 | - | ||
| 245 | num_pipeline_model_parallel_groups: int = world_size // infer_pipeline_model_parallel_size | 201 | num_pipeline_model_parallel_groups: int = world_size // infer_pipeline_model_parallel_size |
| 246 | global _PP | 202 | global _PP |
| 247 | if _PP is not None: | 203 | if _PP is not None: |
| @@ -340,7 +296,6 @@ def initialize_model_parallel_for_vllm( | |||
| 340 | backend, | 296 | backend, |
| 341 | group_name="dp") | 297 | group_name="dp") |
| 342 | 298 | ||
| 343 | - | ||
| 344 | os.environ["VLLM_DP_RANK"] = str(ps._DP.rank_in_group) | 299 | os.environ["VLLM_DP_RANK"] = str(ps._DP.rank_in_group) |
| 345 | envs.VLLM_DP_RANK = int(os.environ["VLLM_DP_RANK"]) | 300 | envs.VLLM_DP_RANK = int(os.environ["VLLM_DP_RANK"]) |
| 346 | ip_list = get_cluster_info() | 301 | ip_list = get_cluster_info() |
| @@ -40,8 +40,6 @@ class VLLMInferEngine(BaseInferEngine): | |||
| 40 | infer_tensor_parallel_size: int, | 40 | infer_tensor_parallel_size: int, |
| 41 | infer_pipeline_parallel_size: int, | 41 | infer_pipeline_parallel_size: int, |
| 42 | infer_expert_parallel_size: int, | 42 | infer_expert_parallel_size: int, |
| 43 | - infer_prefill_context_parallel_size: int, | ||
| 44 | - infer_decode_context_parallel_size: int, | ||
| 45 | sampling_config: dict, | 43 | sampling_config: dict, |
| 46 | prompt_type: str = None, | 44 | prompt_type: str = None, |
| 47 | prompt_type_path: str = None, | 45 | prompt_type_path: str = None, |
| @@ -106,8 +104,6 @@ class VLLMInferEngine(BaseInferEngine): | |||
| 106 | gpu_memory_utilization=gpu_memory_utilization, | 104 | gpu_memory_utilization=gpu_memory_utilization, |
| 107 | trust_remote_code=trust_remote_code, | 105 | trust_remote_code=trust_remote_code, |
| 108 | enable_expert_parallel=enable_expert_parallel, | 106 | enable_expert_parallel=enable_expert_parallel, |
| 109 | - infer_prefill_context_parallel_size=infer_prefill_context_parallel_size, | ||
| 110 | - infer_decode_context_parallel_size=infer_decode_context_parallel_size | ||
| 111 | ) | 107 | ) |
| 112 | # Additional initialization logic for VLLMInferEngine | 108 | # Additional initialization logic for VLLMInferEngine |
| 113 | 109 | ||
| @@ -163,9 +159,7 @@ class VLLMInferEngine(BaseInferEngine): | |||
| 163 | train_pipeline_model_parallel_size=train_pipeline_parallel_size, | 159 | train_pipeline_model_parallel_size=train_pipeline_parallel_size, |
| 164 | train_expert_model_parallel_size=train_expert_parallel_size, | 160 | train_expert_model_parallel_size=train_expert_parallel_size, |
| 165 | infer_expert_model_parallel_size=infer_expert_parallel_size, | 161 | infer_expert_model_parallel_size=infer_expert_parallel_size, |
| 166 | - train_context_model_parallel_size=train_context_parallel_size, | 162 | + train_context_model_parallel_size=train_context_parallel_size |
| 167 | - infer_prefill_context_model_parallel_size=infer_prefill_context_parallel_size, | ||
| 168 | - infer_decode_context_model_parallel_size=infer_decode_context_parallel_size | ||
| 169 | ) | 163 | ) |
| 170 | 164 | ||
| 171 | if load_format == "megatron": | 165 | if load_format == "megatron": |
| @@ -187,7 +181,6 @@ class VLLMInferEngine(BaseInferEngine): | |||
| 187 | model=tokenizer_name_or_path, | 181 | model=tokenizer_name_or_path, |
| 188 | trust_remote_code=trust_remote_code, | 182 | trust_remote_code=trust_remote_code, |
| 189 | tensor_parallel_size=infer_tensor_parallel_size, | 183 | tensor_parallel_size=infer_tensor_parallel_size, |
| 190 | - prefill_context_parallel_size=infer_prefill_context_parallel_size, | ||
| 191 | load_format='dummy' if load_format == 'megatron' else load_format, | 184 | load_format='dummy' if load_format == 'megatron' else load_format, |
| 192 | distributed_executor_backend="external_launcher", | 185 | distributed_executor_backend="external_launcher", |
| 193 | enable_prefix_caching=enable_prefix_caching, | 186 | enable_prefix_caching=enable_prefix_caching, |
| @@ -253,7 +246,8 @@ class VLLMInferEngine(BaseInferEngine): | |||
| 253 | ctx = worker.model_runner.vllm_config.compilation_config.static_forward_context | 246 | ctx = worker.model_runner.vllm_config.compilation_config.static_forward_context |
| 254 | else: | 247 | else: |
| 255 | ctx = self.llm.llm_engine.model_executor.driver_worker.worker.compilation_config.static_forward_context | 248 | ctx = self.llm.llm_engine.model_executor.driver_worker.worker.compilation_config.static_forward_context |
| 256 | - from vllm.attention.backends.abstract import AttentionType | 249 | + from vllm.attention import AttentionType |
| 250 | + | ||
| 257 | layer_need_kv_cache = [] | 251 | layer_need_kv_cache = [] |
| 258 | for layer_name in ctx: | 252 | for layer_name in ctx: |
| 259 | if hasattr(ctx[layer_name], 'attn_type') and ctx[layer_name].attn_type in (AttentionType.DECODER, AttentionType.ENCODER_DECODER): | 253 | if hasattr(ctx[layer_name], 'attn_type') and ctx[layer_name].attn_type in (AttentionType.DECODER, AttentionType.ENCODER_DECODER): |
| @@ -367,14 +361,12 @@ class VLLMInferEngine(BaseInferEngine): | |||
| 367 | idx_list = None | 361 | idx_list = None |
| 368 | else: | 362 | else: |
| 369 | prompts = [self.tokenizer.tokenizer.decode(p, skip_special_tokens=True) for p in idx_list] | 363 | prompts = [self.tokenizer.tokenizer.decode(p, skip_special_tokens=True) for p in idx_list] |
| 370 | - | ||
| 371 | with self.update_sampling_params(**kwargs): | 364 | with self.update_sampling_params(**kwargs): |
| 372 | response = self.llm.generate( | 365 | response = self.llm.generate( |
| 373 | prompts=prompts, | 366 | prompts=prompts, |
| 374 | sampling_params=self.sampling_params, | 367 | sampling_params=self.sampling_params, |
| 375 | use_tqdm=False | 368 | use_tqdm=False |
| 376 | ) | 369 | ) |
| 377 | - | ||
| 378 | outs = self._post_process_outputs(response) | 370 | outs = self._post_process_outputs(response) |
| 379 | self.free_cache_engine() | 371 | self.free_cache_engine() |
| 380 | return outs | 372 | return outs |
| @@ -4,7 +4,7 @@ | |||
| 4 | def get_pipeline_model_parallel_rank(mpu, use_vllm=False): | 4 | def get_pipeline_model_parallel_rank(mpu, use_vllm=False): |
| 5 | if use_vllm: | 5 | if use_vllm: |
| 6 | from vllm.distributed import parallel_state as vpu | 6 | from vllm.distributed import parallel_state as vpu |
| 7 | - return vpu.get_pp_group().rank_in_group | 7 | + return vpu.get_pipeline_model_parallel_group().rank_in_group |
| 8 | else: | 8 | else: |
| 9 | return mpu.get_pipeline_model_parallel_rank() | 9 | return mpu.get_pipeline_model_parallel_rank() |
| 10 | 10 | ||
| @@ -12,7 +12,7 @@ def get_pipeline_model_parallel_rank(mpu, use_vllm=False): | |||
| 12 | def get_pipeline_model_parallel_src_rank(mpu, use_vllm=False): | 12 | def get_pipeline_model_parallel_src_rank(mpu, use_vllm=False): |
| 13 | if use_vllm: | 13 | if use_vllm: |
| 14 | from vllm.distributed import parallel_state as vpu | 14 | from vllm.distributed import parallel_state as vpu |
| 15 | - return vpu.get_pp_group().first_rank | 15 | + return vpu.get_pipeline_model_parallel_group().first_rank |
| 16 | else: | 16 | else: |
| 17 | return mpu.get_pipeline_model_parallel_first_rank() | 17 | return mpu.get_pipeline_model_parallel_first_rank() |
| 18 | 18 | ||
| @@ -20,7 +20,7 @@ def get_pipeline_model_parallel_src_rank(mpu, use_vllm=False): | |||
| 20 | def get_pipeline_model_parallel_group(mpu, use_vllm=False): | 20 | def get_pipeline_model_parallel_group(mpu, use_vllm=False): |
| 21 | if use_vllm: | 21 | if use_vllm: |
| 22 | from vllm.distributed import parallel_state as vpu | 22 | from vllm.distributed import parallel_state as vpu |
| 23 | - return vpu.get_pp_group().device_group | 23 | + return vpu.get_pipeline_model_parallel_group().device_group |
| 24 | else: | 24 | else: |
| 25 | return mpu.get_pipeline_model_parallel_group() | 25 | return mpu.get_pipeline_model_parallel_group() |
| 26 | 26 | ||
| @@ -28,7 +28,7 @@ def get_pipeline_model_parallel_group(mpu, use_vllm=False): | |||
| 28 | def is_pipeline_last_stage(mpu, use_vllm=False): | 28 | def is_pipeline_last_stage(mpu, use_vllm=False): |
| 29 | if use_vllm: | 29 | if use_vllm: |
| 30 | from vllm.distributed import parallel_state as vpu | 30 | from vllm.distributed import parallel_state as vpu |
| 31 | - return vpu.get_pp_group().is_last_rank | 31 | + return vpu.get_pipeline_model_parallel_group().is_last_rank |
| 32 | else: | 32 | else: |
| 33 | return mpu.is_pipeline_last_stage() | 33 | return mpu.is_pipeline_last_stage() |
| 34 | 34 | ||
| @@ -55,7 +55,7 @@ def get_context_parallel_rank(mpu, use_vllm=False): | |||
| 55 | def get_tensor_model_parallel_src_rank(mpu, use_vllm=False): | 55 | def get_tensor_model_parallel_src_rank(mpu, use_vllm=False): |
| 56 | if use_vllm: | 56 | if use_vllm: |
| 57 | from vllm.distributed import parallel_state as vpu | 57 | from vllm.distributed import parallel_state as vpu |
| 58 | - return vpu.get_tp_group().first_rank | 58 | + return vpu.get_tensor_model_parallel_group().first_rank |
| 59 | else: | 59 | else: |
| 60 | return mpu.get_tensor_model_parallel_src_rank() | 60 | return mpu.get_tensor_model_parallel_src_rank() |
| 61 | 61 | ||
| @@ -70,7 +70,7 @@ def get_context_parallel_src_rank(mpu, use_vllm=False): | |||
| 70 | def get_tensor_model_parallel_group(mpu, use_vllm=False): | 70 | def get_tensor_model_parallel_group(mpu, use_vllm=False): |
| 71 | if use_vllm: | 71 | if use_vllm: |
| 72 | from vllm.distributed import parallel_state as vpu | 72 | from vllm.distributed import parallel_state as vpu |
| 73 | - return vpu.get_tp_group().device_group | 73 | + return vpu.get_tensor_model_parallel_group().device_group |
| 74 | else: | 74 | else: |
| 75 | return mpu.get_tensor_model_parallel_group() | 75 | return mpu.get_tensor_model_parallel_group() |
| 76 | 76 | ||
| @@ -86,7 +86,7 @@ def get_model_parallel_group(mpu, use_vllm=False): | |||
| 86 | if use_vllm: | 86 | if use_vllm: |
| 87 | import vllm | 87 | import vllm |
| 88 | from vllm.distributed import parallel_state as vpu | 88 | from vllm.distributed import parallel_state as vpu |
| 89 | - return vpu.get_tp_group().device_group | 89 | + return vpu.get_tensor_model_parallel_group().device_group |
| 90 | else: | 90 | else: |
| 91 | return mpu.get_model_parallel_group() | 91 | return mpu.get_model_parallel_group() |
| 92 | 92 | ||
| @@ -104,6 +104,6 @@ def get_tensor_and_context_parallel_group(mpu, use_vllm=False): | |||
| 104 | """Get the tensor- and context-parallel group the caller rank belongs to.""" | 104 | """Get the tensor- and context-parallel group the caller rank belongs to.""" |
| 105 | if use_vllm: | 105 | if use_vllm: |
| 106 | from vllm.distributed import parallel_state as vpu | 106 | from vllm.distributed import parallel_state as vpu |
| 107 | - return vpu.get_tp_group().device_group | 107 | + return vpu.get_tensor_model_parallel_group().device_group |
| 108 | else: | 108 | else: |
| 109 | return mpu.get_tensor_and_context_parallel_group() | 109 | return mpu.get_tensor_and_context_parallel_group() |
| @@ -836,8 +836,6 @@ class ActorHybridWorkerBase(BaseWorker): | |||
| 836 | infer_tensor_parallel_size=self.generate_config.infer_tensor_parallel_size, | 836 | infer_tensor_parallel_size=self.generate_config.infer_tensor_parallel_size, |
| 837 | infer_pipeline_parallel_size=self.generate_config.infer_pipeline_parallel_size, | 837 | infer_pipeline_parallel_size=self.generate_config.infer_pipeline_parallel_size, |
| 838 | infer_expert_parallel_size=self.generate_config.infer_expert_parallel_size, | 838 | infer_expert_parallel_size=self.generate_config.infer_expert_parallel_size, |
| 839 | - infer_prefill_context_parallel_size=self.generate_config.infer_prefill_context_parallel_size, | ||
| 840 | - infer_decode_context_parallel_size=self.generate_config.infer_decode_context_parallel_size, | ||
| 841 | megatron_config=self.megatron_config, | 839 | megatron_config=self.megatron_config, |
| 842 | sampling_config=sampling_config, | 840 | sampling_config=sampling_config, |
| 843 | enable_prefix_caching=self.generate_config.enable_prefix_caching, | 841 | enable_prefix_caching=self.generate_config.enable_prefix_caching, |
| @@ -868,7 +866,7 @@ class ActorHybridWorkerBase(BaseWorker): | |||
| 868 | 866 | ||
| 869 | from vllm.distributed import parallel_state as vpu | 867 | from vllm.distributed import parallel_state as vpu |
| 870 | if vpu.get_tensor_model_parallel_rank() == 0 and \ | 868 | if vpu.get_tensor_model_parallel_rank() == 0 and \ |
| 871 | - vpu.get_pp_group().rank_in_group == 0: | 869 | + vpu.get_pipeline_model_parallel_group().rank_in_group == 0: |
| 872 | server_info = ZmqServerInfo() | 870 | server_info = ZmqServerInfo() |
| 873 | server_info.global_rank = self._rank | 871 | server_info.global_rank = self._rank |
| 874 | server_info.dp_world_size = (vpu.get_tensor_model_parallel_world_size() * | 872 | server_info.dp_world_size = (vpu.get_tensor_model_parallel_world_size() * |
| @@ -330,8 +330,6 @@ class TestActor(): | |||
| 330 | infer_tensor_parallel_size=args.infer_tp, | 330 | infer_tensor_parallel_size=args.infer_tp, |
| 331 | infer_pipeline_parallel_size=args.infer_pp, | 331 | infer_pipeline_parallel_size=args.infer_pp, |
| 332 | infer_expert_parallel_size=args.infer_ep, | 332 | infer_expert_parallel_size=args.infer_ep, |
| 333 | - infer_prefill_context_parallel_size=args.infer_prefill_context_parallel_size, | ||
| 334 | - infer_decode_context_parallel_size=args.infer_decode_context_parallel_size, | ||
| 335 | sampling_config=sampling_config, | 333 | sampling_config=sampling_config, |
| 336 | max_num_seqs=16, | 334 | max_num_seqs=16, |
| 337 | max_model_len=4096, | 335 | max_model_len=4096, |
| @@ -397,8 +395,6 @@ def parse_args(): | |||
| 397 | parser.add_argument("--infer-tp", type=int, default=4) | 395 | parser.add_argument("--infer-tp", type=int, default=4) |
| 398 | parser.add_argument("--infer-pp", type=int, default=1) | 396 | parser.add_argument("--infer-pp", type=int, default=1) |
| 399 | parser.add_argument("--infer-ep", type=int, default=1) | 397 | parser.add_argument("--infer-ep", type=int, default=1) |
| 400 | - parser.add_argument("--infer-prefill-context-parallel-size", type=int, default=1) | ||
| 401 | - parser.add_argument("--infer-decode-context-parallel-size", type=int, default=1) | ||
| 402 | return parser.parse_args() | 398 | return parser.parse_args() |
| 403 | 399 | ||
| 404 | 400 | ||