新增verl适配vllm 0.11.0-MindSpeed-RL-AtomGit

Mverl_npu/README.MD+5-5

	@@ -14,7 +14,7 @@
14	# vllm	14	# vllm
15	git clone https://github.com/vllm-project/vllm.git	15	git clone https://github.com/vllm-project/vllm.git
16	cd vllm	16	cd vllm
17	-git checkout 38217877aa70041c0115ee367b75197af9cbc5ad	17	+git checkout b8b302cde434df8c9289a2b465406b47ebab1c2d
18	pip install -r requirements/build.txt	18	pip install -r requirements/build.txt
19	VLLM_TARGET_DEVICE=empty pip install -v -e .	19	VLLM_TARGET_DEVICE=empty pip install -v -e .
20	cd ..	20	cd ..
	@@ -22,7 +22,7 @@ cd ..
22	# vllm-ascend	22	# vllm-ascend
23	git clone https://github.com/vllm-project/vllm-ascend.git	23	git clone https://github.com/vllm-project/vllm-ascend.git
24	cd vllm-ascend	24	cd vllm-ascend
25	-git checkout 1de16ead8eecfec8903ec1b330b27a4fa2593c35	25	+git checkout c5fe179cef7f91096c5090bdd3de846cab92b736
26	pip install -r requirements.txt	26	pip install -r requirements.txt
27	export COMPILE_CUSTOM_KERNELS=1	27	export COMPILE_CUSTOM_KERNELS=1
28	python setup.py install	28	python setup.py install
	@@ -31,7 +31,7 @@ cd ..
31	# 源码安装transformers	31	# 源码安装transformers
32	git clone https://github.com/huggingface/transformers.git	32	git clone https://github.com/huggingface/transformers.git
33	cd transformers	33	cd transformers
34	-git checkout 8365f70e925	34	+git checkout e7d351cebad5f6dcdd169b0c034fdee0a000e6a9
35	pip install -e .	35	pip install -e .
36	```	36	```
37		37
	@@ -40,7 +40,7 @@ pip install -e .
40	# MindSpeed	40	# MindSpeed
41	git clone https://gitcode.com/Ascend/MindSpeed.git	41	git clone https://gitcode.com/Ascend/MindSpeed.git
42	cd MindSpeed	42	cd MindSpeed
43	-git checkout 1cdd0abd75e40936ad31721c092f57c695dd72c4	43	+git checkout 2cb32f6d67e8fc0cb84f50ad09d57965e2113d85
44	pip install -e .	44	pip install -e .
45	cd ..	45	cd ..
46		46
	@@ -52,7 +52,7 @@ pip install git+https://github.com/NVIDIA/Megatron-LM.git@core_v0.12.1
52	```bash	52	```bash
53	git clone https://github.com/volcengine/verl.git	53	git clone https://github.com/volcengine/verl.git
54	cd verl	54	cd verl
55	-git checkout 796871d7d092f7cbc6a64e7f4a3796f7a2217f5e	55	+git checkout ddd86f527a4af75095e4677b02b5aa272913a088
56	pip install -e .	56	pip install -e .
57	cd ..	57	cd ..
58	```	58	```

Mverl_npu/verl_npu/patch/verl/796871d7d/megatron_workers.patch+19-16

	@@ -1,20 +1,23 @@
1	diff --git a/verl/workers/megatron_workers.py b/verl/workers/megatron_workers.py	1	diff --git a/verl/workers/megatron_workers.py b/verl/workers/megatron_workers.py
2	-index 7a9c303a..982eb843 100644	2	+index 0d5fbc99..6f64a080 100644
3	--- a/verl/workers/megatron_workers.py	3	--- a/verl/workers/megatron_workers.py
4	+++ b/verl/workers/megatron_workers.py	4	+++ b/verl/workers/megatron_workers.py
5	-@@ -582,7 +582,6 @@ class ActorRolloutRefWorker(MegatronWorker, DistProfilerExtension):	5	+@@ -29,6 +29,8 @@ from codetiming import Timer
6	- if self._is_offload_optimizer:	6	+ from omegaconf import DictConfig, OmegaConf
7	- load_megatron_optimizer(self.actor_optimizer)
8	- log_gpu_memory_usage("After load actor optimizer during update_actor", logger=logger)
9	-- data.batch = data.batch.to(get_device_name())
10		7
11	- micro_batch_size = self.config.actor.ppo_micro_batch_size_per_gpu	8	+ try:
12	- data.meta_info["micro_batch_size"] = micro_batch_size	9	++ from verl_npu.utils import init_torch_compile, replace_torch_compile
13	-@@ -688,7 +687,6 @@ class ActorRolloutRefWorker(MegatronWorker, DistProfilerExtension):	10	++ init_torch_compile(torch.compile)
14	- data.meta_info["max_token_len"] = self.config.rollout.log_prob_max_token_len_per_gpu	11	+ from mindspeed.megatron_adaptor import repatch
15	- data.meta_info["use_dynamic_bsz"] = self.config.rollout.log_prob_use_dynamic_bsz	12	+ except ImportError:
16	- data.meta_info["temperature"] = self.config.rollout.temperature	13	+ repatch = None
17	-- data = data.to(get_device_id())	14	+@@ -510,7 +512,8 @@ class ActorRolloutRefWorker(MegatronWorker, DistProfilerExtension):
18	- output, entropys = self.actor.compute_log_prob(data=data, calculate_entropy=True)	15	+ log_gpu_memory_usage("After MegatronPPOActor init", logger=logger)
19	- output = DataProto.from_dict(	16	+
20	- tensors={"old_log_probs": output, "entropys": entropys},	17	+ if self._is_rollout:
		18	+- self._build_rollout(trust_remote_code=self.config.model.get("trust_remote_code", False))
		19	++ with replace_torch_compile():
		20	++ self._build_rollout(trust_remote_code=self.config.model.get("trust_remote_code", False))
		21	+ log_gpu_memory_usage("After rollout init", logger=logger)
		22	+
		23	+ if self._is_ref:

Averl_npu/verl_npu/patch/verl/796871d7d/rollout.patch+12-0

@@ -0,0 +1,12 @@
	1	+diff --git a/verl/workers/config/rollout.py b/verl/workers/config/rollout.py
	2	+index 9a2514f4..5f95bc0d 100644
	3	+--- a/verl/workers/config/rollout.py
	4	++++ b/verl/workers/config/rollout.py
	5	+@@ -119,6 +119,7 @@ class RolloutConfig(BaseConfig):
	6	+ cudagraph_capture_sizes: Optional[list] = None
	7	+ free_cache_engine: bool = True
	8	+ data_parallel_size: int = 1
	9	++ rollout_data_parallel_size: int = 1
	10	+ expert_parallel_size: int = 1
	11	+ tensor_model_parallel_size: int = 2
	12	+ pipeline_model_parallel_size: int = 1

Mverl_npu/verl_npu/patch/verl/796871d7d/vllm_rollout_spmd.patch+60-58

	@@ -1,66 +1,68 @@
1	diff --git a/verl/workers/rollout/vllm_rollout/vllm_rollout_spmd.py b/verl/workers/rollout/vllm_rollout/vllm_rollout_spmd.py	1	diff --git a/verl/workers/rollout/vllm_rollout/vllm_rollout_spmd.py b/verl/workers/rollout/vllm_rollout/vllm_rollout_spmd.py
2	-index 23eb3346..653703e3 100644	2	+index 40613f1f..23b5b856 100644
3	--- a/verl/workers/rollout/vllm_rollout/vllm_rollout_spmd.py	3	--- a/verl/workers/rollout/vllm_rollout/vllm_rollout_spmd.py
4	+++ b/verl/workers/rollout/vllm_rollout/vllm_rollout_spmd.py	4	+++ b/verl/workers/rollout/vllm_rollout/vllm_rollout_spmd.py
5	-@@ -55,6 +55,7 @@ from verl import DataProto	5	+@@ -49,6 +49,7 @@ from filelock import FileLock
6	- from verl.utils.profiler import GPUMemoryLogger	6	+ from omegaconf import ListConfig
7	- from verl.utils.torch_functional import get_response_mask, pad_2d_list_to_length	7	+ from tensordict import TensorDict
8	- from verl.workers.rollout.base import BaseRollout	8	+ from torch.distributed.device_mesh import DeviceMesh
9	-+from verl.workers.sharding_manager.hybrid_tp_config import HybridTPConfig	9	++import vllm.envs as envs
10	-	10	+ from vllm import LLM, SamplingParams
11	- logger = logging.getLogger(__file__)	11	+ from vllm.config import CompilationConfig, CompilationLevel, LoRAConfig
12	- logger.setLevel(os.getenv("VERL_LOGGING_LEVEL", "WARN"))	12	+ from vllm.lora.request import LoRARequest
13	-@@ -89,6 +90,13 @@ class vLLMRollout(BaseRollout):	13	+@@ -177,6 +178,11 @@ class vLLMRollout(BaseRollout):
14	- super().__init__()
15	- self.config = config
16	-
17	-+ # create HybridTPConfig
18	-+ self.hybrid_tp_config = HybridTPConfig.from_dict_config(
19	-+ self.config.get("hybrid_tp", {}),
20	-+ )
21	-+
22	-+ print(f"[NPU Patch] hybrid_tp_config is : {self.hybrid_tp_config if self.hybrid_tp_config else '{}'}")
23	-+
24	- tensor_parallel_size = self.config.get("tensor_model_parallel_size", 1)
25	- assert tensor_parallel_size <= torch.distributed.get_world_size(), (
26	- "tensor parallel size should be less than or equal to the world size"
27	-@@ -162,6 +170,26 @@ class vLLMRollout(BaseRollout):
28	- engine_kwargs = {key: val for key, val in engine_kwargs.items() if val is not None}
29	if config.get("limit_images", None): # support for multi-image data	14	if config.get("limit_images", None): # support for multi-image data
30	engine_kwargs["limit_mm_per_prompt"] = {"image": config.get("limit_images")}	15	engine_kwargs["limit_mm_per_prompt"] = {"image": config.get("limit_images")}
31	-+ # patch this for npu
32	-+ if hasattr(config, "dp_model_parallel_size") and config.dp_model_parallel_size > 1:
33	-+ self._init_dp_env(config)
34	-+
35	-+ # Extract hybrid TP config for additional_config
36	-+ additional_config = {}
37	-+ if self.hybrid_tp_config.enabled:
38	-+ # Extract tp_size values from hybrid_tp_config
39	-+ if self.hybrid_tp_config.qkv_proj_tp_size is not None:
40	-+ additional_config["qkvproj_tensor_parallel_size"] = self.hybrid_tp_config.qkv_proj_tp_size
41	-+ if self.hybrid_tp_config.o_proj_tp_size is not None:
42	-+ additional_config["oproj_tensor_parallel_size"] = self.hybrid_tp_config.o_proj_tp_size
43	-+ if self.hybrid_tp_config.lm_head_tp_size is not None:
44	-+ additional_config["lmhead_tensor_parallel_size"] = self.hybrid_tp_config.lm_head_tp_size
45	-+
46	-+ print(f"[NPU Patch] vLLM additional_config: {additional_config if additional_config else '{}'}")
47	-+
48	-+ # Add additional_config to engine_kwargs if not empty
49	-+ if additional_config:
50	-+ engine_kwargs["additional_config"] = additional_config
51		16
		17	++ rollout_expert_parallel_size=False
		18	++ if hasattr(config, "rollout_data_parallel_size") and config.rollout_data_parallel_size > 1:
		19	++ self._init_dp_env(config)
		20	++ rollout_expert_parallel_size=True
		21	++
		22	+ compilation_config = {}
		23	+
		24	+ cudagraph_capture_sizes = config.get("cudagraph_capture_sizes")
		25	+@@ -192,6 +198,7 @@ class vLLMRollout(BaseRollout):
52	self.inference_engine = LLM(	26	self.inference_engine = LLM(
53	model=model_path,	27	model=model_path,
54	-@@ -178,8 +206,11 @@ class vLLMRollout(BaseRollout):	28	+ enable_sleep_mode=config.free_cache_engine,
55	- disable_log_stats=config.disable_log_stats,	29	++ enable_expert_parallel=rollout_expert_parallel_size,
56	- max_num_batched_tokens=max_num_batched_tokens,	30	+ tensor_parallel_size=tensor_parallel_size,
57	- enable_chunked_prefill=config.enable_chunked_prefill,	31	+ distributed_executor_backend="external_launcher",
58	-- enable_prefix_caching=True,	32	+ dtype=config.dtype,
59	-+ enable_prefix_caching=config.get("enable_prefix_caching", True),	33	+@@ -456,6 +463,35 @@ class vLLMRollout(BaseRollout):
60	- trust_remote_code=trust_remote_code,	34	+ patch_vllm_moe_model_weight_loader(model)
61	-+ enable_expert_parallel=config.get("enable_expert_parallel", False),	35	+ model.load_weights(weights)
62	-+ max_num_seqs=int(config.get("max_num_seqs", 256)),	36	+
63	-+ compilation_config={"cudagraph_capture_sizes": [8, 16, 32, 64, 128, 192, 256, 384]},	37	++ def _init_dp_env(self, config):
64	- seed=config.get("seed", 0),	38	++ rank = torch.distributed.get_rank()
65	- **lora_kwargs,	39	++ world_size = torch.distributed.get_world_size()
66	- **engine_kwargs,	40	++ tp_size = int(config.get("tensor_model_parallel_size", 1))
		41	++ dp_size = int(config.get("rollout_data_parallel_size", 1))
		42	++
		43	++ all_ranks = torch.arange(world_size).reshape(-1, dp_size, 1, tp_size) # noqa
		44	++ group_ranks = all_ranks.transpose(1, 3).reshape(-1, dp_size).unbind(0)
		45	++ group_ranks = [x.tolist() for x in group_ranks]
		46	++
		47	++ # all gather ip
		48	++ ip_addr = ray.util.get_node_ip_address()
		49	++ ip_list = [None] * world_size
		50	++ torch.distributed.all_gather_object(ip_list, ip_addr)
		51	++
		52	++ for index, group_rank in enumerate(group_ranks):
		53	++ if torch.distributed.get_rank() in group_rank:
		54	++ os.environ["VLLM_DP_MASTER_PORT"] = str(int(os.environ.get("MASTER_PORT")) + 1 + index)
		55	++ os.environ["VLLM_DP_MASTER_IP"] = ip_list[group_rank[0]]
		56	++ local_dp_rank = rank // tp_size % dp_size
		57	++ os.environ["VLLM_DP_RANK"] = str(local_dp_rank)
		58	++ os.environ["VLLM_DP_SIZE"] = str(dp_size)
		59	++ os.environ["VLLM_PORT"] = os.environ["VLLM_DP_MASTER_PORT"]
		60	++ envs.VLLM_DP_RANK = int(os.environ["VLLM_DP_RANK"])
		61	++ envs.VLLM_DP_MASTER_IP = os.environ["VLLM_DP_MASTER_IP"]
		62	++ envs.VLLM_DP_MASTER_PORT = int(os.environ["VLLM_DP_MASTER_PORT"])
		63	++
		64	++ print(f"[VLLM] using TP={tp_size}, DP={dp_size}", flush=True)
		65	++
		66	+
		67	+ # https://github.com/vllm-project/vllm/issues/13175
		68	+ def _monkey_patch_compute_logits(model, vocab_size: int):

Mverl_npu/verl_npu/utils/__init__.py+16-0

@@ -0,0 +1,16 @@
	1	+# coding=utf-8
	2	+# Copyright 2025 Huawei Technologies Co., Ltd
	3	+#
	4	+# Licensed under the Apache License, Version 2.0 (the "License");
	5	+# you may not use this file except in compliance with the License.
	6	+# You may obtain a copy of the License at
	7	+#
	8	+# http://www.apache.org/licenses/LICENSE-2.0
	9	+#
	10	+# Unless required by applicable law or agreed to in writing, software
	11	+# distributed under the License is distributed on an "AS IS" BASIS,
	12	+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	13	+# See the License for the specific language governing permissions and
	14	+# limitations under the License.
	15	+
	16	+from .torch_compile import init_torch_compile, replace_torch_compile

Averl_npu/verl_npu/utils/torch_compile.py+34-0

@@ -0,0 +1,34 @@
	1	+# coding=utf-8
	2	+# Copyright 2025 Huawei Technologies Co., Ltd
	3	+#
	4	+# Licensed under the Apache License, Version 2.0 (the "License");
	5	+# you may not use this file except in compliance with the License.
	6	+# You may obtain a copy of the License at
	7	+#
	8	+# http://www.apache.org/licenses/LICENSE-2.0
	9	+#
	10	+# Unless required by applicable law or agreed to in writing, software
	11	+# distributed under the License is distributed on an "AS IS" BASIS,
	12	+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	13	+# See the License for the specific language governing permissions and
	14	+# limitations under the License.
	15	+
	16	+import torch
	17	+from contextlib import contextmanager
	18	+
	19	+_COMPILE = None
	20	+
	21	+def init_torch_compile(compile):
	22	+ global _COMPILE
	23	+ _COMPILE = compile
	24	+
	25	+@contextmanager
	26	+def replace_torch_compile():
	27	+ """Context manager to temporarily replace torch.compile with a dummy function"""
	28	+ original_compile = torch.compile # Save the original function
	29	+ torch.compile = _COMPILE # Replace with our dummy
	30	+
	31	+ try:
	32	+ yield # Execute the code inside the 'with' block
	33	+ finally:
	34	+ torch.compile = original_compile # Restore the original function