[verl-npu] delete hybrid_tp_config patch and fix patch

已开启

[verl-npu] delete hybrid_tp_config patch and fix patch_summary.yaml #886

张健翔创建于 1月6日从McFlurry100/MindSpeed-RL： master 合入到 Ascend/MindSpeed-RL： master

[verl-npu] delete hybrid_tp_config patch and fix patch_summary.yaml #886

已开启

张健翔创建于 1月6日

从McFlurry100/MindSpeed-RL master合入到Ascend/MindSpeed-RL master

共 3 个文件变更+45-265

Dverl_npu/verl_npu/patch/verl/d62da4950/hybrid_tp_config.patch+0-171

	@@ -1,171 +0,0 @@
1	-diff --git a/verl/workers/sharding_manager/hybrid_tp_config.py b/verl/workers/sharding_manager/hybrid_tp_config.py
2	-new file mode 100644
3	-index 00000000..f472a177
4	---- /dev/null
5	-+++ b/verl/workers/sharding_manager/hybrid_tp_config.py
6	-@@ -0,0 +1,164 @@
7	-+# Copyright (c) 2025, HUAWEI CORPORATION. All rights reserved.
8	-+# Copyright 2025 Snowflake Inc.
9	-+# SPDX-License-Identifier: Apache-2.0
10	-+#
11	-+# Licensed under the Apache License, Version 2.0 (the "License");
12	-+# you may not use this file except in compliance with the License.
13	-+# You may obtain a copy of the License at
14	-+#
15	-+# http://www.apache.org/licenses/LICENSE-2.0
16	-+#
17	-+# Unless required by applicable law or agreed to in writing, software
18	-+# distributed under the License is distributed on an "AS IS" BASIS,
19	-+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
20	-+# See the License for the specific language governing permissions and
21	-+# limitations under the License.
22	-+
23	-+from dataclasses import dataclass
24	-+from typing import Dict, List, Optional
25	-+
26	-+from omegaconf import DictConfig
27	-+
28	-+
29	-+@dataclass
30	-+class HybridTPConfig:
31	-+ """Configuration for hybrid TP strategy.
32	-+
33	-+ This class defines the configuration for applying different TP sizes
34	-+ to different layers of the model (qkv_proj, o_proj, lm_head).
35	-+ """
36	-+
37	-+ # Whether to enable hybrid TP strategy
38	-+ enabled: bool = False
39	-+
40	-+ # TP size for qkv_proj layer (attention q/k/v projection)
41	-+ # None means follow external tensor_model_parallel_size
42	-+ qkv_proj_tp_size: Optional[int] = None
43	-+
44	-+ # TP size for o_proj layer (attention output projection)
45	-+ # None means follow external tensor_model_parallel_size
46	-+ o_proj_tp_size: Optional[int] = None
47	-+
48	-+ # TP size for lm_head layer
49	-+ # None means follow external tensor_model_parallel_size
50	-+ lm_head_tp_size: Optional[int] = None
51	-+
52	-+ # Custom layer name mappings for non-standard models
53	-+ custom_layer_mappings: Optional[Dict[str, List[str]]] = None
54	-+
55	-+ # Tp size from rollout config
56	-+ external_tp_size: Optional[int] = None
57	-+
58	-+ def __post_init__(self):
59	-+ """Post-initialization validation."""
60	-+
61	-+ def _validate_custom_mappings(self):
62	-+ """validate custom mappings"""
63	-+ required_layers = ["attention_output", "lm_head"]
64	-+ for layer in required_layers:
65	-+ if layer not in self.custom_layer_mappings:
66	-+ raise ValueError(f"Custom layer mappings must include '{layer}'")
67	-+
68	-+
69	-+ @classmethod
70	-+ def from_dict_config(cls, config: DictConfig, external_tp_size: Optional[int] = None) -> "HybridTPConfig":
71	-+ """Create HybridTPConfig from DictConfig.
72	-+
73	-+ Args:
74	-+ config: DictConfig containing hybrid_tp configuration
75	-+ external_tp_size: External tensor_model_parallel_size
76	-+
77	-+ Returns:
78	-+ HybridTPConfig instance
79	-+ """
80	-+ if not config or not config.get("enabled", False):
81	-+ return cls(enabled=False)
82	-+
83	-+ qkv_proj_tp_size = config.get("qkvproj_tensor_parallel_size")
84	-+ if qkv_proj_tp_size is None and external_tp_size is not None:
85	-+ qkv_proj_tp_size = external_tp_size
86	-+
87	-+ # Get TP sizes, use external_tp_size as default if not specified
88	-+ o_proj_tp_size = config.get("oproj_tensor_parallel_size")
89	-+ if o_proj_tp_size is None and external_tp_size is not None:
90	-+ o_proj_tp_size = external_tp_size
91	-+
92	-+ lm_head_tp_size = config.get("lmhead_tensor_parallel_size")
93	-+ if lm_head_tp_size is None and external_tp_size is not None:
94	-+ lm_head_tp_size = external_tp_size
95	-+
96	-+ custom_layer_mappings = config.get("custom_layer_mappings")
97	-+
98	-+ return cls(
99	-+ enabled=True,
100	-+ qkv_proj_tp_size=qkv_proj_tp_size,
101	-+ o_proj_tp_size=o_proj_tp_size,
102	-+ lm_head_tp_size=lm_head_tp_size,
103	-+ external_tp_size=external_tp_size,
104	-+ custom_layer_mappings=custom_layer_mappings,
105	-+ )
106	-+
107	-+
108	-+
109	-+ def validate(self) -> bool:
110	-+ """Validate config fields correctness"""
111	-+ if not self.enabled:
112	-+ return True
113	-+
114	-+ if not self.is_hybrid_enabled():
115	-+ return True
116	-+
117	-+ # basic tp size check
118	-+ for tp_size in [self.o_proj_tp_size, self.qkv_proj_tp_size, self.lm_head_tp_size]:
119	-+ if tp_size is not None:
120	-+ if tp_size <= 0:
121	-+ raise ValueError(f"TP size must be positive, got {tp_size}")
122	-+
123	-+ if self.custom_layer_mappings:
124	-+ self._validate_custom_mappings()
125	-+
126	-+ return True
127	-+
128	-+ def get_tp_size_for_layer(self, layer_name: str) -> int:
129	-+ """Get TP size for a specific layer.
130	-+
131	-+ Args:
132	-+ layer_name: Name of the layer
133	-+ external_tp_size: External tensor_model_parallel_size
134	-+
135	-+ Returns:
136	-+ TP size for the layer
137	-+ """
138	-+ if not self.enabled:
139	-+ return self.external_tp_size
140	-+
141	-+ # Apply custom layer mappings
142	-+ mapped_name = self.custom_layer_mappings.get(layer_name, layer_name)
143	-+
144	-+ # Determine TP size based on layer type
145	-+ if "o_proj" in mapped_name or "self_attn" in mapped_name:
146	-+ return self.o_proj_tp_size
147	-+ elif "lm_head" in mapped_name or "output_layer" in mapped_name:
148	-+ return self.lm_head_tp_size
149	-+ else:
150	-+ return self.external_tp_size
151	-+
152	-+ def get_tp_size_for_layer_type(self, layer_type: str) -> int:
153	-+ """Get tp size for layer type"""
154	-+ size_map = {
155	-+ "attention_output": self.o_proj_tp_size,
156	-+ "lm_head": self.lm_head_tp_size
157	-+ }
158	-+ return size_map.get(layer_type) or self.external_tp_size
159	-+
160	-+ def is_hybrid_enabled(self) -> bool:
161	-+ """Check if hybrid TP is enabled and at least one layer has different TP size."""
162	-+ if not self.enabled:
163	-+ return False
164	-+
165	-+ # Check if at least one layer has different TP size
166	-+ # This will be checked against external_tp_size in the processor
167	-+ return any([
168	-+ self.o_proj_tp_size is not None,
169	-+ self.lm_head_tp_size is not None
170	-+ ])
171	-

Mverl_npu/verl_npu/patch/verl/d62da4950/vllm_rollout_spmd.patch+11-60

	@@ -1,39 +1,16 @@
1	diff --git a/verl/workers/rollout/vllm_rollout/vllm_rollout_spmd.py b/verl/workers/rollout/vllm_rollout/vllm_rollout_spmd.py	1	diff --git a/verl/workers/rollout/vllm_rollout/vllm_rollout_spmd.py b/verl/workers/rollout/vllm_rollout/vllm_rollout_spmd.py
2	-index 63af4f79..adf8e18c 100644	2	+index 63af4f79..94cb6937 100644
3	--- a/verl/workers/rollout/vllm_rollout/vllm_rollout_spmd.py	3	--- a/verl/workers/rollout/vllm_rollout/vllm_rollout_spmd.py
4	+++ b/verl/workers/rollout/vllm_rollout/vllm_rollout_spmd.py	4	+++ b/verl/workers/rollout/vllm_rollout/vllm_rollout_spmd.py
5	-@@ -49,6 +49,7 @@ from omegaconf import ListConfig	5	+@@ -51,6 +51,7 @@ from torch.distributed.device_mesh import DeviceMesh
6	- from tensordict import TensorDict
7	- from torch.distributed.device_mesh import DeviceMesh
8	from vllm import LLM, SamplingParams	6	from vllm import LLM, SamplingParams
9	-+import vllm.envs as envs
10	from vllm.config import CompilationConfig, LoRAConfig	7	from vllm.config import CompilationConfig, LoRAConfig
11	from vllm.lora.request import LoRARequest	8	from vllm.lora.request import LoRARequest
		9	++import vllm.envs as envs
12		10
13	-@@ -81,6 +82,7 @@ from verl.utils.torch_functional import get_response_mask, pad_2d_list_to_length	11	+ try:
14	- from verl.utils.vllm import TensorLoRARequest, VLLMHijack, is_version_ge	12	+ # https://github.com/vllm-project/vllm/commit/96b9aa5aa076e64c68765232aec343e4d0006e2a
15	- from verl.workers.config import HFModelConfig, RolloutConfig	13	+@@ -212,6 +213,10 @@ class vLLMRollout(BaseRollout):
16	- from verl.workers.rollout.base import BaseRollout
17	-+from verl.workers.sharding_manager.hybrid_tp_config import HybridTPConfig
18	- from verl.workers.rollout.utils import get_free_port, is_valid_ipv6_address
19	- from verl.workers.rollout.vllm_rollout.utils import (
20	- VLLM_LORA_INT_ID,
21	-@@ -152,7 +154,14 @@ class vLLMRollout(BaseRollout):
22	- if model_config.lora_rank > 0
23	- else {}
24	- )
25	-+
26	-+ # create HybridTPConfig
27	-+ self.hybrid_tp_config = HybridTPConfig.from_dict_config(
28	-+ self.config.get("hybrid_tp", {}),
29	-+ )
30	-
31	-+ print(f"[NPU Patch] hybrid_tp_config is : {self.hybrid_tp_config if self.hybrid_tp_config else '{}'}")
32	-+
33	- tensor_parallel_size = self.config.get("tensor_model_parallel_size", 1)
34	- assert tensor_parallel_size <= torch.distributed.get_world_size(), (
35	- "tensor parallel size should be less than or equal to the world size"
36	-@@ -212,6 +221,27 @@ class vLLMRollout(BaseRollout):
37	engine_kwargs = {key: val for key, val in engine_kwargs.items() if val is not None}	14	engine_kwargs = {key: val for key, val in engine_kwargs.items() if val is not None}
38	if config.get("limit_images", None): # support for multi-image data	15	if config.get("limit_images", None): # support for multi-image data
39	engine_kwargs["limit_mm_per_prompt"] = {"image": config.get("limit_images")}	16	engine_kwargs["limit_mm_per_prompt"] = {"image": config.get("limit_images")}
	@@ -41,27 +18,10 @@ index 63af4f79..adf8e18c 100644
41	+ # patch this for npu	18	+ # patch this for npu
42	+ if hasattr(config, "dp_model_parallel_size") and config.dp_model_parallel_size > 1:	19	+ if hasattr(config, "dp_model_parallel_size") and config.dp_model_parallel_size > 1:
43	+ self._init_dp_env(config)	20	+ self._init_dp_env(config)
44	-+
45	-+ # Extract hybrid TP config for additional_config
46	-+ additional_config = {}
47	-+ if self.hybrid_tp_config.enabled:
48	-+ # Extract tp_size values from hybrid_tp_config
49	-+ if self.hybrid_tp_config.qkv_proj_tp_size is not None:
50	-+ additional_config["qkvproj_tensor_parallel_size"] = self.hybrid_tp_config.qkv_proj_tp_size
51	-+ if self.hybrid_tp_config.o_proj_tp_size is not None:
52	-+ additional_config["oproj_tensor_parallel_size"] = self.hybrid_tp_config.o_proj_tp_size
53	-+ if self.hybrid_tp_config.lm_head_tp_size is not None:
54	-+ additional_config["lmhead_tensor_parallel_size"] = self.hybrid_tp_config.lm_head_tp_size
55	-+
56	-+ print(f"[NPU Patch] vLLM additional_config: {additional_config if additional_config else '{}'}")
57	-+
58	-+ # Add additional_config to engine_kwargs if not empty
59	-+ if additional_config:
60	-+ engine_kwargs["additional_config"] = additional_config
61		21
62	compilation_config = {}	22	compilation_config = {}
63		23
64	-@@ -243,6 +273,7 @@ class vLLMRollout(BaseRollout):	24	+@@ -243,6 +248,7 @@ class vLLMRollout(BaseRollout):
65	load_format=load_format,	25	load_format=load_format,
66	disable_log_stats=config.disable_log_stats,	26	disable_log_stats=config.disable_log_stats,
67	max_num_batched_tokens=max_num_batched_tokens,	27	max_num_batched_tokens=max_num_batched_tokens,
	@@ -69,11 +29,10 @@ index 63af4f79..adf8e18c 100644
69	enable_chunked_prefill=config.enable_chunked_prefill,	29	enable_chunked_prefill=config.enable_chunked_prefill,
70	enable_prefix_caching=config.enable_prefix_caching,	30	enable_prefix_caching=config.enable_prefix_caching,
71	trust_remote_code=trust_remote_code,	31	trust_remote_code=trust_remote_code,
72	-@@ -270,6 +301,35 @@ class vLLMRollout(BaseRollout):	32	+@@ -271,6 +277,35 @@ class vLLMRollout(BaseRollout):
73	- self.sampling_params = SamplingParams(**kwargs)
74		33
75	self.pad_token_id = tokenizer.pad_token_id	34	self.pad_token_id = tokenizer.pad_token_id
76	-+	35	+
77	+ def _init_dp_env(self, config):	36	+ def _init_dp_env(self, config):
78	+ rank = torch.distributed.get_rank()	37	+ rank = torch.distributed.get_rank()
79	+ world_size = torch.distributed.get_world_size()	38	+ world_size = torch.distributed.get_world_size()
	@@ -102,15 +61,7 @@ index 63af4f79..adf8e18c 100644
102	+ envs.VLLM_DP_MASTER_PORT = int(os.environ["VLLM_DP_MASTER_PORT"])	61	+ envs.VLLM_DP_MASTER_PORT = int(os.environ["VLLM_DP_MASTER_PORT"])
103	+	62	+
104	+ print(f"[VLLM] using TP={tp_size}, DP={dp_size}", flush=True)	63	+ print(f"[VLLM] using TP={tp_size}, DP={dp_size}", flush=True)
105	-	64	++
106	@contextmanager	65	@contextmanager
107	def update_sampling_params(self, **kwargs):	66	def update_sampling_params(self, **kwargs):
108	-@@ -384,7 +444,7 @@ class vLLMRollout(BaseRollout):	67	+ # update sampling params
109	- prompts=vllm_inputs, # because we have already convert it to prompt token id
110	- sampling_params=self.sampling_params,
111	- lora_request=lora_requests,
112	-- use_tqdm=False,
113	-+ use_tqdm=True,
114	- )
115	-
116	- # TODO(sgm): disable logprob when recompute_log_prob is enable

Mverl_npu/verl_npu/patch_summary/patch_summary.yaml+34-34

	@@ -1,44 +1,44 @@
1	# 描述patch信息	1	# 描述patch信息
2	patches:	2	patches:
3	- repo: verl	3	- repo: verl
4	- current_rev: d62da4950573d7a4b7ef2362337952e7ab59e78d # 描述当前repo使用的commit id	4	+ current_rev: d62da4950573d7a4b7ef2362337952e7ab59e78d
5	versions:	5	versions:
6	- rev: d62da4950573d7a4b7ef2362337952e7ab59e78d	6	- rev: d62da4950573d7a4b7ef2362337952e7ab59e78d
7	- dir: d62da4950 # 对应的文件夹名称	7	+ dir: d62da4950
8	files:	8	files:
9	- - name: hybrid_tp_config
10	- diff:
11	- class_changes: # 类变化
12	- - action: added # [added, replaced, deleted]
13	- name: hybrid_tp_config
14	- changes:
15	- - action: added # [added, replaced, deleted]
16	- kind: module_attr # [method, attribute, module_attr]
17	- name: Dict
18	- - action: added # [added, replaced, deleted]
19	- kind: module_attr # [method, attribute, module_attr]
20	- name: DictConfig
21	- - action: added # [added, replaced, deleted]
22	- kind: module_attr # [method, attribute, module_attr]
23	- name: HybridTPConfig
24	- - action: added # [added, replaced, deleted]
25	- kind: module_attr # [method, attribute, module_attr]
26	- name: List
27	- - action: added # [added, replaced, deleted]
28	- kind: module_attr # [method, attribute, module_attr]
29	- name: Optional
30	- - action: added # [added, replaced, deleted]
31	- kind: module_attr # [method, attribute, module_attr]
32	- name: dataclass
33	- name: vllm_rollout_spmd	9	- name: vllm_rollout_spmd
34	diff:	10	diff:
35	- class_changes: # [action, kind, name]	11	+ class_changes:
36	- - action: updated # [added, replaced, deleted, updated]	12	+ - action: updated
37	name: vLLMRollout	13	name: vLLMRollout
38	- changes: # [action, kind, name]	14	+ changes:
39	- - action: replaced # [added, replaced, deleted]	15	+ - action: replaced
40	- kind: method # [class, method, attribute]	16	+ kind: method
41	name: __init__	17	name: __init__
42	- - action: replaced # [added, replaced, deleted]	18	+ - action: added
43	- kind: method # [class, method, attribute]	19	+ kind: method
44	- name: _init_dp_env	20	+ name: _init_dp_env
		21	+ - name: retool
		22	+ diff:
		23	+ class_changes:
		24	+ - action: updated
		25	+ name: CustomRLHFDataset
		26	+ changes:
		27	+ - action: replaced
		28	+ kind: method
		29	+ name: compute_score
		30	+ - name: agent_loop
		31	+ diff:
		32	+ class_changes:
		33	+ - action: updated
		34	+ name: AgentLoopWorkerBase
		35	+ changes:
		36	+ - action: added
		37	+ kind: method
		38	+ name: _run_agent_loop
		39	+ - name: sglang_deepscaler_data
		40	+ diff:
		41	+ module_changes:
		42	+ - action: replaced
		43	+ kind: method
		44	+ name: default_compute_score