Star73
136
代码
代码
Issues4
Pull Requests22
项目讨论
Wiki
分析
Star73
136
已开启
add qwen3next model on a3 script #888
Zhijun创建于 1月7日
add qwen3next model on a3 script #888
已开启
Zhijun创建于 1月7日
从zjchenn/MindSpeed-RLfeature/qwen3next合入到Ascend/MindSpeed-RLmaster
共 5 个文件变更+348-0
Atests/verl_examples/configs/test_grpo_qwen3next_80b_fsdp_A3.sh+175-0
@@ -0,0 +1,175 @@
1+ 
2+set -xeuo pipefail
3+ 
4+project_name="verl_grpo_qwen3-next-80b"
5+experiment_name="Qwen3_Next_80B_Instruct"
6+ 
7+# Paths
8+WORK_DIR=${WORK_DIR:-"${HOME}/verl"}
9+MODEL_PATH=${WORK_DIR}/Qwen3-Next-80B-A3B-Instruct
10+TRAIN_FILE=${WORK_DIR}/datasets/dapo-math-17k/dapo-math-17k.parquet
11+TEST_FILE=${WORK_DIR}/datasets/aime/aime-2024.parquet
12+ 
13+# algorithm
14+adv_estimator=grpo
15+ 
16+use_kl_in_reward=False
17+kl_coef=0.0
18+use_kl_loss=True
19+kl_loss_coef=0.001
20+ 
21+clip_ratio_low=0.2
22+clip_ratio_high=0.28
23+ 
24+temperature=1.0
25+top_p=1.0
26+top_k=-1 # 0 for HF rollout, -1 for vLLM rollout
27+val_top_p=0.7
28+ 
29+# batch
30+train_batch_size=16
31+rollout_n=16
32+ppo_mini_batch_size=8
33+ 
34+# length
35+max_prompt_length=$((1024 * 2))
36+max_response_length=$((1024 * 20))
37+ 
38+# algorithm
39+learning_rate=1e-6
40+warmup_steps=0
41+# enable_filter_groups=True
42+ 
43+# performance
44+sp_size=8
45+gen_tp=4
46+use_dynamic_bsz=True
47+actor_ppo_max_token_len=$(((max_prompt_length + max_response_length) / sp_size))
48+infer_ppo_max_token_len=$(((max_prompt_length + max_response_length) / sp_size))
49+offload=True
50+ 
51+DATA=(
张
张张健翔1月7日

这是参考了verl上nv提的脚本格式吗 ，专业 👍👍👍

likedislike
52+ data.train_files="${TRAIN_FILE}"
53+ data.val_files="${TEST_FILE}"
54+ data.train_batch_size=${train_batch_size}
55+ data.max_prompt_length=${max_prompt_length}
56+ data.max_response_length=${max_response_length}
57+ data.truncation='error'
58+)
59+ 
60+ACTOR=(
61+ actor_rollout_ref.actor.strategy=fsdp2
62+ actor_rollout_ref.nccl_timeout=14400
63+ 
64+ # fsdp
65+ actor_rollout_ref.actor.fsdp_config.use_orig_params=True
66+ actor_rollout_ref.actor.fsdp_config.model_dtype=bfloat16
67+ actor_rollout_ref.actor.fsdp_config.param_offload=${offload}
68+ actor_rollout_ref.actor.fsdp_config.optimizer_offload=${offload}
69+ actor_rollout_ref.actor.fsdp_config.forward_prefetch=False
70+ actor_rollout_ref.actor.fsdp_config.fsdp_size=-1
71+ +actor_rollout_ref.actor.fsdp_config.mixed_precision.reduce_dtype=bf16
72+ 
73+ # optimizer
74+ actor_rollout_ref.actor.optim.lr=${learning_rate}
75+ actor_rollout_ref.actor.optim.lr_warmup_steps=${warmup_steps}
76+ 
77+ # ppo config
78+ actor_rollout_ref.actor.ppo_mini_batch_size=${ppo_mini_batch_size}
79+ actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=1
80+ actor_rollout_ref.actor.ppo_max_token_len_per_gpu=${actor_ppo_max_token_len}
81+ actor_rollout_ref.actor.ulysses_sequence_parallel_size=${sp_size}
82+ 
83+ # entropy
84+ actor_rollout_ref.actor.entropy_checkpointing=True
85+ actor_rollout_ref.actor.entropy_from_logits_with_chunking=True
86+ 
87+ actor_rollout_ref.actor.use_kl_loss=${use_kl_loss}
88+ actor_rollout_ref.actor.kl_loss_coef=${kl_loss_coef}
89+ actor_rollout_ref.actor.kl_loss_type=low_var_kl
90+ actor_rollout_ref.actor.clip_ratio_low=${clip_ratio_low}
91+ actor_rollout_ref.actor.clip_ratio_high=${clip_ratio_high}
92+ actor_rollout_ref.actor.clip_ratio_c=10.0
93+ 
94+ actor_rollout_ref.actor.use_dynamic_bsz=${use_dynamic_bsz}
95+ actor_rollout_ref.actor.use_torch_compile=False
96+)
97+ 
98+ROLLOUT=(
99+ actor_rollout_ref.rollout.name=vllm
100+ actor_rollout_ref.rollout.n=${rollout_n}
101+ actor_rollout_ref.rollout.tensor_model_parallel_size=${gen_tp}
102+ actor_rollout_ref.rollout.gpu_memory_utilization=0.8
103+ actor_rollout_ref.rollout.load_format=auto
104+ actor_rollout_ref.rollout.enforce_eager=True
105+ actor_rollout_ref.rollout.max_num_batched_tokens=$((max_prompt_length + max_response_length))
106+ actor_rollout_ref.rollout.calculate_log_probs=True
107+ 
108+ actor_rollout_ref.rollout.temperature=${temperature}
109+ actor_rollout_ref.rollout.top_p=${top_p}
110+ actor_rollout_ref.rollout.top_k=${top_k}
111+ actor_rollout_ref.rollout.val_kwargs.temperature=${temperature}
112+ actor_rollout_ref.rollout.val_kwargs.top_p=${val_top_p}
113+ actor_rollout_ref.rollout.val_kwargs.top_k=${top_k}
114+ actor_rollout_ref.rollout.val_kwargs.do_sample=True
115+ actor_rollout_ref.rollout.val_kwargs.n=1
116+ 
117+ actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=1
118+ actor_rollout_ref.rollout.log_prob_use_dynamic_bsz=${use_dynamic_bsz}
119+ actor_rollout_ref.rollout.log_prob_max_token_len_per_gpu=${infer_ppo_max_token_len}
120+)
121+ 
122+REF=(
123+ actor_rollout_ref.ref.ulysses_sequence_parallel_size=${sp_size}
124+ actor_rollout_ref.ref.use_torch_compile=False
125+ actor_rollout_ref.ref.fsdp_config.param_offload=${offload}
126+ actor_rollout_ref.ref.fsdp_config.optimizer_offload=${offload}
127+ actor_rollout_ref.ref.fsdp_config.forward_prefetch=False
128+ 
129+ actor_rollout_ref.ref.entropy_checkpointing=True
130+ actor_rollout_ref.ref.entropy_from_logits_with_chunking=True
131+ 
132+ actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=1
133+ actor_rollout_ref.ref.log_prob_use_dynamic_bsz=${use_dynamic_bsz}
134+ actor_rollout_ref.ref.log_prob_max_token_len_per_gpu=${infer_ppo_max_token_len}
135+)
136+ 
137+TRAINER=(
138+ trainer.logger='["console"]'
139+ trainer.project_name="${project_name}"
140+ trainer.experiment_name="${experiment_name}"
141+ trainer.n_gpus_per_node=16
142+ trainer.nnodes=4
143+ trainer.val_before_train=False
144+ trainer.save_freq=5
145+ trainer.test_freq=-1
146+ trainer.total_epochs=1
147+ trainer.device=npu
148+)
149+ 
150+MODEL=(
151+ actor_rollout_ref.model.path=${MODEL_PATH}
152+ actor_rollout_ref.model.use_remove_padding=True
153+ actor_rollout_ref.model.enable_activation_offload=${offload}
154+)
155+ 
156+ALGORITHM=(
157+ algorithm.adv_estimator=${adv_estimator}
158+ algorithm.use_kl_in_reward=${use_kl_in_reward}
159+ algorithm.kl_ctrl.kl_coef=${kl_coef}
160+)
161+ 
162+# =========================================================
163+echo "Starting Training with:"
164+echo "Project: ${project_name}, Exp: ${experiment_name}"
165+echo "Rollout N: ${rollout_n}, Batch Size: ${train_batch_size}, LR: ${learning_rate}"
166+ 
167+ 
168+python3 -m verl.trainer.main_ppo \
169+ "${DATA[@]}" \
170+ "${ACTOR[@]}" \
171+ "${ROLLOUT[@]}" \
172+ "${REF[@]}" \
173+ "${TRAINER[@]}" \
174+ "${ALGORITHM[@]}" \
175+ "${MODEL[@]}" \
Averl_npu/docs/model/qwen3next.md+57-0
@@ -0,0 +1,57 @@
1+# Qwen3-Next 模型在昇腾 NPU 上的安装指南
2+ 
3+## 环境依赖
4+ 
5+| MindSpeed RL版本 | PyTorch版本 | torch_npu版本 | CANN版本 | Python版本 |
6+| --------------- | ------------ | ----------- | ---------- | ---------- |
7+| master（主线） | 2.8.0 | 2.8.0 | 9.0.0.B020 | Python3.11 |
8+ 
9+### 1、安装 vllm 和 vllm-ascend
10+```bash
11+# vllm==0.13.0
12+git clone https://github.com/vllm-project/vllm.git
13+cd vllm
14+git checkout v0.13.0
15+python use_existing_torch.py
16+pip install -r requirements/build.txt
17+export VLLM_TARGET_DEVICE=empty
18+export COMPILE_CUSTOM_KERNELS=1
19+pip install -v -e .
20+cd ..
21+ 
22+# vllm-ascend==0.13.0
23+git clone https://github.com/vllm-project/vllm-ascend.git
24+cd vllm-ascend
25+git checkout releases/v0.13.0
26+pip install -r requirements.txt
27+pip install -v -e .
28+cd ..
29+```
30+ 
31+### 2、安装 triton-ascend
32+由于 vllm-ascend 依赖，triton-ascend，详情请见 [vllm-ascend 文档](https://github.com/vllm-project/vllm-ascend/blob/main/docs/source/tutorials/Qwen3-Next.md)：
33+ 
34+triton-ascend 安装：
35+ 
36+```bash
37+pip install triton-ascend==3.2.0
38+```
39+ 
40+### 3、安装 verl
41+```bash
42+# verl==main
43+git clone https://github.com/volcengine/verl.git
44+cd verl
45+pip install -v -e .
46+cd ..
47+```
48+ 
49+### 4、安装 patch
50+```bash
51+# 请确保 vllm 已正确安装并且之后不会做覆盖
52+git clone https://gitcode.com/Ascend/MindSpeed-RL.git
53+cd MindSpeed-RL/verl_npu
54+pip install -v -e .
55+cd ../..
56+```
57+ 
Averl_npu/verl_npu/patch/verl/d62da4950/ppo_trainer.patch+16-0
@@ -0,0 +1,16 @@
1+diff --git a/verl/trainer/config/ppo_trainer.yaml b/verl/trainer/config/ppo_trainer.yaml
2+index 39a338ea..6d346476 100644
3+--- a/verl/trainer/config/ppo_trainer.yaml
4++++ b/verl/trainer/config/ppo_trainer.yaml
5+@@ -312,6 +312,10 @@ ray_kwargs:
6+
7+ # Number of CPUs for Ray. Use a fixed number instead of null when using SLURM.
8+ num_cpus: null
9+-
10++ runtime_env:
11++ env_vars:
12++ VLLM_VERSION: "0.13.0"
13++ VLLM_WORKER_MULTIPROC_METHOD: "spawn"
14++ VLLM_ASCEND_ENABLE_NZ: "0" # 这个不加会有精度问题
15+ # Path to save Ray timeline JSON for performance profiling
16+ timeline_json_file: null
Averl_npu/verl_npu/patch/vllm/72506c983/qwen3_next.patch+82-0
@@ -0,0 +1,82 @@
1+diff --git a/vllm/model_executor/models/qwen3_next.py b/vllm/model_executor/models/qwen3_next.py
2+index 661a18215..36dcae853 100644
3+--- a/vllm/model_executor/models/qwen3_next.py
4++++ b/vllm/model_executor/models/qwen3_next.py
5+@@ -30,10 +30,7 @@ from vllm.distributed import (
6+ )
7+ from vllm.forward_context import ForwardContext, get_forward_context
8+ from vllm.logger import init_logger
9+-from vllm.model_executor.layers.fla.ops import (
10+- chunk_gated_delta_rule,
11+- fused_recurrent_gated_delta_rule,
12+-)
13++from vllm.model_executor.layers.fla import ops
14+ from vllm.model_executor.layers.fused_moe import SharedFusedMoE
15+ from vllm.model_executor.layers.fused_moe.config import RoutingMethodType
16+ from vllm.model_executor.layers.layernorm import (
17+@@ -53,10 +50,7 @@ from vllm.model_executor.layers.mamba.mamba_utils import (
18+ MambaStateDtypeCalculator,
19+ MambaStateShapeCalculator,
20+ )
21+-from vllm.model_executor.layers.mamba.ops.causal_conv1d import (
22+- causal_conv1d_fn,
23+- causal_conv1d_update,
24+-)
25++from vllm.model_executor.layers.mamba.ops import causal_conv1d
26+ from vllm.model_executor.layers.quantization import QuantizationConfig
27+ from vllm.model_executor.layers.rotary_embedding import get_rope
28+ from vllm.model_executor.layers.vocab_parallel_embedding import (
29+@@ -543,7 +537,7 @@ class Qwen3NextGatedDeltaNet(nn.Module, MambaBase):
30+
31+ # 1.1: Process the multi-query part
32+ if spec_sequence_masks is not None:
33+- mixed_qkv_spec = causal_conv1d_update(
34++ mixed_qkv_spec = causal_conv1d.causal_conv1d_update(
35+ mixed_qkv_spec,
36+ conv_state,
37+ conv_weights,
38+@@ -563,7 +557,7 @@ class Qwen3NextGatedDeltaNet(nn.Module, MambaBase):
39+ mixed_qkv_non_spec_T = mixed_qkv_non_spec.transpose(0, 1)
40+ # - "cache_indices" updates the conv_state cache in positions
41+ # pointed to by "state_indices_tensor"
42+- mixed_qkv_non_spec = causal_conv1d_fn(
43++ mixed_qkv_non_spec = causal_conv1d.causal_conv1d_fn(
44+ mixed_qkv_non_spec_T,
45+ conv_weights,
46+ self.conv1d.bias,
47+@@ -575,7 +569,7 @@ class Qwen3NextGatedDeltaNet(nn.Module, MambaBase):
48+ metadata=attn_metadata,
49+ ).transpose(0, 1)
50+ elif attn_metadata.num_decodes > 0:
51+- mixed_qkv_non_spec = causal_conv1d_update(
52++ mixed_qkv_non_spec = causal_conv1d.causal_conv1d_update(
53+ mixed_qkv_non_spec,
54+ conv_state,
55+ conv_weights,
56+@@ -617,7 +611,7 @@ class Qwen3NextGatedDeltaNet(nn.Module, MambaBase):
57+
58+ # 2.1: Process the multi-query part
59+ if spec_sequence_masks is not None:
60+- core_attn_out_spec, last_recurrent_state = fused_recurrent_gated_delta_rule(
61++ core_attn_out_spec, last_recurrent_state = ops.fused_recurrent_gated_delta_rule(
62+ q=query_spec,
63+ k=key_spec,
64+ v=value_spec,
65+@@ -640,7 +634,7 @@ class Qwen3NextGatedDeltaNet(nn.Module, MambaBase):
66+ (
67+ core_attn_out_non_spec,
68+ last_recurrent_state,
69+- ) = chunk_gated_delta_rule(
70++ ) = ops.chunk_gated_delta_rule(
71+ q=query_non_spec,
72+ k=key_non_spec,
73+ v=value_non_spec,
74+@@ -658,7 +652,7 @@ class Qwen3NextGatedDeltaNet(nn.Module, MambaBase):
75+ )
76+ elif attn_metadata.num_decodes > 0:
77+ core_attn_out_non_spec, last_recurrent_state = (
78+- fused_recurrent_gated_delta_rule(
79++ ops.fused_recurrent_gated_delta_rule(
80+ q=query_non_spec,
81+ k=key_non_spec,
82+ v=value_non_spec,
Mverl_npu/verl_npu/patch_summary/patch_summary.yaml+18-0
@@ -203,3 +203,21 @@ patches:
203 kind: method # [method, attribute, module_attr]203 kind: method # [method, attribute, module_attr]
204 name: forward204 name: forward
205 205
206+ - repo: vllm
207+ current_rev: 72506c98349d6bcd32b4e33eec7b5513453c1502
208+ versions:
209+ - rev: 72506c98349d6bcd32b4e33eec7b5513453c1502
210+ dir: 72506c983
211+ files:
212+ - name: qwen3_next
213+ diff:
214+ class_changes:
215+ - action: updated # [added, replaced, deleted, updated]
216+ name: operators
217+ changes:
218+ - action: update # [added, replaced, deleted]
219+ kind: attribute # [method, parameter, attribute]
220+ name: causal_conv1d
221+ - action: update # [added, replaced, deleted]
222+ kind: attribute # [method, parameter, attribute]
223+ name: chunk_gated_delta_rule