【NEW Support】support deepseek-v3-671b grpo-MindSpeed-RL-AtomGit

MREADME.md+1-0

	@@ -74,6 +74,7 @@ MindSpeed RL是基于昇腾生态的强化学习加速框架，旨在为华为 [
74	<td rowspan="1">	74	<td rowspan="1">
75	<a href="docs/solutions/r1_zero_qwen25_7b.md">Qwen2.5-7B</a> <br>	75	<a href="docs/solutions/r1_zero_qwen25_7b.md">Qwen2.5-7B</a> <br>
76	<a href="docs/solutions/r1_zero_qwen25_32b.md">Qwen2.5-32B</a> <br>	76	<a href="docs/solutions/r1_zero_qwen25_32b.md">Qwen2.5-32B</a> <br>
		77	+ <a href="docs/solutions/r1_zero_deepseek_671b.md"> DeepSeek-R1-671B </a> <br>
77	</td>	78	</td>
78	<td> Released</td>	79	<td> Released</td>
79	</tr>	80	</tr>

Mcli/train_dapo.py+5-1

	@@ -19,7 +19,7 @@ from mindspeed_rl.utils import seed_all
19	from mindspeed_rl.utils import get_tokenizer	19	from mindspeed_rl.utils import get_tokenizer
20	from mindspeed_rl.utils.utils import MsProbe, get_node_nums	20	from mindspeed_rl.utils.utils import MsProbe, get_node_nums
21	from mindspeed_rl.utils.loggers import Loggers	21	from mindspeed_rl.utils.loggers import Loggers
22	-from mindspeed_rl.utils.utils import parse_args_from_config	22	+from mindspeed_rl.utils.utils import parse_args_from_config, init_torch_compile
23	from mindspeed_rl.config_cls.validate_config import validate_rl_args	23	from mindspeed_rl.config_cls.validate_config import validate_rl_args
24	from mindspeed_rl.config_cls.megatron_config import MegatronConfig	24	from mindspeed_rl.config_cls.megatron_config import MegatronConfig
25	from mindspeed_rl.config_cls.rl_config import RLConfig	25	from mindspeed_rl.config_cls.rl_config import RLConfig
	@@ -395,6 +395,10 @@ def initialize_megatron(
395	origin_sys_argv = sys.argv	395	origin_sys_argv = sys.argv
396	sys.argv = [sys.argv[0]]	396	sys.argv = [sys.argv[0]]
397	parse_args_from_config(config)	397	parse_args_from_config(config)
		398	+
		399	+ # Initialize torch.compile global variables to avoid training-related patches affecting vLLM graph mode enabling.
		400	+ init_torch_compile(torch.compile)
		401	+ # Note: Importing this line activates the megatron_adapter.
398	from mindspeed_llm.training.arguments import parse_args_decorator	402	from mindspeed_llm.training.arguments import parse_args_decorator
399	import megatron	403	import megatron
400		404

Mcli/train_grpo.py+5-1

	@@ -21,7 +21,7 @@ from mindspeed_rl.datasets.build_dataset import build_train_valid_test_datasets
21	from mindspeed_rl.utils import seed_all	21	from mindspeed_rl.utils import seed_all
22	from mindspeed_rl.utils.utils import MsProbe, get_node_nums	22	from mindspeed_rl.utils.utils import MsProbe, get_node_nums
23	from mindspeed_rl.utils.loggers import Loggers	23	from mindspeed_rl.utils.loggers import Loggers
24	-from mindspeed_rl.utils.utils import parse_args_from_config	24	+from mindspeed_rl.utils.utils import parse_args_from_config, init_torch_compile
25	from mindspeed_rl.config_cls.megatron_config import MegatronConfig	25	from mindspeed_rl.config_cls.megatron_config import MegatronConfig
26	from mindspeed_rl.config_cls.rl_config import RLConfig	26	from mindspeed_rl.config_cls.rl_config import RLConfig
27	from mindspeed_rl.config_cls.generate_config import GenerateConfig	27	from mindspeed_rl.config_cls.generate_config import GenerateConfig
	@@ -406,6 +406,10 @@ def initialize_megatron(
406	origin_sys_argv = sys.argv	406	origin_sys_argv = sys.argv
407	sys.argv = [sys.argv[0]]	407	sys.argv = [sys.argv[0]]
408	parse_args_from_config(config)	408	parse_args_from_config(config)
		409	+
		410	+ # Initialize torch.compile global variables to avoid training-related patches affecting vLLM graph mode enabling.
		411	+ init_torch_compile(torch.compile)
		412	+ # Note: Importing this line activates the megatron_adapter.
409	from mindspeed_llm.training.arguments import parse_args_decorator	413	from mindspeed_llm.training.arguments import parse_args_decorator
410	import megatron	414	import megatron
411		415

Mconfigs/envs/runtime_env.yaml+5-5

	@@ -12,7 +12,7 @@ env_vars:
12	#设置 HCCL 执行超时时间	12	#设置 HCCL 执行超时时间
13	HCCL_EXEC_TIMEOUT: '3600'	13	HCCL_EXEC_TIMEOUT: '3600'
14	#设置 HCCL 通信端口	14	#设置 HCCL 通信端口
15	- HCCL_IF_BASE_PORT: '48000'	15	+ HCCL_IF_BASE_PORT: '12345'
16	#设置设备最大连接数	16	#设置设备最大连接数
17	CUDA_DEVICE_MAX_CONNECTIONS: '1'	17	CUDA_DEVICE_MAX_CONNECTIONS: '1'
18	#设置 HYDRA 是否输出完整错误日志	18	#设置 HYDRA 是否输出完整错误日志
	@@ -26,11 +26,11 @@ env_vars:
26	# 使用vLLM的V1 engine API（v1接口），兼容性选项	26	# 使用vLLM的V1 engine API（v1接口），兼容性选项
27	VLLM_USE_V1: '1'	27	VLLM_USE_V1: '1'
28	# 指定使用的vLLM版本号	28	# 指定使用的vLLM版本号
29	- VLLM_VERSION: '0.9.0'	29	+ VLLM_VERSION: '0.9.1'
30	- # 启用昇腾torchair图模式优化（1=启用），提升执行效率
31	- VLLM_ENABLE_GRAPH_MODE: '0'
32	# 使能vLLM TOPK性能优化	30	# 使能vLLM TOPK性能优化
33	- VLLM_ASCEND_ENABLE_TOPK_OPTIMZE: "1"	31	+ VLLM_ASCEND_ENABLE_TOPK_OPTIMIZE: "1"
		32	+ # vLLM aclop init mode: 0: default, normal init.
		33	+ VLLM_ASCEND_ACL_OP_INIT_MODE: '0'
34	# 指定使用 Level 2 级别的算子下发队列优化	34	# 指定使用 Level 2 级别的算子下发队列优化
35	TASK_QUEUE_ENABLE: "2"	35	TASK_QUEUE_ENABLE: "2"
36	# 指定使能 CPU 绑核	36	# 指定使能 CPU 绑核

Aconfigs/grpo_deepseek_r1_671b_A3.yaml+117-0

@@ -0,0 +1,117 @@
	1	+defaults:
	2	+ - model:
	3	+ - deepseekv3_671b
	4	+
	5	+megatron_training:
	6	+ model: deepseekv3_671b
	7	+ use_flash_attn: true
	8	+ shape_order: BNSD
	9	+ use_fused_rotary_pos_emb: true
	10	+ use_fused_rmsnorm: true
	11	+ use_fused_swiglu: true
	12	+ no_masked_softmax_fusion: true
	13	+ attention_softmax_in_fp32: true
	14	+ bf16: true
	15	+ sequence_parallel: true
	16	+ use_distributed_optimizer: true
	17	+ tokenizer_type: PretrainedFromHF
	18	+ tokenizer_name_or_path: ./DeepSeek-V3-hf/
	19	+ global_batch_size: 384
	20	+ seq_length: 1024
	21	+ save_interval: 100
	22	+ train_iters: 1000
	23	+ stage: ray_grpo
	24	+ attention_dropout: 0.0
	25	+ init_method_std: 0.01
	26	+ hidden_dropout: 0.0
	27	+ distributed_backend: nccl
	28	+ no_shared_storage: true
	29	+ variable_seq_lengths: true
	30	+ dataset_additional_keys: ['labels',]
	31	+ no_shuffle: false
	32	+ data_path: ./deepseek/deepscaler
	33	+ split: 100,0,0
	34	+ swap_optimizer: true
	35	+ moe_alltoall_overlap_comm: true
	36	+ reset_position_ids: true
	37	+
	38	+actor_config:
	39	+ model: deepseekv3_671b
	40	+ micro_batch_size: 1
	41	+ moe_tp_extend_ep: true
	42	+ expert_model_parallel_size: 16
	43	+ tensor_model_parallel_size: 4
	44	+ pipeline_model_parallel_size: 6
	45	+ num_layer_list: 9,10,10,10,11,11
	46	+ recompute_granularity: full
	47	+ recompute_method: block
	48	+ recompute_num_layers: 9
	49	+ lr: 1e-6
	50	+ lr_decay_style: constant
	51	+ min_lr: 0.0
	52	+ weight_decay: 0.0
	53	+ lr_warmup_fraction: 0.0
	54	+ clip_grad: 10000.0
	55	+ adam_beta1: 0.9
	56	+ adam_beta2: 0.95
	57	+ initial_loss_scale: 4096
	58	+ finetune: true
	59	+ load: ./ckpt/
	60	+ save: ./ckpt/
	61	+ no_load_optim: true
	62	+ no_load_rng: true
	63	+
	64	+rl_config:
	65	+ use_integrated_worker: true
	66	+ use_dynamic_bsz: true
	67	+ max_packing_token_size: 4096
	68	+ blocking: true
	69	+ gamma: 1.0
	70	+ lam: 0.95
	71	+ adv_estimator: group_norm
	72	+ kl_penalty: kl
	73	+ kl_ctrl_type: fixed
	74	+ init_kl_coef: 0.001
	75	+ mini_batch_size: 384
	76	+ use_remove_padding: true
	77	+ max_prompt_length: 1024
	78	+ epochs: 1
	79	+ clip_ratio: 0.2
	80	+ entropy_coeff: 0
	81	+ n_samples_per_prompt: 32
	82	+ rule_reward: true
	83	+ verifier_function: ["base_acc"]
	84	+ verifier_weight: [1.0]
	85	+ actor_resource:
	86	+ num_npus: 384
	87	+
	88	+generate_config:
	89	+ # tokenizer相关配置
	90	+ trust_remote_code: true
	91	+ # 推理时的并行配置
	92	+ infer_tensor_parallel_size: 2
	93	+ infer_pipeline_parallel_size: 1
	94	+ infer_expert_parallel_size: 64
	95	+ # vllm 模型相关设置
	96	+ max_num_seqs: 64
	97	+ max_model_len: 3072
	98	+ max_num_batched_tokens: 1024
	99	+ dtype: "bfloat16"
	100	+ gpu_memory_utilization: 0.75
	101	+ enforce_eager: false
	102	+ torchair_graph: true
	103	+ enable_expert_parallel: true
	104	+
	105	+ offload_train_optimizer: false
	106	+ offload_train_grad: true
	107	+ offload_train_param: true
	108	+
	109	+ # 采样配置
	110	+ sampling_config:
	111	+ logprobs: 1
	112	+ max_tokens: 2048
	113	+ top_p: 1
	114	+ top_k: -1
	115	+ min_p: 0
	116	+ temperature: 1.0
	117	+ detokenize: false

Mconfigs/grpo_qwen25_32b_A2.yaml+1-0

	@@ -77,6 +77,7 @@ rl_config:
77	shuffle_mini_batch: false	77	shuffle_mini_batch: false
78	n_samples_per_prompt: 16	78	n_samples_per_prompt: 16
79	rule_reward: true	79	rule_reward: true
		80	+ log_max_throughput: false
80	verifier_function: ["base_acc"]	81	verifier_function: ["base_acc"]
81	verifier_weight: [1.0]	82	verifier_weight: [1.0]
82	actor_resource:	83	actor_resource:

Mconfigs/grpo_qwen25_32b_A3.yaml+1-0

	@@ -77,6 +77,7 @@ rl_config:
77	shuffle_mini_batch: false	77	shuffle_mini_batch: false
78	n_samples_per_prompt: 16	78	n_samples_per_prompt: 16
79	rule_reward: true	79	rule_reward: true
		80	+ log_max_throughput: false
80	verifier_function: ["base_acc"]	81	verifier_function: ["base_acc"]
81	verifier_weight: [1.0]	82	verifier_weight: [1.0]
82	actor_resource:	83	actor_resource:

Mconfigs/grpo_qwen25_7b_A3.yaml+1-0

	@@ -78,6 +78,7 @@ rl_config:
78	rule_reward: true	78	rule_reward: true
79	verifier_function: ["math_17k_acc"]	79	verifier_function: ["math_17k_acc"]
80	verifier_weight: [1.0]	80	verifier_weight: [1.0]
		81	+ log_max_throughput: false
81	num_cpus_for_local_task: 1.0	82	num_cpus_for_local_task: 1.0
82	use_tensorboard: true	83	use_tensorboard: true
83	actor_resource:	84	actor_resource:

Aconfigs/model/deepseekv3_671b.yaml+55-0

@@ -0,0 +1,55 @@
	1	+deepseekv3_671b:
	2	+ use_mcore_models: true
	3	+ spec:
	4	+ - mindspeed_llm.tasks.models.spec.deepseek_spec
	5	+ - layer_spec
	6	+ num_layers: 61
	7	+ num_experts: 256
	8	+ moe_intermediate_size: 2048
	9	+ ffn_hidden_size: 18432
	10	+ num_attention_heads: 128
	11	+ hidden_size: 7168
	12	+ untie_embeddings_and_output_weights: true
	13	+ disable_bias_linear: true
	14	+
	15	+ multi_head_latent_attention: true
	16	+ qk_rope_head_dim: 64
	17	+ qk_nope_head_dim: 128
	18	+ q_lora_rank: 1536
	19	+ kv_lora_rank: 512
	20	+ v_head_dim: 128
	21	+ qk_layernorm: true
	22	+ moe_grouped_gemm: true
	23	+ moe_permutation_async_comm: true
	24	+ use_fused_moe_token_permute_and_unpermute: true
	25	+ moe_token_dispatcher_type: alltoall
	26	+ first_k_dense_replace: 3
	27	+ moe_layer_freq: 1
	28	+ n_shared_experts: 1
	29	+ moe_router_topk: 8
	30	+ moe_router_load_balancing_type: noaux_tc
	31	+ n_group: 8
	32	+ topk_group: 4
	33	+ routed_scaling_factor: 2.5
	34	+ seq_aux: true
	35	+ norm_topk_prob: true
	36	+ moe_router_score_function: sigmoid
	37	+ moe_router_enable_expert_bias: true
	38	+
	39	+ position_embedding_type: rope
	40	+ use_rotary_position_embeddings: true
	41	+ rotary_base: 10000
	42	+ rope_scaling_type: yarn
	43	+ rope_scaling_beta_fast: 32
	44	+ rope_scaling_beta_slow: 1
	45	+ rope_scaling_factor: 40
	46	+ rope_scaling_mscale: 1.0
	47	+ rope_scaling_mscale_all_dim: 1.0
	48	+ rope_scaling_original_max_position_embeddings: 4096
	49	+ max_position_embeddings: 163840
	50	+ padded_vocab_size: 129280
	51	+ make_vocab_size_divisible_by: 1
	52	+
	53	+ normalization: RMSNorm
	54	+ norm_epsilon: 1e-6
	55	+ swiglu: true

Mdocs/algorithms/grpo.md+9-7

Mdocs/features/grpo_yaml.md+10-3

	@@ -99,9 +99,13 @@ wandb开关:
99	vllm 模型参数可以参照 [vllm官网参数介绍](https://docs.vllm.ai/en/latest/serving/engine_args.html)：	99	vllm 模型参数可以参照 [vllm官网参数介绍](https://docs.vllm.ai/en/latest/serving/engine_args.html)：
100	* `max_num_seqs`：vllm 推理并发最大样本限制；	100	* `max_num_seqs`：vllm 推理并发最大样本限制；
101	* `max_model_len`：vllm 能够处理的最大输入序列长度(prompt+response)；	101	* `max_model_len`：vllm 能够处理的最大输入序列长度(prompt+response)；
		102	+* `max_num_batched_tokens`：vllm 单步能处理的最大 token 数量；
		103	+* `enforce_eager`：使能PyTorch eager模式，默认开启，仅 DeepSeek V3 开启 torchair_graph 时需要关闭；
		104	+* `torchair_graph`：DeepSeek V3 使能 torchair 图模式；
		105	+* `enable_expert_parallel`：MOE 模型使能专家切分，需要 MOE 模型支持；
102	* `dtype`：vllm 推理所使用的数据类型；	106	* `dtype`：vllm 推理所使用的数据类型；
103	* `gpu_memory_utilization`：GPU 内存利用率，指定推理时使用 GPU 内存的比例；	107	* `gpu_memory_utilization`：GPU 内存利用率，指定推理时使用 GPU 内存的比例；
104	-* `num_scheduler_steps `：指的是在一个完整的调度周期内，调度器会将批处理请求分成多少个子步骤来执行；	108	+* `num_scheduler_steps`：指的是在一个完整的调度周期内，调度器会将批处理请求分成多少个子步骤来执行；
105	#### 采样配置	109	#### 采样配置
106	* `logprobs`：是否生成logprobs；	110	* `logprobs`：是否生成logprobs；
107	* `max_tokens`：单条response最大生成token数量；	111	* `max_tokens`：单条response最大生成token数量；
	@@ -126,8 +130,11 @@ vllm 模型参数可以参照 [vllm官网参数介绍](https://docs.vllm.ai/en/
126	* `HCCL_BUFFSIZE`：HCCL通信层单次传输的最大缓冲区大小（单位MB），影响跨设备通信效率	130	* `HCCL_BUFFSIZE`：HCCL通信层单次传输的最大缓冲区大小（单位MB），影响跨设备通信效率
127	* `VLLM_USE_V1`：使用vLLM的V1 engine API（v1接口），当前只支持 v1 ，需设置为 '1'。	131	* `VLLM_USE_V1`：使用vLLM的V1 engine API（v1接口），当前只支持 v1 ，需设置为 '1'。
128	* `VLLM_VERSION`：指定使用的vLLM版本号	132	* `VLLM_VERSION`：指定使用的vLLM版本号
129	-* `VLLM_ENABLE_GRAPH_MODE`：启用昇腾torchair图模式优化（1=启用），提升执行效率
130	* `VLLM_ENABLE_TOPK_OPTIMZE`：使能vLLM TOPK性能优化	133	* `VLLM_ENABLE_TOPK_OPTIMZE`：使能vLLM TOPK性能优化
		134	+* `VLLM_ASCEND_ACL_OP_INIT_MODE`：vLLM aclop 初始化模式: 0: default, normal init.
131	* `TASK_QUEUE_ENABLE`：控制开启task_queue算子下发队列优化的等级，推荐设置为 '2' 使能 Level 2 优化。	135	* `TASK_QUEUE_ENABLE`：控制开启task_queue算子下发队列优化的等级，推荐设置为 '2' 使能 Level 2 优化。
132	* `CPU_AFFINITY_CONF`：指定使用绑核优化，推荐设置为 '1'。	136	* `CPU_AFFINITY_CONF`：指定使用绑核优化，推荐设置为 '1'。
133	-* `LCAL_COMM_ID`: 开启coc特性时配套启用，设置为'127.0.0.1:27001'。	137	+* `LCAL_COMM_ID`: 开启coc特性时配套启用，设置为'127.0.0.1:27001'。
		138	+* `GLOO_SOCKET_IFNAME`：指定 GLOO 框架通信网卡。
		139	+* `TP_SOCKET_IFNAME`：指定 TP 相关通信网卡。
		140	+* `HCCL_SOCKET_IFNAME`：指定 HCCL 通信网卡。

Mdocs/install_guide.md+5-4

	@@ -84,18 +84,19 @@ source /usr/local/Ascend/nnal/atb/set_env.sh
84	### vllm及相关依赖安装：	84	### vllm及相关依赖安装：
85	（注：环境中需要安装git，因为vllm的安装过程依赖git）	85	（注：环境中需要安装git，因为vllm的安装过程依赖git）
86	```shell	86	```shell
87	-git clone https://github.com/vllm-project/vllm.git	87	+git clone -b releases/v0.9.1 https://github.com/vllm-project/vllm.git
88	cd vllm	88	cd vllm
89	-git checkout 5bc1ad6cee754405464a9957e86cf3a9302e4986	89	+git checkout b6553be1bc75f046b00046a4ad7576364d03c835
90	VLLM_TARGET_DEVICE=empty pip install .	90	VLLM_TARGET_DEVICE=empty pip install .
91	cd ..	91	cd ..
92	```	92	```
93		93
94	### vllm_ascend安装	94	### vllm_ascend安装
95	```shell	95	```shell
96	-git clone -b main https://github.com/vllm-project/vllm-ascend.git	96	+git clone -b v0.9.1-dev https://github.com/vllm-project/vllm-ascend.git
97	cd vllm-ascend	97	cd vllm-ascend
98	-git checkout c46632439a59dd43f9062396128119b143561e8a	98	+git checkout dedace4c
		99	+pip install -r requirements.txt
99	pip install -e .	100	pip install -e .
100	```	101	```
101		102

Adocs/solutions/r1_zero_deepseek_671b.md+111-0

@@ -0,0 +1,111 @@
	1	+# DeepSeek-R1-Zero-671B
	2	+R1-Zero模型是使用base模型，基于GPRO+规则奖励打分进行训练，本篇工作使用DeepSeek-671B Base模型复现DeepSeek-R1-Zero在Math领域的工作。
	3	+
	4	+
	5	+## 复现效果
	6	+### 训练细节
	7	+
	8	+我们使用DeepSeek-671B Base模型在deepscaler数据集上训练，使用标准的格式奖励和准确性奖励，训练超参如下：
	9	+
	10	+\| 迭代 \| 学习率 \| gbs \| 采样数 \| 温度 \| kl-coef \| 输入长度 \| 输出长度 \| 规则奖励 \| 奖励模型 \|
	11	+\|:----:\|:----:\|:----:\|:----:\|:----:\|:----:\|:----:\|:----:\|:----:\|:----:\|
	12	+\| 100 \| 1e-6 (constant) \| 384 \| 32 \| 1.0 \| 0.001 \| 1024 \| 2048 \| base_acc \| - \|
	13	+
	14	+训练过程记录如下：
	15	+
	16	+Reward曲线：
	17	+
	18	+![](../../sources/images/r1_zero/deepseek-671b-reward-curses.png)
	19	+
	20	+Response length曲线：
	21	+
	22	+![](../../sources/images/r1_zero/deepseek-671b-response-curses.png)
	23	+
	24	+
	25	+## 环境配置
	26	+配置MindSpeed-RL基础环境以及准备代码，参考[安装指南](../install_guide.md)
	27	+
	28	+## 模型选择
	29	+* DeepSeek-671B-Base [[下载]](https://huggingface.co/deepseek-ai/DeepSeek-V3)
	30	+该模型指令遵从度高，有一定概率能引导模型输出`<think>...</think><answer>...$\boxed{}</answer>`格式回复，训练曲线符合预期。
	31	+
	32	+### 权重转换
	33	+在进行RL训练之前，模型需要从HuggingFace权重转换为megatron权重，可参考[权重转换部分](../algorithms/grpo.md)
	34	+
	35	+## 模板构造
	36	+
	37	+* R1-Zero复现需要在数据处理时加上prompt模板激发`<think>...</think><answer>...$\boxed{}</answer>`
	38	+ ```
	39	+ "A conversation between User and Assistant. The user asks a question, and the Assistant solves it. The assistant first thinks about the reasoning process in the mind and then provides the user with the answer. The reasoning process and answer are enclosed within <think> </think> and <answer> </answer> tags, respectively, i.e., <think> reasoning process here </think><answer> answer here </answer>. Put your final answer within \\boxed{}. <｜User｜>{{content}}<｜Assistant｜>"
	40	+ ```
	41	+
	Z Z zhangjianxiang42025年7月10日已修改
	42	+
	43	+* 以上为默认的deepseek3_r1模板，根据模型和数据的不同，用户可以在`configs/model/templates.json`添加自己的自定义模板
	44	+
	45	+
	46	+## 数据集
	47	+对于deepseekv3-671B模型我们使用DeepScaler 40K来训练
	48	+
	49	+* [DeepScaler](https://huggingface.co/datasets/agentica-org/DeepScaleR-Preview-Dataset/tree/main)
	50	+
	51	+### 数据预处理
	52	+需要先配置数据处理的yaml文件(configs/datasets/deepscaler.yaml)
	53	+自定义数据集需要设置--map-keys映射，或重写自定义handler；具体参考[数据集处理部分](../algorithms/grpo.md)
	54	+
	55	+
	56	+DeepSeek-671B
	57	+* 处理的时候默认使用deepseek3_r1的模板
	58	+
	59	+ ```shell
	60	+ # 启动转换
	61	+ bash examples/data/preprocess_data.sh deepscaler
	62	+ ```
	63	+
	64	+## 打分器
	65	+DeepSeek-R1-Zero训练的过程中仅使用了基于程序的打分器而没有使用ORM，我们在数学领域上的打分逻辑分为以下几个部分：
	66	+
	67	+![](../../sources/images/r1_zero/rule_reward.png)
	68	+
	69	+## 训练
	70	+
	71	+### 配置准备
	72	+
	73	+模型结构的配置文件位于configs/model下，训练配置文件位于configs/目录下，我们以deepseek-671b的A3配置为例[grpo_deepseek_r1_671b_A3.yaml]，该配置用到了384die。[参数配置具体含义参考](../features/grpo_yaml.md)
	N N ningbenzhe12025年7月10日 [grpo_deepseek_r1_671b_A3.yaml]中括号去掉
	74	+
	75	+### 手动启动训练
	76	+与基于ray的其他强化训练一样，我们多机需要先在主节点初始化ray：
	77	+
	78	+```shell
	79	+# 创建一个集群，端口6344，dashboard端口8260
	80	+ray start --head --port 6344 --dashboard-host=0.0.0.0 --dashboard-port=8260
	81	+```
	82	+
	83	+随后，在其他节点加入主节点的集群：
	84	+```shell
	85	+# IP_ADDRESS 处填写主节点 IP 地址
	86	+ray start --address="IP_ADDRESS:6344"
	87	+```
	88	+
	N N ningbenzhe12025年7月10日这个地方补充一下环境变量配置的修改点，dp_size
	89	+最后，在主节点上启动训练：
	90	+```shell
	91	+export HCCL_CONNECT_TIMEOUT=1800
	92	+export CUDA_DEVICE_MAX_CONNECTIONS=1
	93	+
	94	+python cli/train_grpo.py --config-name grpo_deepseek_r1_671b_A3.yaml \| tee logs/r1_zero_deepseek_671b_full.log
	95	+```
	96	+
	97	+### 脚本启动训练
	98	+
	99	+[参数配置具体含义参考](../algorithms/grpo.md)
	100	+```shell
	101	+# 主节点在脚本文件中修改节点数、每个节点的卡数以及主节点的IP地址
	102	+bash examples/grpo/grpo_trainer_deepseek_r1_671b.sh
	103	+```
	104	+
	105	+```shell
	106	+# 其余子节点在脚本文件中修改节点数、每个节点的卡数以及主节点的IP地址
	107	+bash examples/grpo/grpo_trainer_deepseek_r1_671b.sh
	108	+```
	109	+
	110	+
	111	+*注意：所有节点的代码、权重、数据等路径的层级要保持一致，且启动ray的时候都位于MindSpeed-RL目录下*

Mexamples/dapo/dapo_trainer_qwen25_32b.sh+6-3

	@@ -3,6 +3,7 @@ ray stop --force
3	export RAY_DEDUP_LOGS=0	3	export RAY_DEDUP_LOGS=0
4	export HYDRA_FULL_ERROR=1	4	export HYDRA_FULL_ERROR=1
5		5
		6	+SOCKET_IFNAME="Your SOCKET IFNAME"
6	DEFAULT_YAML="dapo_qwen25_32b_A3"	7	DEFAULT_YAML="dapo_qwen25_32b_A3"
7	YAML=${1:-$DEFAULT_YAML}	8	YAML=${1:-$DEFAULT_YAML}
8	echo "Use $YAML"	9	echo "Use $YAML"
	@@ -19,9 +20,11 @@ export LCAL_COMM_ID=127.0.0.1:27001
19	NNODES=1	20	NNODES=1
20	NPUS_PER_NODE=16	21	NPUS_PER_NODE=16
21	#修改为对应主节点IP	22	#修改为对应主节点IP
22	-MASTER_ADDR="localhost"	23	+MASTER_ADDR="IP FOR MASTER NODE"
23	-#获取当前机器IP	24	+#修改为当前节点的通信网卡
24	-CURRENT_IP=$(ip -4 addr show $(ip -o -4 route show to default \| awk '{print $5}') \| grep -oP '(?<=inet\s)\d+(\.\d+){3}')	25	+SOCKET_IFNAME="SOCKET IFNAME FOR CURRENT NODE"
		26	+#获取当前节点IP
		27	+CURRENT_IP=$(ifconfig $SOCKET_IFNAME \| grep -Eo 'inet (addr:)?([0-9]{1,3}\.){3}[0-9]{1,3}' \| awk '{print $NF}')
25		28
26	if [ "$MASTER_ADDR" = "$CURRENT_IP" ]; then	29	if [ "$MASTER_ADDR" = "$CURRENT_IP" ]; then
27	# 主节点启动	30	# 主节点启动

Aexamples/grpo/grpo_trainer_deepseek_r1_671b.sh+68-0

@@ -0,0 +1,68 @@
	1	+pkill -9 python
	2	+ray stop --force
	3	+export RAY_DEDUP_LOGS=0
	4	+export HYDRA_FULL_ERROR=1
	5	+
	6	+SOCKET_IFNAME="Your SOCKET IFNAME"
	7	+DEFAULT_YAML="grpo_deepseek_r1_671b_A3"
	8	+YAML=${1:-$DEFAULT_YAML}
	9	+echo "Use $YAML"
	10	+
	11	+ulimit -n 32768
	12	+mkdir logs
	13	+
	14	+export TASK_QUEUE_ENABLE=2
	15	+export HCCL_IF_BASE_PORT=24703
	16	+export LD_PRELOAD=/usr/local/lib/libjemalloc.so.2
	17	+export MALLOC_MMAP_THRESHOLD_=512768
	18	+export LCAL_COMM_ID=127.0.0.1:27001
	19	+
	20	+
	21	+NNODES=24
	22	+NPUS_PER_NODE=16
	23	+#修改为对应主节点IP
	24	+MASTER_ADDR="IP FOR MASTER NODE"
	25	+#修改为当前节点的通信网卡
	26	+SOCKET_IFNAME="SOCKET IFNAME FOR CURRENT NODE"
	27	+#获取当前节点IP
	28	+CURRENT_IP=$(ifconfig $SOCKET_IFNAME \| grep -Eo 'inet (addr:)?([0-9]{1,3}\.){3}[0-9]{1,3}' \| awk '{print $NF}')
	29	+if [ "$MASTER_ADDR" = "$CURRENT_IP" ]; then
	30	+ # 主节点启动
	31	+ ray start --head --port 6766 --dashboard-host=0.0.0.0 --node-ip-address=$CURRENT_IP --dashboard-port=8260 --resources='{"NPU": '$NPUS_PER_NODE'}'
	32	+
	33	+ while true; do
	34	+ ray_status_output=$(ray status)
	35	+ npu_count=$(echo "$ray_status_output" \| grep -oP '(?<=/)\d+\.\d+(?=\s*NPU)' \| head -n 1)
	36	+ npu_count_int=$(echo "$npu_count" \| awk '{print int($1)}')
	37	+ device_count=$((npu_count_int / $NPUS_PER_NODE))
	38	+
	39	+ # 判断 device_count 是否与 NNODES 相等
	40	+ if [ "$device_count" -eq "$NNODES" ]; then
	41	+ echo "Ray cluster is ready with $device_count devices (from $npu_count NPU resources), starting Python script."
	42	+ ray status
	43	+ python cli/train_grpo.py --config-name $YAML 2>&1 \| tee logs/training.log
	44	+ break
	45	+ else
	46	+ echo "Waiting for Ray to allocate $NNODES devices. Current device count: $device_count"
	47	+ sleep 5
	48	+ fi
	49	+ done
	50	+else
	51	+ # 子节点尝试往主节点注册ray直到成功
	52	+ while true; do
	53	+ # 尝试连接 Ray 集群
	54	+ ray start --address="$MASTER_ADDR:6766" --resources='{"NPU": '$NPUS_PER_NODE'}' --node-ip-address=$CURRENT_IP
	55	+
	56	+ # 检查连接是否成功
	57	+ ray status
	58	+ if [ $? -eq 0 ]; then
	59	+ echo "Successfully connected to the Ray cluster!"
	60	+ break
	61	+ else
	62	+ echo "Failed to connect to the Ray cluster. Retrying in 5 seconds..."
	63	+ sleep 5
	64	+ fi
	65	+ done
	66	+fi
	67	+
	68	+sleep 999999

Mexamples/grpo/grpo_trainer_qwen25_32b.sh+5-3

	@@ -19,9 +19,11 @@ export LCAL_COMM_ID=127.0.0.1:27001
19	NNODES=1	19	NNODES=1
20	NPUS_PER_NODE=16	20	NPUS_PER_NODE=16
21	#修改为对应主节点IP	21	#修改为对应主节点IP
22	-MASTER_ADDR="localhost"	22	+MASTER_ADDR="IP FOR MASTER NODE"
23	-#获取当前机器IP	23	+#修改为当前节点的通信网卡
24	-CURRENT_IP=$(ip -4 addr show $(ip -o -4 route show to default \| awk '{print $5}') \| grep -oP '(?<=inet\s)\d+(\.\d+){3}')	24	+SOCKET_IFNAME="SOCKET IFNAME FOR CURRENT NODE"
		25	+#获取当前节点IP
		26	+CURRENT_IP=$(ifconfig $SOCKET_IFNAME \| grep -Eo 'inet (addr:)?([0-9]{1,3}\.){3}[0-9]{1,3}' \| awk '{print $NF}')
25		27
26	if [ "$MASTER_ADDR" = "$CURRENT_IP" ]; then	28	if [ "$MASTER_ADDR" = "$CURRENT_IP" ]; then
27	# 主节点启动	29	# 主节点启动

Mexamples/grpo/grpo_trainer_qwen25_7b.sh+6-3

	@@ -3,6 +3,7 @@ ray stop --force
3	export RAY_DEDUP_LOGS=0	3	export RAY_DEDUP_LOGS=0
4	export HYDRA_FULL_ERROR=1	4	export HYDRA_FULL_ERROR=1
5		5
		6	+SOCKET_IFNAME="Your SOCKET IFNAME"
6	DEFAULT_YAML="grpo_qwen25_7b_A3"	7	DEFAULT_YAML="grpo_qwen25_7b_A3"
7	YAML=${1:-$DEFAULT_YAML}	8	YAML=${1:-$DEFAULT_YAML}
8	echo "Use $YAML"	9	echo "Use $YAML"
	@@ -16,9 +17,11 @@ export HCCL_IF_BASE_PORT=24703
16	NNODES=1	17	NNODES=1
17	NPUS_PER_NODE=16	18	NPUS_PER_NODE=16
18	#修改为对应主节点IP	19	#修改为对应主节点IP
19	-MASTER_ADDR="localhost"	20	+MASTER_ADDR="IP FOR MASTER NODE"
20	-#获取当前机器IP	21	+#修改为当前节点的通信网卡
21	-CURRENT_IP=$(ip -4 addr show $(ip -o -4 route show to default \| awk '{print $5}') \| grep -oP '(?<=inet\s)\d+(\.\d+){3}')	22	+SOCKET_IFNAME="SOCKET IFNAME FOR CURRENT NODE"
		23	+#获取当前节点IP
		24	+CURRENT_IP=$(ifconfig $SOCKET_IFNAME \| grep -Eo 'inet (addr:)?([0-9]{1,3}\.){3}[0-9]{1,3}' \| awk '{print $NF}')
22		25
23	if [ "$MASTER_ADDR" = "$CURRENT_IP" ]; then	26	if [ "$MASTER_ADDR" = "$CURRENT_IP" ]; then
24	# 主节点启动	27	# 主节点启动

Mmindspeed_rl/config_cls/generate_config.py+7-4

	@@ -22,12 +22,14 @@ class GenerateConfig(BaseConfig):
22		22
23	max_num_seqs: Maximum number of sequences to process simultaneously. Default is 256.	23	max_num_seqs: Maximum number of sequences to process simultaneously. Default is 256.
24	max_model_len: Maximum model length (in tokens). Default is 2048.	24	max_model_len: Maximum model length (in tokens). Default is 2048.
		25	+ max_num_batched_tokens: The maximum number of tokens model can run in a single batch. Default is 2048.
25	dtype: Data type for model weights. Default is "bfloat16".	26	dtype: Data type for model weights. Default is "bfloat16".
26	gpu_memory_utilization: GPU memory utilization factor. Default is 0.5.	27	gpu_memory_utilization: GPU memory utilization factor. Default is 0.5.
27		28
28	- enforce_eager: Whether to always use eager-mode PyTorch. If True, we will disable ACL graph and always execute the model in eager mode.	29	+ enforce_eager: Whether to always use eager-mode PyTorch. If True, we will disable ACL graph and always execute the model in eager mode.
29	- If False, we will use ACL graph and eager execution in hybrid for maximal performance and flexibility.	30	+ If False, we will use ACL graph and eager execution in hybrid for maximal performance and flexibility.
30	-	31	+ torchair_graph: Whether to enable TorchAir graph optimization. If True, uses accelerated computational graph optimizations.
		32	+ enable_expert_parallel: Whether to enable expert parallel computation for Mixture-of-Experts (MoE) layers.
31	sampling_config: Configuration for text generation sampling. Default values are set for various sampling parameters.	33	sampling_config: Configuration for text generation sampling. Default values are set for various sampling parameters.
32	- num_completions: The number of independent completions to generate for each input prompt. Default is 1.	34	- num_completions: The number of independent completions to generate for each input prompt. Default is 1.
33	- logprobs: The number of top tokens to return log probabilities for. Default is 1.	35	- logprobs: The number of top tokens to return log probabilities for. Default is 1.
	@@ -81,6 +83,7 @@ class GenerateConfig(BaseConfig):
81	self.enable_prefix_caching = False	83	self.enable_prefix_caching = False
82	self.num_scheduler_steps = 1	84	self.num_scheduler_steps = 1
83	self.enforce_eager = True	85	self.enforce_eager = True
		86	+ self.torchair_graph = False
84	self.enable_expert_parallel = False	87	self.enable_expert_parallel = False
85		88
86	# 采样配置的默认值，用于生成文本时的采样策略设置	89	# 采样配置的默认值，用于生成文本时的采样策略设置
	@@ -99,7 +102,7 @@ class GenerateConfig(BaseConfig):
99	for key, _ in config_dict["sampling_config"].items():	102	for key, _ in config_dict["sampling_config"].items():
100	if key not in self.sampling_config:	103	if key not in self.sampling_config:
101	raise ValueError(f"The key: {key} is missing, causing the setup to fail. Please check."	104	raise ValueError(f"The key: {key} is missing, causing the setup to fail. Please check."
102	- f" If necessary, register it in the config file.")	105	+ f" If necessary, register it in the config file.")
103		106
104	# 如果提供了配置字典，则更新默认值	107	# 如果提供了配置字典，则更新默认值
105	self.update(config_dict)	108	self.update(config_dict)

Mmindspeed_rl/config_cls/megatron_config.py+2-0

	@@ -207,6 +207,7 @@ class MegatronConfig(BaseConfig):
207	coc_mode: 0=original, 1=rewrite, 2=coc default	207	coc_mode: 0=original, 1=rewrite, 2=coc default
208	coc_parallel_num: number of parallel in CoC features (default: 1)	208	coc_parallel_num: number of parallel in CoC features (default: 1)
209	coc_fused_kernel: switch to use fused kernel in CoC (default: False)	209	coc_fused_kernel: switch to use fused kernel in CoC (default: False)
		210	+ swap_optimizer: switch to swap-optimizer feature (default: False)
210	mm_model: config for multimodal models	211	mm_model: config for multimodal models
211	'''	212	'''
212		213
	@@ -379,6 +380,7 @@ class MegatronConfig(BaseConfig):
379	self.coc_mode = -1	380	self.coc_mode = -1
380	self.coc_parallel_num = 1	381	self.coc_parallel_num = 1
381	self.coc_fused_kernel = False	382	self.coc_fused_kernel = False
		383	+ self.swap_optimizer = False
382		384
383	# used for multimodal models	385	# used for multimodal models
384	self.mm_model = None	386	self.mm_model = None

Mmindspeed_rl/config_cls/rl_config.py+1-0

	@@ -125,6 +125,7 @@ class RLConfig(BaseConfig):
125		125
126	self.use_dynamic_bsz = False	126	self.use_dynamic_bsz = False
127	self.max_packing_token_size = 4096	127	self.max_packing_token_size = 4096
		128	+ self.log_max_throughput = True
128	self.dynamic_max_batch_size = None	129	self.dynamic_max_batch_size = None
129		130
130	# token level loss	131	# token level loss

Mmindspeed_rl/config_cls/validate_config.py+0-5

	@@ -80,11 +80,6 @@ def validate_rl_args(
80	"'multimodal' models cannot use 'use_remove_padding' feature! "	80	"'multimodal' models cannot use 'use_remove_padding' feature! "
81	"Please set 'use_remove_padding=False' in the RLConfig.")	81	"Please set 'use_remove_padding=False' in the RLConfig.")
82		82
83	- # 校验图模式配置
84	- if not generate_config.enforce_eager:
85	- raise ValueError(
86	- "'enforce eager' feature is not available to be False at present.")
87	-
88	# 校验资源分配合理性	83	# 校验资源分配合理性
89	def _validate_resource(resource, t_size, p_size, c_size, component):	84	def _validate_resource(resource, t_size, p_size, c_size, component):
90	product = t_size * p_size * c_size	85	product = t_size * p_size * c_size

Mmindspeed_rl/models/base/base_inference_engine.py+4-1

	@@ -25,7 +25,8 @@ class BaseInferEngine(ABC):
25	max_model_len: int = 2048, # Default value set to 2048	25	max_model_len: int = 2048, # Default value set to 2048
26	dtype: str = "bfloat16", # Default value set to "bfloat16"	26	dtype: str = "bfloat16", # Default value set to "bfloat16"
27	gpu_memory_utilization: float = 0.5, # Default value set to 0.5	27	gpu_memory_utilization: float = 0.5, # Default value set to 0.5
28	- trust_remote_code: bool = True	28	+ trust_remote_code: bool = True,
		29	+ enable_expert_parallel: bool = False,
29	):	30	):
30	"""	31	"""
31	Initialize the base inference engine.	32	Initialize the base inference engine.
	@@ -44,6 +45,7 @@ class BaseInferEngine(ABC):
44	dtype (str): Data type for model weights. Default is "bfloat16".	45	dtype (str): Data type for model weights. Default is "bfloat16".
45	gpu_memory_utilization (float): GPU memory utilization factor. Default is 0.5.	46	gpu_memory_utilization (float): GPU memory utilization factor. Default is 0.5.
46	trust_remote_code (bool): Whether to trust remote code (e.g., for custom tokenizers).	47	trust_remote_code (bool): Whether to trust remote code (e.g., for custom tokenizers).
		48	+ enable_expert_parallel (bool): Whether to enable expert parallel.
47	"""	49	"""
48	self.tokenizer_name_or_path = tokenizer_name_or_path	50	self.tokenizer_name_or_path = tokenizer_name_or_path
49	self.prompt_type = prompt_type	51	self.prompt_type = prompt_type
	@@ -60,6 +62,7 @@ class BaseInferEngine(ABC):
60	self.dtype = dtype	62	self.dtype = dtype
61	self.gpu_memory_utilization = gpu_memory_utilization	63	self.gpu_memory_utilization = gpu_memory_utilization
62	self.trust_remote_code = trust_remote_code	64	self.trust_remote_code = trust_remote_code
		65	+ self.enable_expert_parallel = enable_expert_parallel
63		66
64		67
65	@abstractmethod	68	@abstractmethod

Amindspeed_rl/models/rollout/vllm_adapter/engine_core.py+57-0

@@ -0,0 +1,57 @@
	1	+import time
	2	+
	3	+from vllm.config import VllmConfig
	4	+from vllm.v1.core.kv_cache_utils import get_kv_cache_config, unify_kv_cache_configs
	5	+from vllm.v1.engine.core import EngineCore
	6	+from vllm.v1.kv_cache_interface import KVCacheConfig
	7	+from vllm.logger import init_logger
	8	+
	9	+logger = init_logger(__name__)
	10	+
	11	+
	12	+def _initialize_kv_caches(
	13	+ self, vllm_config: VllmConfig) -> tuple[int, int, KVCacheConfig]:
	14	+ start = time.time()
	15	+
	16	+ # Get all kv cache needed by the model
	17	+ kv_cache_specs = self.model_executor.get_kv_cache_specs()
	18	+
	19	+ # Profiles the peak memory usage of the model to determine how much
	20	+ # memory can be allocated for kv cache.
	21	+ available_gpu_memory = self.model_executor.determine_available_memory()
	22	+
	23	+ assert len(kv_cache_specs) == len(available_gpu_memory)
	24	+ # Get the kv cache tensor size
	25	+ self.kv_cache_configs = [
	26	+ get_kv_cache_config(vllm_config, kv_cache_spec_one_worker,
	27	+ available_gpu_memory_one_worker)
	28	+ for kv_cache_spec_one_worker, available_gpu_memory_one_worker in
	29	+ zip(kv_cache_specs, available_gpu_memory)
	30	+ ]
	31	+
	32	+ # Since we use a shared centralized controller, we need the
	33	+ # `kv_cache_config` to be consistent across all workers to make sure
	34	+ # all the memory operators can be applied to all workers.
	35	+ unify_kv_cache_configs(self.kv_cache_configs)
	36	+
	37	+ # All workers have the same kv_cache_config except layer names, so use
	38	+ # an arbitrary one to initialize the scheduler.
	39	+ assert all([
	40	+ cfg.num_blocks == self.kv_cache_configs[0].num_blocks
	41	+ for cfg in self.kv_cache_configs
	42	+ ])
	43	+ num_gpu_blocks = self.kv_cache_configs[0].num_blocks
	44	+ num_cpu_blocks = 0
	45	+ scheduler_kv_cache_config = self.kv_cache_configs[0]
	46	+
	47	+ # Initialize kv cache and warmup the execution
	48	+ self.model_executor.initialize_from_config(self.kv_cache_configs)
	49	+
	50	+ elapsed = time.time() - start
	51	+ logger.info(("init engine (profile, create kv cache, "
	52	+ "warmup model) took %.2f seconds"), elapsed)
	53	+ return num_gpu_blocks, num_cpu_blocks, scheduler_kv_cache_config
	54	+
	55	+
	56	+EngineCore._initialize_kv_caches = _initialize_kv_caches
	57	+

Mmindspeed_rl/models/rollout/vllm_adapter/megatron_weight_loaders.py+1-0

	@@ -234,6 +234,7 @@ MODEL_MEGATRON_WEIGHT_LOADER_REGISTRY = {
234	"CustomQwen3MoeForCausalLM": qwen_megatron_weight_loader,	234	"CustomQwen3MoeForCausalLM": qwen_megatron_weight_loader,
235	"DeepseekV3ForCausalLM": deepseek_megatron_weight_loader,	235	"DeepseekV3ForCausalLM": deepseek_megatron_weight_loader,
236	"DeepseekV2ForCausalLM": deepseek_megatron_weight_loader,	236	"DeepseekV2ForCausalLM": deepseek_megatron_weight_loader,
		237	+ "CustomDeepseekV2ForCausalLM": deepseek_megatron_weight_loader,
237	"CustomDeepseekV3ForCausalLM": deepseek_megatron_weight_loader,	238	"CustomDeepseekV3ForCausalLM": deepseek_megatron_weight_loader,
238	"Qwen2_5_VLForConditionalGeneration": qwen_vl_megatron_weight_loader	239	"Qwen2_5_VLForConditionalGeneration": qwen_vl_megatron_weight_loader
239	}	240	}

Mmindspeed_rl/models/rollout/vllm_adapter/vllm_parallel_state.py+24-99

	@@ -4,18 +4,12 @@
4		4
5	"""Model and data parallel groups."""	5	"""Model and data parallel groups."""
6	import os	6	import os
7	-import re
8	-import socket
9	-import subprocess
10	-from datetime import timedelta
11	from typing import Optional	7	from typing import Optional
12		8
13	import torch	9	import torch
14	-import torch.distributed as dist
15	import vllm.distributed.parallel_state as ps	10	import vllm.distributed.parallel_state as ps
16	import vllm_ascend.distributed.parallel_state as ascend_ps	11	import vllm_ascend.distributed.parallel_state as ascend_ps
17	import vllm.envs as envs	12	import vllm.envs as envs
18	-from vllm.config import get_current_vllm_config
19		13
20	from vllm.distributed.parallel_state import (	14	from vllm.distributed.parallel_state import (
21	get_pp_group,	15	get_pp_group,
	@@ -25,6 +19,7 @@ from vllm.distributed.parallel_state import (
25	)	19	)
26		20
27	from mindspeed_rl.utils.loggers import Loggers	21	from mindspeed_rl.utils.loggers import Loggers
		22	+from mindspeed_rl.utils.utils import get_cluster_info
28		23
29	logger = Loggers(__name__)	24	logger = Loggers(__name__)
30		25
	@@ -66,7 +61,6 @@ def initialize_parallel_state(
66	infer_pipeline_model_parallel_size: int = 1,	61	infer_pipeline_model_parallel_size: int = 1,
67	train_pipeline_model_parallel_size: int = 1,	62	train_pipeline_model_parallel_size: int = 1,
68	infer_expert_tensor_parallel_size: int = 1,	63	infer_expert_tensor_parallel_size: int = 1,
69	- train_expert_tensor_parallel_size: int = 1,
70	train_expert_model_parallel_size: int = 1,	64	train_expert_model_parallel_size: int = 1,
71	infer_expert_model_parallel_size: int = 1,	65	infer_expert_model_parallel_size: int = 1,
72	train_context_model_parallel_size: int = 1,	66	train_context_model_parallel_size: int = 1,
	@@ -81,8 +75,7 @@ def initialize_parallel_state(
81	world_size = int(os.getenv("WORLD_SIZE", "-1"))	75	world_size = int(os.getenv("WORLD_SIZE", "-1"))
82	if world_size == -1:	76	if world_size == -1:
83	raise ValueError("The world_size is set to -1, not initialized by TORCHRUN")	77	raise ValueError("The world_size is set to -1, not initialized by TORCHRUN")
84	- config = get_current_vllm_config()	78	+
85	- config.parallel_config.tensor_parallel_size = infer_tensor_model_parallel_size
86	init_distributed_environment(world_size, rank, distributed_init_method, local_rank, backend)	79	init_distributed_environment(world_size, rank, distributed_init_method, local_rank, backend)
87	if torch.distributed.get_world_size() > 1:	80	if torch.distributed.get_world_size() > 1:
88	# NOTE: build a sepearate inference group with infer tp & micro dp	81	# NOTE: build a sepearate inference group with infer tp & micro dp
	@@ -92,7 +85,6 @@ def initialize_parallel_state(
92	infer_pipeline_model_parallel_size=infer_pipeline_model_parallel_size,	85	infer_pipeline_model_parallel_size=infer_pipeline_model_parallel_size,
93	train_pipeline_model_parallel_size=train_pipeline_model_parallel_size,	86	train_pipeline_model_parallel_size=train_pipeline_model_parallel_size,
94	infer_expert_tensor_parallel_size=infer_expert_tensor_parallel_size,	87	infer_expert_tensor_parallel_size=infer_expert_tensor_parallel_size,
95	- train_expert_tensor_parallel_size=train_expert_tensor_parallel_size,
96	train_expert_model_parallel_size=train_expert_model_parallel_size,	88	train_expert_model_parallel_size=train_expert_model_parallel_size,
97	infer_expert_model_parallel_size=infer_expert_model_parallel_size,	89	infer_expert_model_parallel_size=infer_expert_model_parallel_size,
98	train_context_model_parallel_size=train_context_model_parallel_size	90	train_context_model_parallel_size=train_context_model_parallel_size
	@@ -107,11 +99,9 @@ def initialize_model_parallel_for_vllm(
107	infer_pipeline_model_parallel_size: int = 1,	99	infer_pipeline_model_parallel_size: int = 1,
108	train_pipeline_model_parallel_size: int = 1,	100	train_pipeline_model_parallel_size: int = 1,
109	infer_expert_tensor_parallel_size: int = 1,	101	infer_expert_tensor_parallel_size: int = 1,
110	- train_expert_tensor_parallel_size: int = 1,
111	train_expert_model_parallel_size: int = 1,	102	train_expert_model_parallel_size: int = 1,
112	infer_expert_model_parallel_size: int = 1,	103	infer_expert_model_parallel_size: int = 1,
113	train_context_model_parallel_size: int = 1,	104	train_context_model_parallel_size: int = 1,
114	- num_process: int = 1,
115	rebulid_EP_group: bool = False	105	rebulid_EP_group: bool = False
116	) -> None:	106	) -> None:
117		107
	@@ -230,15 +220,10 @@ def initialize_model_parallel_for_vllm(
230	if config is not None:	220	if config is not None:
231	data_parallel_size = config.parallel_config.data_parallel_size	221	data_parallel_size = config.parallel_config.data_parallel_size
232		222
233	- num_expert_parallel_groups: int = infer_expert_tensor_parallel_size
234	num_expert_tensor_parallel_groups: int = world_size // infer_expert_tensor_parallel_size	223	num_expert_tensor_parallel_groups: int = world_size // infer_expert_tensor_parallel_size
235		224
236	- num_rank_per_process = world_size // num_process
237	- all_ranks = list(range(world_size))
238	-
239	global _EP	225	global _EP
240	assert _EP is None, ("expert parallel group is already initialized")	226	assert _EP is None, ("expert parallel group is already initialized")
241	- group_ranks = []
242		227
243	if rebulid_EP_group:	228	if rebulid_EP_group:
244	# 重新建组	229	# 重新建组
	@@ -272,7 +257,6 @@ def initialize_model_parallel_for_vllm(
272		257
273	else:	258	else:
274	# 保序	259	# 保序
275	- group_ranks = []
276	tensor_model_parallel_size = infer_tensor_model_parallel_size	260	tensor_model_parallel_size = infer_tensor_model_parallel_size
277	context_parallel_size = 1	261	context_parallel_size = 1
278	expert_model_parallel_size = infer_expert_model_parallel_size	262	expert_model_parallel_size = infer_expert_model_parallel_size
	@@ -316,34 +300,33 @@ def initialize_model_parallel_for_vllm(
316	backend,	300	backend,
317	group_name="etp")	301	group_name="etp")
318		302
319	- if data_parallel_size > 1:	303	+ global _DP
320	- global _DP	304	+ assert _DP is None, ("data parallel group is already initialized")
321	- assert _DP is None, ("data parallel group is already initialized")	305	+ dp_group_ranks = torch.tensor(tp_group_ranks).transpose(0, 1).reshape(-1, data_parallel_size).unbind(0)
322	- dp_group_ranks = torch.tensor(tp_group_ranks).transpose(0, 1).reshape(-1, data_parallel_size).unbind(0)	306	+ group_ranks = [x.tolist() for x in dp_group_ranks]
323	- group_ranks = [x.tolist() for x in dp_group_ranks]	307	+ logger.info(f"DP rank: {group_ranks}")
324	- logger.info(f"DP rank: {group_ranks}")
325		308
326	- ps._DP = init_model_parallel_group(group_ranks,	309	+ ps._DP = init_model_parallel_group(group_ranks,
327	- get_world_group().local_rank,	310	+ get_world_group().local_rank,
328	- backend,	311	+ backend,
329	- group_name="dp")	312	+ group_name="dp")
330		313
331	- os.environ["VLLM_DP_RANK"] = str(ps._DP.rank_in_group)	314	+ os.environ["VLLM_DP_RANK"] = str(ps._DP.rank_in_group)
332	- envs.VLLM_DP_RANK = int(os.environ["VLLM_DP_RANK"])	315	+ envs.VLLM_DP_RANK = int(os.environ["VLLM_DP_RANK"])
333	- ip_list = get_cluster_info()	316	+ ip_list = get_cluster_info()
334		317
335	- for index, group_rank in enumerate(group_ranks):	318	+ for index, group_rank in enumerate(group_ranks):
336	- if torch.distributed.get_rank() in group_rank:	319	+ if torch.distributed.get_rank() in group_rank:
337	- os.environ["VLLM_DP_MASTER_PORT"] = str(	320	+ os.environ["VLLM_DP_MASTER_PORT"] = str(
338	- int(os.environ.get("MASTER_PORT")) + 1 + index)	321	+ int(os.environ.get("MASTER_PORT")) + 1 + index)
339	- os.environ["VLLM_DP_MASTER_IP"] = ip_list[group_rank[0]]	322	+ os.environ["VLLM_DP_MASTER_IP"] = ip_list[group_rank[0]]
340		323
341	- envs.VLLM_DP_MASTER_IP = os.environ["VLLM_DP_MASTER_IP"]	324	+ envs.VLLM_DP_MASTER_IP = os.environ["VLLM_DP_MASTER_IP"]
342	- envs.VLLM_DP_MASTER_PORT = int(os.environ["VLLM_DP_MASTER_PORT"])	325	+ envs.VLLM_DP_MASTER_PORT = int(os.environ["VLLM_DP_MASTER_PORT"])
343	- os.environ["VLLM_PORT"] = os.environ["VLLM_DP_MASTER_PORT"]	326	+ os.environ["VLLM_PORT"] = os.environ["VLLM_DP_MASTER_PORT"]
344	- envs.VLLM_PORT = envs.VLLM_DP_MASTER_PORT	327	+ envs.VLLM_PORT = envs.VLLM_DP_MASTER_PORT
345		328
346	- logger.info(f"rank: {torch.distributed.get_rank()}>>>>>>VLLM_DP_MASTER_IP: {envs.VLLM_DP_MASTER_IP}, VLLM_DP_MASTER_PORT: {envs.VLLM_DP_MASTER_PORT}")	329	+ logger.info(f"rank: {torch.distributed.get_rank()}, VLLM_DP_MASTER_IP: {envs.VLLM_DP_MASTER_IP}, VLLM_DP_MASTER_PORT: {envs.VLLM_DP_MASTER_PORT}")
347		330
348		331
349	def initialize_model_parallel(	332	def initialize_model_parallel(
	@@ -414,61 +397,3 @@ def initialize_model_parallel(
414	)	397	)
415		398
416	ps._PP = _PP # for verl	399	ps._PP = _PP # for verl
417	-
418	-
419	-def get_cluster_info():
420	- # 确保分布式环境已初始化
421	- if not dist.is_initialized():
422	- raise RuntimeError("Distributed environment not initialized")
423	-
424	- world_size = dist.get_world_size()
425	-
426	- # 获取当前节点的IP地址
427	- ip_address = _get_current_node_ip()
428	-
429	- # 收集所有rank的IP地址
430	- ip_list = [None] * world_size
431	- dist.all_gather_object(ip_list, ip_address)
432	-
433	- return ip_list
434	-
435	-
436	-def _get_current_node_ip() -> str:
437	- try:
438	- # 创建一个 UDP 套接字（仅用于获取接口信息）
439	- with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as s:
440	- # 连接到一个外部地址（无需真实通信）
441	- s.connect(("8.8.8.8", 80)) # Google DNS 服务器
442	- local_ip = s.getsockname()[0]
443	- except Exception:
444	- local_ip = _get_ip_by_ifname()
445	- if not local_ip:
446	- # 如果失败，回退到遍历接口
447	- local_ip = "127.0.0.1"
448	- hostname = socket.gethostname()
449	- for addr in socket.getaddrinfo(hostname, None):
450	- ip = addr[4][0]
451	- if not ip.startswith("::"):
452	- local_ip = ip
453	- break
454	- return local_ip
455	-
456	-
457	-def _get_ip_by_ifname():
458	- """
459	- 通过接口名称（如 eth0、en0）获取 IPv4 地址
460	- 返回 IP 字符串，失败返回 None
461	- """
462	- try:
463	- # 执行 ifconfig 命令并捕获输出
464	- ifname = os.environ.get("HCCL_SOCKET_IFNAME", 0)
465	- if ifname:
466	- output = subprocess.check_output(["ifconfig", ifname], stderr=subprocess.STDOUT).decode()
467	- # 正则匹配 IPv4 地址（排除 127.0.0.1）
468	- matches = re.findall(r'inet (?:addr:)?((?:\d{1,3}\.){3}\d{1,3})', output)
469	- for ip in matches:
470	- if ip != "127.0.0.1":
471	- return ip
472	- return None
473	- except subprocess.CalledProcessError:
474	- return None

Mmindspeed_rl/models/rollout/vllm_engine.py+20-63

	@@ -15,21 +15,7 @@ from torch_npu.contrib import transfer_to_npu
15	from torch.nn.utils.rnn import pad_sequence	15	from torch.nn.utils.rnn import pad_sequence
16	from transformers import AutoConfig	16	from transformers import AutoConfig
17		17
18	-
19	-def dummy_compile(compile_args, *compile_kwargs):
20	- def decorate(fn):
21	- def wrapper(args, *kwargs):
22	- return fn(args, *kwargs)
23	-
24	- return wrapper
25	-
26	- return decorate
27	-
28	-torch.compile = dummy_compile
29	-torch.jit.script = dummy_compile
30	-
31	from vllm import LLM, SamplingParams	18	from vllm import LLM, SamplingParams
32	-from vllm.v1.core.kv_cache_utils import get_kv_cache_config, unify_kv_cache_configs
33	from mindspeed_rl.utils.loggers import Loggers	19	from mindspeed_rl.utils.loggers import Loggers
34	from mindspeed_rl.models.base.base_inference_engine import BaseInferEngine	20	from mindspeed_rl.models.base.base_inference_engine import BaseInferEngine
35	from mindspeed_rl.models.rollout.vllm_adapter.vllm_parallel_state import initialize_parallel_state	21	from mindspeed_rl.models.rollout.vllm_adapter.vllm_parallel_state import initialize_parallel_state
	@@ -56,18 +42,19 @@ class VLLMInferEngine(BaseInferEngine):
56	infer_pipeline_parallel_size: int,	42	infer_pipeline_parallel_size: int,
57	infer_expert_parallel_size: int,	43	infer_expert_parallel_size: int,
58	sampling_config: dict,	44	sampling_config: dict,
59	- infer_expert_tensor_parallel_size: int = 1,
60	prompt_type: str = None,	45	prompt_type: str = None,
61	prompt_type_path: str = None,	46	prompt_type_path: str = None,
62	enable_prefix_caching: bool = False,	47	enable_prefix_caching: bool = False,
63	num_scheduler_steps: int = 1,	48	num_scheduler_steps: int = 1,
64	max_num_seqs: int = 1,	49	max_num_seqs: int = 1,
65	max_model_len: int = 2048,	50	max_model_len: int = 2048,
		51	+ max_num_batched_tokens: int = 2048,
66	dtype: str = "bfloat16",	52	dtype: str = "bfloat16",
67	gpu_memory_utilization: float = 0.5,	53	gpu_memory_utilization: float = 0.5,
68	trust_remote_code: bool = True,	54	trust_remote_code: bool = True,
69	load_format: str = "megatron",	55	load_format: str = "megatron",
70	enforce_eager: bool = False,	56	enforce_eager: bool = False,
		57	+ torchair_graph: bool = False,
71	limit_mm_image_per_prompt: int = 1,	58	limit_mm_image_per_prompt: int = 1,
72	limit_mm_video_per_prompt: int = 0,	59	limit_mm_video_per_prompt: int = 0,
73	enable_expert_parallel: bool = False,	60	enable_expert_parallel: bool = False,
	@@ -86,7 +73,6 @@ class VLLMInferEngine(BaseInferEngine):
86	infer_pipeline_parallel_size (int): Pipeline parallel size during inference.	73	infer_pipeline_parallel_size (int): Pipeline parallel size during inference.
87	infer_expert_parallel_size (int): Expert parallel size during inference.	74	infer_expert_parallel_size (int): Expert parallel size during inference.
88	sampling_config (dict): Configuration for text generation sampling.	75	sampling_config (dict): Configuration for text generation sampling.
89	- infer_expert_tensor_parallel_size (int): Expert tensor parallel size during inference.
90	enable_prefix_caching (bool): Whether to enable prefix caching.	76	enable_prefix_caching (bool): Whether to enable prefix caching.
91	num_scheduler_steps (int): Num scheduler steps. Default is 1.	77	num_scheduler_steps (int): Num scheduler steps. Default is 1.
92	max_num_seqs (int): Maximum number of sequences to process simultaneously. Default is 1.	78	max_num_seqs (int): Maximum number of sequences to process simultaneously. Default is 1.
	@@ -112,14 +98,15 @@ class VLLMInferEngine(BaseInferEngine):
112	max_model_len=max_model_len,	98	max_model_len=max_model_len,
113	dtype=dtype,	99	dtype=dtype,
114	gpu_memory_utilization=gpu_memory_utilization,	100	gpu_memory_utilization=gpu_memory_utilization,
115	- trust_remote_code=trust_remote_code	101	+ trust_remote_code=trust_remote_code,
		102	+ enable_expert_parallel=enable_expert_parallel,
116	)	103	)
117	# Additional initialization logic for VLLMInferEngine	104	# Additional initialization logic for VLLMInferEngine
118		105
119	- torch.compile = dummy_compile
120	# vLLM Ascend must be patched in advance	106	# vLLM Ascend must be patched in advance
121	from vllm_ascend.patch import platform	107	from vllm_ascend.patch import platform
122	from vllm_ascend.patch import worker	108	from vllm_ascend.patch import worker
		109	+ from mindspeed_rl.models.rollout.vllm_adapter import engine_core
123		110
124	# Initialize sampling parameters from SamplingConfig	111	# Initialize sampling parameters from SamplingConfig
125	self.sampling_config = sampling_config	112	self.sampling_config = sampling_config
	@@ -186,24 +173,31 @@ class VLLMInferEngine(BaseInferEngine):
186	enable_prefix_caching=enable_prefix_caching,	173	enable_prefix_caching=enable_prefix_caching,
187	num_scheduler_steps=num_scheduler_steps,	174	num_scheduler_steps=num_scheduler_steps,
188	dtype=dtype,	175	dtype=dtype,
189	- enforce_eager=True,	176	+ enforce_eager=enforce_eager,
190	skip_tokenizer_init=False,	177	skip_tokenizer_init=False,
191	gpu_memory_utilization=gpu_memory_utilization,	178	gpu_memory_utilization=gpu_memory_utilization,
192	max_num_seqs=max_num_seqs,	179	max_num_seqs=max_num_seqs,
193	max_model_len=max_model_len,	180	max_model_len=max_model_len,
194	seed=self.sampling_params.seed,	181	seed=self.sampling_params.seed,
195	- limit_mm_per_prompt=limit_mm_per_prompt_dict,	182	+ max_num_batched_tokens=max_num_batched_tokens,
196	enable_expert_parallel=enable_expert_parallel,	183	enable_expert_parallel=enable_expert_parallel,
		184	+ limit_mm_per_prompt=limit_mm_per_prompt_dict,
197	additional_config={	185	additional_config={
198	- 'expert_tensor_parallel_size': infer_expert_tensor_parallel_size,	186	+ "torchair_graph_config": {
199	- 'enable_graph_mode': int(os.environ.get('VLLM_ENABLE_GRAPH_MODE', '0')),	187	+ "enabled": torchair_graph,
200	- 'ascend_scheduler_config': {},	188	+ "use_cached_graph": False,
		189	+ "graph_batch_sizes_init": False,
		190	+ "graph_batch_sizes": [max_num_seqs],
		191	+ },
		192	+ "ascend_scheduler_config": {
		193	+ "enabled": True,
		194	+ },
		195	+ "refresh": True,
201	}	196	}
202	)	197	)
203		198
204	self.engine = self.llm.llm_engine	199	self.engine = self.llm.llm_engine
205	self.model = self.llm.llm_engine.model_executor.driver_worker.worker.model_runner.get_model()	200	self.model = self.llm.llm_engine.model_executor.driver_worker.worker.model_runner.get_model()
206	- self.kv_cache_configs = None
207		201
208	self.cpu_model = {}	202	self.cpu_model = {}
209	for name, params in self.model.named_parameters():	203	for name, params in self.model.named_parameters():
	@@ -211,49 +205,15 @@ class VLLMInferEngine(BaseInferEngine):
211		205
212	if load_format == "megatron":	206	if load_format == "megatron":
213	self.free_cache_engine()	207	self.free_cache_engine()
214	- if os.environ['VLLM_USE_V1'] == '1':
215	- self._initialize_kv_caches(self.llm.llm_engine.vllm_config)
216	self.offload_model_weights()	208	self.offload_model_weights()
217		209
218	- from vllm.config import VllmConfig
219	-
220	- def _initialize_kv_caches(self, vllm_config: VllmConfig):
221	-
222	- # Get all kv cache needed by the model
223	- kv_cache_specs = self.llm.llm_engine.engine_core.engine_core.model_executor.get_kv_cache_specs()
224	-
225	- # Profiles the peak memory usage of the model to determine how much
226	- # memory can be allocated for kv cache.
227	- available_gpu_memory = self.llm.llm_engine.engine_core.engine_core.model_executor.determine_available_memory()
228	-
229	- assert len(kv_cache_specs) == len(available_gpu_memory)
230	- # Get the kv cache tensor size
231	- self.kv_cache_configs = [
232	- get_kv_cache_config(vllm_config, kv_cache_spec_one_worker,
233	- available_gpu_memory_one_worker)
234	- for kv_cache_spec_one_worker, available_gpu_memory_one_worker in
235	- zip(kv_cache_specs, available_gpu_memory)
236	- ]
237	-
238	- # Since we use a shared centralized controller, we need the
239	- # `kv_cache_config` to be consistent across all workers to make sure
240	- # all the memory operators can be applied to all workers.
241	- unify_kv_cache_configs(self.kv_cache_configs)
242	-
243	- # All workers have the same kv_cache_config except layer names, so use
244	- # an arbitrary one to initialize the scheduler.
245	- assert all([
246	- cfg.num_blocks == self.kv_cache_configs[0].num_blocks
247	- for cfg in self.kv_cache_configs
248	- ])
249	-
250	def init_cache_engine(self):	210	def init_cache_engine(self):
251	if os.environ['VLLM_USE_V1'] == '1':	211	if os.environ['VLLM_USE_V1'] == '1':
252	worker = self.llm.llm_engine.model_executor.driver_worker.worker	212	worker = self.llm.llm_engine.model_executor.driver_worker.worker
253	if not worker.model_runner.kv_caches:	213	if not worker.model_runner.kv_caches:
254	# v1 使用显式初始化方法	214	# v1 使用显式初始化方法
255	self.llm.llm_engine.engine_core.engine_core.model_executor.initialize_from_config(	215	self.llm.llm_engine.engine_core.engine_core.model_executor.initialize_from_config(
256	- self.kv_cache_configs)	216	+ self.llm.llm_engine.engine_core.engine_core.kv_cache_configs)
257	else:	217	else:
258	if self.llm.llm_engine.model_executor.driver_worker.worker.cache_engine is None:	218	if self.llm.llm_engine.model_executor.driver_worker.worker.cache_engine is None:
259	self.llm.llm_engine.model_executor.driver_worker.worker._init_cache_engine()	219	self.llm.llm_engine.model_executor.driver_worker.worker._init_cache_engine()
	@@ -261,9 +221,7 @@ class VLLMInferEngine(BaseInferEngine):
261	def free_cache_engine(self):	221	def free_cache_engine(self):
262	if os.environ['VLLM_USE_V1'] == '1':	222	if os.environ['VLLM_USE_V1'] == '1':
263	worker = self.llm.llm_engine.model_executor.driver_worker.worker	223	worker = self.llm.llm_engine.model_executor.driver_worker.worker
264	-
265	ctx = worker.model_runner.vllm_config.compilation_config.static_forward_context	224	ctx = worker.model_runner.vllm_config.compilation_config.static_forward_context
266	-
267	else:	225	else:
268	ctx = self.llm.llm_engine.model_executor.driver_worker.worker.compilation_config.static_forward_context	226	ctx = self.llm.llm_engine.model_executor.driver_worker.worker.compilation_config.static_forward_context
269	from vllm.attention import AttentionType	227	from vllm.attention import AttentionType
	@@ -304,7 +262,6 @@ class VLLMInferEngine(BaseInferEngine):
304	gc.collect()	262	gc.collect()
305	torch.cuda.empty_cache()	263	torch.cuda.empty_cache()
306		264
307	-
308	def offload_model_weights(self):	265	def offload_model_weights(self):
309	for name, params in self.model.named_parameters():	266	for name, params in self.model.named_parameters():
310	params.data = self.cpu_model[name]	267	params.data = self.cpu_model[name]
	@@ -464,4 +421,4 @@ def get_local_rank() -> int:
464		421
465	# Default to 0 (for testing or single-process scenarios)	422	# Default to 0 (for testing or single-process scenarios)
466	logger.warning("Warning: Unable to determine local rank. Defaulting to 0.")	423	logger.warning("Warning: Unable to determine local rank. Defaulting to 0.")
467	- return 0	424	+ return 0

Mmindspeed_rl/trainer/grpo_trainer_hybrid.py+7-3

	@@ -189,9 +189,13 @@ class RayGRPOTrainer(RayBaseTrainer):
189		189
190	metrics_result = metrics_post_processing(metrics_result)	190	metrics_result = metrics_post_processing(metrics_result)
191	metrics_result = metrics_sort(metrics_result, all_timer.last)	191	metrics_result = metrics_sort(metrics_result, all_timer.last)
192	- tps = compute_tps(self.kwargs, grpo_data_metrics, self.global_batch_size, self.n_samples_per_prompt, all_timer.last)	192	+ log_max_throughput = self.actor_worker.rl_config.log_max_throughput
193	- update_tps = compute_tps(self.kwargs, grpo_data_metrics, self.global_batch_size, self.n_samples_per_prompt, metrics_result["timing/update"])	193	+ tps = compute_tps(self.kwargs, grpo_data_metrics, self.global_batch_size, self.n_samples_per_prompt,
194	- vllm_tps = compute_tps(self.kwargs, grpo_data_metrics, self.global_batch_size, self.n_samples_per_prompt, metrics_result["timing/rollout"])	194	+ all_timer.last, log_max_throughput)
		195	+ update_tps = compute_tps(self.kwargs, grpo_data_metrics, self.global_batch_size, self.n_samples_per_prompt,
		196	+ metrics_result["timing/update"], log_max_throughput)
		197	+ vllm_tps = compute_tps(self.kwargs, grpo_data_metrics, self.global_batch_size, self.n_samples_per_prompt,
		198	+ metrics_result["timing/rollout"], log_max_throughput)
195	metrics.update(value=metrics_result)	199	metrics.update(value=metrics_result)
196	metrics.update(value=grpo_data_metrics)	200	metrics.update(value=grpo_data_metrics)
197	metrics.update("e2e_tps", tps)	201	metrics.update("e2e_tps", tps)

Mmindspeed_rl/utils/utils.py+86-5

	@@ -2,11 +2,14 @@
2	# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.	2	# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
3		3
4	import os	4	import os
		5	+import re
		6	+import socket
		7	+import subprocess
5	import sys	8	import sys
6	import json	9	import json
7	import time	10	import time
8	-import math
9	import random	11	import random
		12	+from contextlib import contextmanager
10	from functools import wraps	13	from functools import wraps
11	from typing import Dict, List	14	from typing import Dict, List
12		15
	@@ -15,6 +18,7 @@ import omegaconf
15	import numpy as np	18	import numpy as np
16	import torch	19	import torch
17	import torch_npu	20	import torch_npu
		21	+import torch.distributed as dist
18	from torch import Tensor	22	from torch import Tensor
19		23
20		24
	@@ -259,7 +263,6 @@ def metrics_sort(metrics, time_all) -> Dict[str, Tensor]:
259	non_overlap_reward_model_time = max(reward_end_time - max(old_log_p_end_time, reward_start_time), 0)	263	non_overlap_reward_model_time = max(reward_end_time - max(old_log_p_end_time, reward_start_time), 0)
260	metrics["timing/non_overlap_reward_model"] = non_overlap_reward_model_time	264	metrics["timing/non_overlap_reward_model"] = non_overlap_reward_model_time
261		265
262	-
263	metrics["timing/non_overlap_reference_model"] = non_overlap_reference_model_time	266	metrics["timing/non_overlap_reference_model"] = non_overlap_reference_model_time
264	metrics["timing/non_overlap_adv"] = non_overlap_adv_time	267	metrics["timing/non_overlap_adv"] = non_overlap_adv_time
265	metrics["timing/all"] = time_all	268	metrics["timing/all"] = time_all
	@@ -276,8 +279,7 @@ def metrics_sort(metrics, time_all) -> Dict[str, Tensor]:
276	return sorted_metric	279	return sorted_metric
277		280
278		281
279	-def compute_tps(compute_kwargs, metrics_result, gbs, n_samples, time_all):	282	+def compute_tps(compute_kwargs, metrics_result, gbs, n_samples, time_all, log_max_throughput):
280	-
281	actor_resource = compute_kwargs.get('actor_resource', {})	283	actor_resource = compute_kwargs.get('actor_resource', {})
282	reference_resource = compute_kwargs.get('reference_resource', {})	284	reference_resource = compute_kwargs.get('reference_resource', {})
283	reward_resource = compute_kwargs.get('reward_resource', None)	285	reward_resource = compute_kwargs.get('reward_resource', None)
	@@ -288,7 +290,8 @@ def compute_tps(compute_kwargs, metrics_result, gbs, n_samples, time_all):
288	reward_npus = reward_resource.get('num_npus', 0) if reward_resource is not None else 0	290	reward_npus = reward_resource.get('num_npus', 0) if reward_resource is not None else 0
289		291
290	world_size = actor_npus + reference_npus + reward_npus if not actor_resource_only else actor_npus	292	world_size = actor_npus + reference_npus + reward_npus if not actor_resource_only else actor_npus
291	- tps = (metrics_result['response_length/mean'] + metrics_result['prompt_length/mean']) * gbs * n_samples / world_size / time_all	293	+ length_type = 'max' if log_max_throughput else 'mean'
		294	+ tps = (metrics_result[f'response_length/{length_type}'] + metrics_result[f'prompt_length/{length_type}']) * gbs * n_samples / world_size / time_all
292	return tps	295	return tps
293		296
294		297
	@@ -573,5 +576,83 @@ def profiler_step(profiler):
573	profiler.step()	576	profiler.step()
574		577
575		578
		579	+_COMPILE = None
		580	+
		581	+
		582	+def init_torch_compile(compile):
		583	+ global _COMPILE
		584	+ _COMPILE = compile
		585	+
		586	+
		587	+@contextmanager
		588	+def replace_torch_compile():
		589	+ """Context manager to temporarily replace torch.compile with a dummy function"""
		590	+ original_compile = torch.compile # Save the original function
		591	+ torch.compile = _COMPILE # Replace with our dummy
		592	+
		593	+ try:
		594	+ yield # Execute the code inside the 'with' block
		595	+ finally:
		596	+ torch.compile = original_compile # Restore the original function
		597	+
		598	+
		599	+def get_cluster_info():
		600	+ # 确保分布式环境已初始化
		601	+ if not dist.is_initialized():
		602	+ raise RuntimeError("Distributed environment not initialized")
		603	+
		604	+ world_size = dist.get_world_size()
		605	+
		606	+ # 获取当前节点的IP地址
		607	+ ip_address = get_current_node_ip()
		608	+
		609	+ # 收集所有rank的IP地址
		610	+ ip_list = [None] * world_size
		611	+ dist.all_gather_object(ip_list, ip_address)
		612	+
		613	+ return ip_list
		614	+
		615	+
		616	+def get_current_node_ip() -> str:
		617	+ try:
		618	+ # 创建一个 UDP 套接字（仅用于获取接口信息）
		619	+ with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as s:
		620	+ # 连接到一个外部地址（无需真实通信）
		621	+ s.connect(("8.8.8.8", 80)) # Google DNS 服务器
		622	+ local_ip = s.getsockname()[0]
		623	+ except Exception:
		624	+ local_ip = _get_ip_by_ifname()
		625	+ if not local_ip:
		626	+ # 如果失败，回退到遍历接口
		627	+ local_ip = "127.0.0.1"
		628	+ hostname = socket.gethostname()
		629	+ for addr in socket.getaddrinfo(hostname, None):
		630	+ ip = addr[4][0]
		631	+ if not ip.startswith("::"):
		632	+ local_ip = ip
		633	+ break
		634	+ return local_ip
		635	+
		636	+
		637	+def _get_ip_by_ifname():
		638	+ """
		639	+ 通过接口名称（如 eth0、en0）获取 IPv4 地址
		640	+ 返回 IP 字符串，失败返回 None
		641	+ """
		642	+ try:
		643	+ # 执行 ifconfig 命令并捕获输出
		644	+ ifname = os.environ.get("HCCL_SOCKET_IFNAME", 0)
		645	+ if ifname:
		646	+ output = subprocess.check_output(["ifconfig", ifname], stderr=subprocess.STDOUT).decode()
		647	+ # 正则匹配 IPv4 地址（排除 127.0.0.1）
		648	+ matches = re.findall(r'inet (?:addr:)?((?:\d{1,3}\.){3}\d{1,3})', output)
		649	+ for ip in matches:
		650	+ if ip != "127.0.0.1":
		651	+ return ip
		652	+ return None
		653	+ except subprocess.CalledProcessError:
		654	+ return None
		655	+
		656	+
576	def is_multimodal():	657	def is_multimodal():
577	return eval(os.getenv("IS_MULTIMODAL", "False"))	658	return eval(os.getenv("IS_MULTIMODAL", "False"))

Mmindspeed_rl/workers/actor_hybrid_worker.py+21-17

	@@ -24,7 +24,8 @@ from mindspeed_rl.utils.tokenizer import BaseTokenizer
24	from mindspeed_rl.utils.utils import MsProbe	24	from mindspeed_rl.utils.utils import MsProbe
25	from mindspeed_rl.workers.base_worker import BaseWorker	25	from mindspeed_rl.workers.base_worker import BaseWorker
26	from mindspeed_rl.workers.resharding.megatron_sharding_manager import MegatronShardingManager, MegatronOffLoader	26	from mindspeed_rl.workers.resharding.megatron_sharding_manager import MegatronShardingManager, MegatronOffLoader
27	-from mindspeed_rl.utils.utils import num_floating_point_operations, get_attr_wrapped_model, mstx_timer_decorator, profiler_start, profiler_step, is_multimodal	27	+from mindspeed_rl.utils.utils import (num_floating_point_operations, get_attr_wrapped_model, mstx_timer_decorator,
		28	+ profiler_start, profiler_step, is_multimodal, replace_torch_compile)
28	from mindspeed_rl.utils.pad_process import remove_padding_and_split_to_list, truncate_rows	29	from mindspeed_rl.utils.pad_process import remove_padding_and_split_to_list, truncate_rows
29		30
30		31
	@@ -95,8 +96,8 @@ class ActorHybridWorkerBase(BaseWorker):
95	self.actor_offloader.offload_grad()	96	self.actor_offloader.offload_grad()
96	if self.generate_config.offload_train_param:	97	if self.generate_config.offload_train_param:
97	self.actor_offloader.offload_param()	98	self.actor_offloader.offload_param()
98	-	99	+ with replace_torch_compile():
99	- self.inference_model = self._build_rollout()	100	+ self.inference_model = self._build_rollout()
100	self.sharding_manager = self._build_sharding_manager()	101	self.sharding_manager = self._build_sharding_manager()
101		102
102	if self.generate_config.offload_train_param:	103	if self.generate_config.offload_train_param:
	@@ -348,10 +349,11 @@ class ActorHybridWorkerBase(BaseWorker):
348	prompts = truncate_rows(prompts_data, prompt_length_data)	349	prompts = truncate_rows(prompts_data, prompt_length_data)
349	prompts_list = [prompt.numpy().tolist() for prompt in prompts]	350	prompts_list = [prompt.numpy().tolist() for prompt in prompts]
350		351
351	- responses_pad_right = self.actor_hybrid.generate_sequences(copy.deepcopy(prompts_list), indexes,	352	+ with replace_torch_compile():
352	- n_samples_per_prompt=self.rl_config.n_samples_per_prompt,	353	+ responses_pad_right = self.actor_hybrid.generate_sequences(copy.deepcopy(prompts_list), indexes,
353	- async_engine=self.rl_config.async_engine,	354	+ n_samples_per_prompt=self.rl_config.n_samples_per_prompt,
354	- extra_info=batch_data)	355	+ async_engine=self.rl_config.async_engine,
		356	+ extra_info=batch_data)
355	responses = remove_padding_and_split_to_list(responses_pad_right, self.tokenizer.eod, pad_token_id)	357	responses = remove_padding_and_split_to_list(responses_pad_right, self.tokenizer.eod, pad_token_id)
356	responses_length = [torch.tensor([len(response)]) for response in responses]	358	responses_length = [torch.tensor([len(response)]) for response in responses]
357	if is_multimodal():	359	if is_multimodal():
	@@ -386,14 +388,15 @@ class ActorHybridWorkerBase(BaseWorker):
386	def async_generate_process(self, experience_count, index, pad_token_id, prompts_list, start_time):	388	def async_generate_process(self, experience_count, index, pad_token_id, prompts_list, start_time):
387	# inference	389	# inference
388	self.actor_hybrid.inference_actor.init_cache_engine()	390	self.actor_hybrid.inference_actor.init_cache_engine()
389	- response_generator = self.actor_hybrid.generate_sequences(	391	+ with replace_torch_compile():
390	- copy.deepcopy(prompts_list),	392	+ response_generator = self.actor_hybrid.generate_sequences(
391	- indexes=index,	393	+ copy.deepcopy(prompts_list),
392	- max_tokens=self.generate_config.sampling_config["max_tokens"],	394	+ indexes=index,
393	- n_samples_per_prompt=1,	395	+ max_tokens=self.generate_config.sampling_config["max_tokens"],
394	- n=1,	396	+ n_samples_per_prompt=1,
395	- async_engine=True,	397	+ n=1,
396	- )	398	+ async_engine=True,
		399	+ )
397	for samples, idx in response_generator:	400	for samples, idx in response_generator:
398	prompts, responses, log_probs = samples	401	prompts, responses, log_probs = samples
399	responses = remove_padding_and_split_to_list(responses, self.tokenizer.eod, pad_token_id)	402	responses = remove_padding_and_split_to_list(responses, self.tokenizer.eod, pad_token_id)
	@@ -493,7 +496,6 @@ class ActorHybridWorkerBase(BaseWorker):
493	self.megatron_config.tokenizer_name_or_path, trust_remote_code=self.generate_config.trust_remote_code)	496	self.megatron_config.tokenizer_name_or_path, trust_remote_code=self.generate_config.trust_remote_code)
494		497
495	sampling_config = {"num_completions": self.rl_config.n_samples_per_prompt,	498	sampling_config = {"num_completions": self.rl_config.n_samples_per_prompt,
496	- "best_of": self.rl_config.n_samples_per_prompt,
497	**self.generate_config.sampling_config}	499	**self.generate_config.sampling_config}
498		500
499	rollout = VLLMInferEngine(	501	rollout = VLLMInferEngine(
	@@ -515,9 +517,11 @@ class ActorHybridWorkerBase(BaseWorker):
515	gpu_memory_utilization=self.generate_config.gpu_memory_utilization,	517	gpu_memory_utilization=self.generate_config.gpu_memory_utilization,
516	trust_remote_code=self.generate_config.trust_remote_code,	518	trust_remote_code=self.generate_config.trust_remote_code,
517	enforce_eager=self.generate_config.enforce_eager,	519	enforce_eager=self.generate_config.enforce_eager,
		520	+ torchair_graph=self.generate_config.torchair_graph,
		521	+ enable_expert_parallel=self.generate_config.enable_expert_parallel,
		522	+ max_num_batched_tokens=self.generate_config.max_num_batched_tokens,
518	limit_mm_image_per_prompt=self.generate_config.limit_mm_image_per_prompt,	523	limit_mm_image_per_prompt=self.generate_config.limit_mm_image_per_prompt,
519	limit_mm_video_per_prompt=self.generate_config.limit_mm_video_per_prompt,	524	limit_mm_video_per_prompt=self.generate_config.limit_mm_video_per_prompt,
520	- enable_expert_parallel=self.generate_config.enable_expert_parallel,
521	)	525	)
522	return rollout	526	return rollout
523		527

Mmindspeed_rl/workers/base_worker.py+2-45

	@@ -1,8 +1,6 @@
1	# Copyright (c) 2025, HUAWEI CORPORATION. All rights reserved.	1	# Copyright (c) 2025, HUAWEI CORPORATION. All rights reserved.
2		2
3	import os	3	import os
4	-import re
5	-import subprocess
6	from abc import ABC	4	from abc import ABC
7	from typing import List, Callable	5	from typing import List, Callable
8	import socket	6	import socket
	@@ -10,7 +8,6 @@ import socket
10	import torch	8	import torch
11	import torch_npu	9	import torch_npu
12	import ray	10	import ray
13	-import torch.distributed as dist
14		11
15	from mindspeed_rl.models.rollout.vllm_adapter.vllm_parallel_state import get_vllm_tp_group_ranks	12	from mindspeed_rl.models.rollout.vllm_adapter.vllm_parallel_state import get_vllm_tp_group_ranks
16	from mindspeed_rl.utils.loggers import Loggers	13	from mindspeed_rl.utils.loggers import Loggers
	@@ -35,7 +32,7 @@ from mindspeed_rl.trainer.utils.parallel_state import (
35	get_context_parallel_group	32	get_context_parallel_group
36	)	33	)
37	from mindspeed_rl.utils.compute import set_parallel_state, set_vocab_parallel	34	from mindspeed_rl.utils.compute import set_parallel_state, set_vocab_parallel
38	-from mindspeed_rl.utils.utils import get_current_dp_range_indexes	35	+from mindspeed_rl.utils.utils import get_current_dp_range_indexes, get_current_node_ip
39	from mindspeed_rl.trainer.utils.transfer_dock import pack_experience_columns, unpack_pad_experience	36	from mindspeed_rl.trainer.utils.transfer_dock import pack_experience_columns, unpack_pad_experience
40	from mindspeed_rl.trainer.utils.mm_transfer_dock import unpack_mm_experience	37	from mindspeed_rl.trainer.utils.mm_transfer_dock import unpack_mm_experience
41	from mindspeed_rl.utils.utils import mstx_timer_decorator, is_multimodal	38	from mindspeed_rl.utils.utils import mstx_timer_decorator, is_multimodal
	@@ -55,7 +52,7 @@ class BaseRayWorker:
55	torch.npu.set_device(self._local_rank)	52	torch.npu.set_device(self._local_rank)
56	current_device = torch.npu.current_device()	53	current_device = torch.npu.current_device()
57	if os.environ.get("MASTER_ADDR", 0) == "localhost":	54	if os.environ.get("MASTER_ADDR", 0) == "localhost":
58	- self._master_addr = self._get_current_node_ip()	55	+ self._master_addr = get_current_node_ip()
59	self._master_port = self._get_free_port()	56	self._master_port = self._get_free_port()
60	os.environ["MASTER_ADDR"] = self._master_addr	57	os.environ["MASTER_ADDR"] = self._master_addr
61	os.environ["MASTER_PORT"] = str(self._master_port)	58	os.environ["MASTER_PORT"] = str(self._master_port)
	@@ -75,46 +72,6 @@ class BaseRayWorker:
75	def rank(self):	72	def rank(self):
76	return self._rank	73	return self._rank
77		74
78	- def _get_current_node_ip(self) -> str:
79	- try:
80	- # 创建一个 UDP 套接字（仅用于获取接口信息）
81	- with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as s:
82	- # 连接到一个外部地址（无需真实通信）
83	- s.connect(("8.8.8.8", 80)) # Google DNS 服务器
84	- local_ip = s.getsockname()[0]
85	- except Exception:
86	- local_ip = self._get_ip_by_ifname()
87	- if not local_ip:
88	- # 如果失败，回退到遍历接口
89	- local_ip = "127.0.0.1"
90	- hostname = socket.gethostname()
91	- for addr in socket.getaddrinfo(hostname, None):
92	- ip = addr[4][0]
93	- if not ip.startswith("::"):
94	- local_ip = ip
95	- break
96	- return local_ip
97	-
98	- @staticmethod
99	- def _get_ip_by_ifname():
100	- """
101	- 通过接口名称（如 eth0、en0）获取 IPv4 地址
102	- 返回 IP 字符串，失败返回 None
103	- """
104	- try:
105	- # 执行 ifconfig 命令并捕获输出
106	- ifname = os.environ.get("HCCL_SOCKET_IFNAME", 0)
107	- if ifname:
108	- output = subprocess.check_output(["ifconfig", ifname], stderr=subprocess.STDOUT).decode()
109	- # 正则匹配 IPv4 地址（排除 127.0.0.1）
110	- matches = re.findall(r'inet (?:addr:)?((?:\d{1,3}\.){3}\d{1,3})', output)
111	- for ip in matches:
112	- if ip != "127.0.0.1":
113	- return ip
114	- return None
115	- except subprocess.CalledProcessError:
116	- return None
117	-
118	@staticmethod	75	@staticmethod
119	def _get_free_port():	76	def _get_free_port():
120	with socket.socket() as sock:	77	with socket.socket() as sock:

Mmindspeed_rl/workers/resharding/weight_adaptor.py+1-0

	@@ -658,6 +658,7 @@ WEIGHT_ADAPTOR_REGISTRY = {
658	"Qwen2ForCausalLM": QwenMVWeightAdaptor,	658	"Qwen2ForCausalLM": QwenMVWeightAdaptor,
659	"DeepseekV3ForCausalLM": DeepSeekMVWeightAdaptor,	659	"DeepseekV3ForCausalLM": DeepSeekMVWeightAdaptor,
660	"DeepseekV2ForCausalLM": DeepSeekMVWeightAdaptor,	660	"DeepseekV2ForCausalLM": DeepSeekMVWeightAdaptor,
		661	+ "CustomDeepseekV2ForCausalLM": DeepSeekMVWeightAdaptor,
661	"CustomDeepseekV3ForCausalLM": DeepSeekMVWeightAdaptor,	662	"CustomDeepseekV3ForCausalLM": DeepSeekMVWeightAdaptor,
662	"Qwen2_5_VLForConditionalGeneration": Qwen2_5_VLWeightAdaptor,	663	"Qwen2_5_VLForConditionalGeneration": Qwen2_5_VLWeightAdaptor,
663	"CustomQwen3MoeForCausalLM": Qwen3MoeMVWeightAdaptor,	664	"CustomQwen3MoeForCausalLM": Qwen3MoeMVWeightAdaptor,

Asources/images/r1_zero/deepseek-671b-response-curses.png+0-0

Asources/images/r1_zero/deepseek-671b-reward-curses.png+0-0

Mtests/st/resharding/test_resharding.py+8-2

	@@ -11,6 +11,11 @@ import torch
11	import torch.nn as nn	11	import torch.nn as nn
12	from transformers import AutoTokenizer	12	from transformers import AutoTokenizer
13		13
		14	+from mindspeed_rl.utils.utils import init_torch_compile, replace_torch_compile
		15	+
		16	+# Initialize torch.compile global variables to avoid training-related patches affecting vLLM graph mode enabling.
		17	+init_torch_compile(torch.compile)
		18	+
14	from mindspeed_llm import megatron_adaptor	19	from mindspeed_llm import megatron_adaptor
15	import megatron	20	import megatron
16	from megatron.core import mpu	21	from megatron.core import mpu
	@@ -375,5 +380,6 @@ def parse_args():
375		380
376	if __name__ == "__main__":	381	if __name__ == "__main__":
377	args = parse_args()	382	args = parse_args()
378	- test_actor = TestActor(args)	383	+ with replace_torch_compile():
379	- test_actor.generate_sequence(prompt_list)	384	+ test_actor = TestActor(args)
		385	+ test_actor.generate_sequence(prompt_list)

	@@ -73,9 +73,10 @@ bash examples/data/preprocess_data.sh deepscaler
73		73
74	以 Qwen25 7B 模型为例,在启动训练之前，需要修改[ 启动脚本 ](../../examples/grpo/grpo_trainer_qwen25_7b.sh)的配置：	74	以 Qwen25 7B 模型为例,在启动训练之前，需要修改[ 启动脚本 ](../../examples/grpo/grpo_trainer_qwen25_7b.sh)的配置：
75	1. 根据实际安装路径设置 jemalloc 环境变量，用于更好管理内存，避免长跑过程中内存 OOM ，例如：export LD_PRELOAD=/usr/local/lib/libjemalloc.so.2	75	1. 根据实际安装路径设置 jemalloc 环境变量，用于更好管理内存，避免长跑过程中内存 OOM ，例如：export LD_PRELOAD=/usr/local/lib/libjemalloc.so.2
76	-2. 修改 DEFAULT_YAML 为指定的 yaml，目前已支持的配置文件放置在 configs / 文件夹下，具体参数说明可见 [配置文件参数介绍](../features/grpo_yaml.md)；	76	+2. 修改 DEFAULT_YAML 为指定的 yaml，目前已支持的配置文件放置在 configs / 文件夹下，同时需要对[ 环境变量配置文件 ](../../configs/envs/runtime_env.yaml) 中的 VLLM_DP_SIZE 及 HCCL_SOCKET_IFNAME 等参数进行配置，具体参数说明可见 [配置文件参数介绍](../features/grpo_yaml.md)；
77	3. 根据使用机器的情况，修改 NNODES 、NPUS_PER_NODE 配置，例如单机 A3 可设置 NNODES 为 1 、NPUS_PER_NODE 为16；	77	3. 根据使用机器的情况，修改 NNODES 、NPUS_PER_NODE 配置，例如单机 A3 可设置 NNODES 为 1 、NPUS_PER_NODE 为16；
78	-4. 如果是单机，需要保证 MASTER_ADDR 与 CURRENT_IP 一致，如果为多机，需要保证各个机器的 MASTER_ADDR 一致，CURRENT_IP 为各个节点的 IP (需要注意的是MASTER_ADDR 与 CURRENT_IP 不能设置为 localhost)；；	78	+4. 如果是单机，需要保证 MASTER_ADDR 与 CURRENT_IP 一致，如果为多机，需要保证各个机器的 MASTER_ADDR 一致，CURRENT_IP 为各个节点的 IP (需要注意的是MASTER_ADDR 与 CURRENT_IP 不能设置为 localhost)；
		79	+5. 启动脚本中的 SOCKET_IFNAME 需要设置为 CURRENT_IP 所对应的通信网卡名；
79	```bash	80	```bash
80	#上述注意点修改完毕后，可启动脚本开启训练	81	#上述注意点修改完毕后，可启动脚本开启训练
81	bash examples/grpo/grpo_trainer_qwen25_7b.sh	82	bash examples/grpo/grpo_trainer_qwen25_7b.sh
	@@ -172,10 +173,11 @@ $$
172	(\text{response_length_mean} + \text{prompt_length_mean}) \times \text{global_batch_size} \times \text{n_samples_per_prompt} / \text{world_size} \ / \text{time_rollout}	173	(\text{response_length_mean} + \text{prompt_length_mean}) \times \text{global_batch_size} \times \text{n_samples_per_prompt} / \text{world_size} \ / \text{time_rollout}
173	$$	174	$$
174	## 性能数据	175	## 性能数据
175	-\| 模型 \| 机器型号 \| GBS \| n_samples \| max_prompt_length \| max_tokens \| 端到端 tps \|	176	+\| 模型 \| 机器型号 \| GBS \| n_samples \| max_prompt_length \| max_tokens \| 端到端 tps \|
176	-\|------------\|------\|-----\|-----------\|-------------------\|------------\|---------\|	177	+\|---------------------\|----------\|-----\|-----------\|-------------------\|------------\|---------\|
177	-\| Qwen25-7B \| Atlas A3 \| 32 \| 8 \| 2048 \| 2048 \| 220 \|	178	+\| Qwen25-7B \| Atlas A3 \| 32 \| 8 \| 2048 \| 2048 \| 220 \|
178	-\| Qwen25-32B \| Atlas A3 \| 64 \| 16 \| 1024 \| 2048 \| 220 \|	179	+\| Qwen25-32B \| Atlas A3 \| 64 \| 16 \| 1024 \| 2048 \| 220 \|
179	-\| Qwen25-32B \| Atlas A2 \| 64 \| 16 \| 1024 \| 1024 \| 100 \|	180	+\| Qwen25-32B \| Atlas A2 \| 64 \| 16 \| 1024 \| 1024 \| 100 \|
		181	+\| DeepSeek-R1-671B \| Atlas A3 \| 384 \| 32 \| 1024 \| 2048 \| 210 \|
180		182
181	注：模型 token/p/s 性能数据会打印在日志中, 当前计算公式下，A3单卡性能需要将日志打印的token/p/s性能指数*2。	183	注：模型 token/p/s 性能数据会打印在日志中, 当前计算公式下，A3单卡性能需要将日志打印的token/p/s性能指数*2。