【PART 3 Trl PPO】添加st看护-MindSpeed-LLM-AtomGit

MREADME.md+9-1

	@@ -537,7 +537,7 @@ MindSpeed-LLM目前已内置支持百余个业界常用稠密和MOE大模型的
537	<td>【China Mobile Cloud】</td>	537	<td>【China Mobile Cloud】</td>
538	</tr>	538	</tr>
539	<tr>	539	<tr>
540	- <td rowspan="3">奖励模型</td>	540	+ <td rowspan="4">奖励模型</td>
541	<td><a href="./docs/features/outcome_reward_model.md">ORM</a></td>	541	<td><a href="./docs/features/outcome_reward_model.md">ORM</a></td>
542	<td>✅</td>	542	<td>✅</td>
543	<td>❌</td>	543	<td>❌</td>
	@@ -561,6 +561,14 @@ MindSpeed-LLM目前已内置支持百余个业界常用稠密和MOE大模型的
561	<td>❌</td>	561	<td>❌</td>
562	<td>【Ascend】</td>	562	<td>【Ascend】</td>
563	</tr>	563	</tr>
		564	+ <tr>
		565	+ <td><a href="./docs/features/trl_ppo.md">Trl PPO</a></td>
		566	+ <td>✅</td>
		567	+ <td>❌</td>
		568	+ <td>✅</td>
		569	+ <td>❌</td>
		570	+ <td>【Ascend】</td>
		571	+ </tr>
564	</tbody></table>	572	</tbody></table>
565		573
566	#### 加速特性	574	#### 加速特性

Mtests/README.md+48-12

	@@ -13,7 +13,7 @@
13	<th>Mem.</th>	13	<th>Mem.</th>
14	</tr>	14	</tr>
15	<tr>	15	<tr>
16	- <td rowspan="24">ST</td>	16	+ <td rowspan="28">ST</td>
17	<td rowspan="13">Pretrain</td>	17	<td rowspan="13">Pretrain</td>
18	<td>Mcore</td>	18	<td>Mcore</td>
19	<td>TP，PP，VPP，distributed_optimizer，o2_gradient，o2_optimizer，重计算，enable_recompute_layers_per_pp_rank，FA_TND，use_fused_rotary_pos_emb_new</td>	19	<td>TP，PP，VPP，distributed_optimizer，o2_gradient，o2_optimizer，重计算，enable_recompute_layers_per_pp_rank，FA_TND，use_fused_rotary_pos_emb_new</td>
	@@ -118,6 +118,23 @@
118	<td>Y</td>	118	<td>Y</td>
119	<td>Y</td>	119	<td>Y</td>
120	</tr>	120	</tr>
		121	+ <tr>
		122	+ <td rowspan="2">FullSFT</td>
		123	+ <td>Legacy</td>
		124	+ <td>prompt_type, variable_seq_lengths, matmul_add</td>
		125	+ <td><a href="st/shell_scripts/tune_qwen7b_tp8_pp1_full_ptd.sh">tune_qwen7b_tp8_pp1_full_ptd.sh</a></td>
		126	+ <td>Y</td>
		127	+ <td>Y</td>
		128	+ <td>Y</td>
		129	+ </tr>
		130	+ <tr>
		131	+ <td>Mcore</td>
		132	+ <td>自适应cp，general_cp，SFT_pack_cp</td>
		133	+ <td><a href="st/shell_scripts/tune_llama2_tp2_cp4_adaptive_cp.sh">tune_llama2_tp2_cp4_adaptive_cp.sh</a></td>
		134	+ <td>Y</td>
		135	+ <td>Y</td>
		136	+ <td>Y</td>
		137	+ </tr>
121	<tr>	138	<tr>
122	<td rowspan="2">LoRA</td>	139	<td rowspan="2">LoRA</td>
123	<td rowspan="1">Legacy</td>	140	<td rowspan="1">Legacy</td>
	@@ -135,7 +152,7 @@
135	<td>Y</td>	152	<td>Y</td>
136	<td>Y</td>	153	<td>Y</td>
137	</tr>	154	</tr>
138	- <tr>	155	+ <tr>
139	<td rowspan="3">DPO</td>	156	<td rowspan="3">DPO</td>
140	<td rowspan="3">Mcore</td>	157	<td rowspan="3">Mcore</td>
141	<td>DPO, CCLoRA, TP, PP, CP, MOE, use_fused_moe_token_permute_and_unpermute</td>	158	<td>DPO, CCLoRA, TP, PP, CP, MOE, use_fused_moe_token_permute_and_unpermute</td>
	@@ -159,21 +176,40 @@
159	<td>Y</td>	176	<td>Y</td>
160	</tr>	177	</tr>
161	<tr>	178	<tr>
162	- <td rowspan="2">FullSFT</td>	179	+ <td rowspan="1">Grpo</td>
163	- <td>Legacy</td>	180	+ <td rowspan="1">Mcore</td>
164	- <td>prompt_type, variable_seq_lengths, matmul_add</td>	181	+ <td>GRPO, tp, pp</td>
165	- <td><a href="st/shell_scripts/tune_qwen7b_tp8_pp1_full_ptd.sh">tune_qwen7b_tp8_pp1_full_ptd.sh</a></td>	182	+ <td><a href="st/shell_scripts/ray_grpo_full_llama32_1b_tp1pp1.sh">ray_grpo_full_llama32_1b_tp1pp1.sh</a></td>
166	- <td>Y</td>
167	- <td>Y</td>
168	<td>Y</td>	183	<td>Y</td>
		184	+ <td></td>
		185	+ <td></td>
169	</tr>	186	</tr>
170	<tr>	187	<tr>
171	- <td>Mcore</td>	188	+ <td rowspan="1">Online_DPO</td>
172	- <td>自适应cp，general_cp，SFT_pack_cp</td>	189	+ <td rowspan="1">Mcore</td>
173	- <td><a href="st/shell_scripts/tune_llama2_tp2_cp4_adaptive_cp.sh">tune_llama2_tp2_cp4_adaptive_cp.sh</a></td>	190	+ <td>Online_DPO, tp, pp</td>
174	- <td>Y</td>	191	+ <td><a href="st/shell_scripts/ray_online_dpo_full_llama32_1b_tp1pp1.sh">ray_online_dpo_full_llama32_1b_tp1pp1.sh</a></td>
		192	+ <td>Y</td>
		193	+ <td></td>
		194	+ <td></td>
		195	+ </tr>
		196	+ <tr>
		197	+ <td rowspan="1">Ray_PPO</td>
		198	+ <td rowspan="1">Mcore</td>
		199	+ <td>PPO, tp, pp</td>
		200	+ <td><a href="st/shell_scripts/ray_ppo_full_llama32_1b_tp1pp1.sh">ray_ppo_full_llama32_1b_tp1pp1.sh</a></td>
		201	+ <td>Y</td>
		202	+ <td></td>
		203	+ <td></td>
		204	+ </tr>
		205	+ <tr>
		206	+ <td rowspan="1">Trl_PPO</td>
		207	+ <td rowspan="1">Mcore</td>
		208	+ <td>PPO, CCLoRA, TP, PP</td>
		209	+ <td><a href="st/shell_scripts/trl_ppo_llama32_1b_ptd_tp2pp2.sh">trl_ppo_llama32_1b_ptd_tp2pp2.sh</a></td>
175	<td>Y</td>	210	<td>Y</td>
176	<td>Y</td>	211	<td>Y</td>
		212	+ <td></td>
177	</tr>	213	</tr>
178	<tr>	214	<tr>
179	<td rowspan="3">OutcomeRewardModel</td>	215	<td rowspan="3">OutcomeRewardModel</td>

Atests/st/baseline_results/trl_ppo_llama32_1b_ptd_tp2pp2.json+67-0

@@ -0,0 +1,67 @@
	1	+{
	2	+ "lm loss":[
	3	+ 8.92967,
	4	+ 3.665214,
	5	+ 9.204677,
	6	+ 3.214607,
	7	+ 8.039564,
	8	+ 1.330519,
	9	+ 8.494115,
	10	+ 4.01243,
	11	+ 8.062996,
	12	+ 5.95002,
	13	+ 7.335029,
	14	+ 4.84882,
	15	+ 7.946905,
	16	+ 6.096703,
	17	+ 8.195155,
	18	+ 1.073325,
	19	+ 8.7603,
	20	+ 6.448283,
	21	+ 8.29144,
	22	+ 4.365718,
	23	+ 8.519714,
	24	+ 5.338232,
	25	+ 8.267178,
	26	+ 5.644434,
	27	+ 6.304187,
	28	+ 3.538953,
	29	+ 7.035871,
	30	+ 2.488997,
	31	+ 7.590432,
	32	+ 4.330103
	33	+ ],
	34	+ "throughput": [
	35	+ 1.4,
	36	+ 12.2,
	37	+ 13.6,
	38	+ 14.6,
	39	+ 13.5,
	40	+ 14.9,
	41	+ 13.9,
	42	+ 15.7,
	43	+ 14.1,
	44	+ 15.7,
	45	+ 13.7,
	46	+ 15.2,
	47	+ 13.6,
	48	+ 15.5,
	49	+ 13.8,
	50	+ 15.6,
	51	+ 13.3,
	52	+ 15.0,
	53	+ 13.7,
	54	+ 15.3,
	55	+ 13.6,
	56	+ 15.3,
	57	+ 12.5,
	58	+ 13.9,
	59	+ 12.3,
	60	+ 13.7,
	61	+ 12.3,
	62	+ 13.7,
	63	+ 13.6,
	64	+ 15.2
	65	+ ],
	66	+ "memo info": []
	67	+}

Atests/st/shell_scripts/trl_ppo_llama32_1b_ptd_tp2pp2.sh+116-0

@@ -0,0 +1,116 @@
	1	+#!/bin/bash
	2	+
	3	+export CUDA_DEVICE_MAX_CONNECTIONS=1
	4	+
	5	+GPUS_PER_NODE=4
	6	+MASTER_ADDR=localhost
	7	+MASTER_PORT=6080
	8	+NNODES=1
	9	+NODE_RANK=0
	10	+WORLD_SIZE=$(($GPUS_PER_NODE*$NNODES))
	11	+
	12	+basepath=$(cd `dirname $0`; cd ../../../; pwd)
	13	+
	14	+DATA_PATH=/data/TrlPPOAlpaca_empty/alpaca
	15	+SFT_LOAD_DIR=/data/llama32-mcore-tp2pp2
	16	+REWARD_LOAD_DIR=/data/llama32_1b_reward_tp2pp2
	17	+TOKENIZER_MODEL=/data/Llama-3.2-1B-Instruct
	18	+
	19	+
	20	+TP=2
	21	+PP=2
	22	+
	23	+DISTRIBUTED_ARGS="
	24	+ --nproc_per_node $GPUS_PER_NODE \
	25	+ --nnodes $NNODES \
	26	+ --node_rank $NODE_RANK \
	27	+ --master_addr $MASTER_ADDR \
	28	+ --master_port $MASTER_PORT
	29	+"
	30	+
	31	+GPT_ARGS="
	32	+ --tensor-model-parallel-size ${TP} \
	33	+ --pipeline-model-parallel-size ${PP} \
	34	+ --use-fused-swiglu \
	35	+ --use-mcore-models \
	36	+ --micro-batch-size 2 \
	37	+ --global-batch-size 2 \
	38	+ --use-mc2 \
	39	+ --use-rotary-position-embeddings \
	40	+ --rope-scaling-type llama3 \
	41	+ --rope-scaling-factor 32.0 \
	42	+ --low-freq-factor 1.0 \
	43	+ --high-freq-factor 4.0 \
	44	+ --original-max-position-embeddings 8192 \
	45	+ --tokenizer-type PretrainedFromHF \
	46	+ --tokenizer-name-or-path ${TOKENIZER_MODEL} \
	47	+ --num-layers 16 \
	48	+ --hidden-size 2048 \
	49	+ --ffn-hidden-size 8192 \
	50	+ --num-attention-heads 32 \
	51	+ --group-query-attention \
	52	+ --num-query-groups 8 \
	53	+ --seq-length 131072 \
	54	+ --max-position-embeddings 131072 \
	55	+ --make-vocab-size-divisible-by 1 \
	56	+ --padded-vocab-size 128256 \
	57	+ --disable-bias-linear \
	58	+ --attention-dropout 0.0 \
	59	+ --init-method-std 0.01 \
	60	+ --hidden-dropout 0.0 \
	61	+ --position-embedding-type rope \
	62	+ --rotary-base 500000 \
	63	+ --normalization RMSNorm \
	64	+ --norm-epsilon 1e-5 \
	65	+ --swiglu \
	66	+ --lr 1e-7 \
	67	+ --train-iters 15 \
	68	+ --lr-decay-style constant \
	69	+ --min-lr 1e-7 \
	70	+ --weight-decay 0.0 \
	71	+ --lr-warmup-fraction 0.00 \
	72	+ --clip-grad 1.0 \
	73	+ --adam-beta1 0.9 \
	74	+ --adam-beta2 0.95 \
	75	+ --initial-loss-scale 4096 \
	76	+ --no-gradient-accumulation-fusion \
	77	+ --no-load-optim \
	78	+ --no-load-rng \
	79	+ --finetune \
	80	+ --is-instruction-dataset \
	81	+ --variable-seq-lengths \
	82	+ --attention-softmax-in-fp32 \
	83	+ --no-masked-softmax-fusion \
	84	+ --seq-length 8192 \
	85	+"
	86	+
	87	+DATA_ARGS="
	88	+ --data-path $DATA_PATH \
	89	+ --no-shuffle \
	90	+ --split 100,0,0
	91	+"
	92	+
	93	+OUTPUT_ARGS="
	94	+ --log-interval 1 \
	95	+ --save-interval 2000 \
	96	+ --eval-interval 2000 \
	97	+ --eval-iters 1 \
	98	+"
	99	+
	100	+RL_ARGS="
	101	+ --stage trl_ppo \
	102	+ --max-new-tokens 256 \
	103	+ --max-length 512
	104	+"
	105	+
	106	+torchrun $DISTRIBUTED_ARGS $basepath/posttrain_gpt.py \
	107	+ $GPT_ARGS \
	108	+ $OUTPUT_ARGS \
	109	+ $DATA_ARGS \
	110	+ $RL_ARGS \
	111	+ $PROFILE_ARGS \
	112	+ --log-throughput \
	113	+ --tokenizer-not-use-fast \
	114	+ --distributed-backend nccl \
	115	+ --ref-model ${SFT_LOAD_DIR} \
	116	+ --reward-model ${REWARD_LOAD_DIR} \

Mtests/test_tools/acquire_json.py+5-0

	@@ -26,6 +26,11 @@ def transfer_logs_as_json(log_file, output_json_file):
26	r"throughput per GPU $TFLOP/s/GPU$:\s+([0-9.]+)\s+\\|.?lm loss:\s+([0-9.]+E[+-][0-9]+) \| . actor/pg_loss : ([0-9.]+)"	26	r"throughput per GPU $TFLOP/s/GPU$:\s+([0-9.]+)\s+\\|.?lm loss:\s+([0-9.]+E[+-][0-9]+) \| . actor/pg_loss : ([0-9.]+)"
27	)	27	)
28		28
		29	+ if "trl_ppo" in log_file:
		30	+ log_pattern = re.compile(
		31	+ r'throughput per GPU $TFLOP/s/GPU$:\s([0-9\.]+).*?(?:abs_pg_loss:\|vf_loss:)\s([0-9\.]+)'
		32	+ )
		33	+
29	memory_pattern = re.compile(	34	memory_pattern = re.compile(
30	r"\[Rank (\d+)\] $after \d+ iterations$ memory $MB$ \\| allocated: ([0-9.]+) \\| max allocated: ([0-9.]+)"	35	r"\[Rank (\d+)\] $after \d+ iterations$ memory $MB$ \\| allocated: ([0-9.]+) \\| max allocated: ([0-9.]+)"
31	)	36	)

	@@ -26,6 +26,11 @@ def transfer_logs_as_json(log_file, output_json_file):
26	r"throughput per GPU \(TFLOP/s/GPU\):\s+([0-9.]+)\s+\\|.?lm loss:\s+([0-9.]+E[+-][0-9]+) \| . actor/pg_loss : ([0-9.]+)"	26	r"throughput per GPU \(TFLOP/s/GPU\):\s+([0-9.]+)\s+\\|.?lm loss:\s+([0-9.]+E[+-][0-9]+) \| . actor/pg_loss : ([0-9.]+)"
27	)	27	)
28		28
		29	+ if "trl_ppo" in log_file:
		30	+ log_pattern = re.compile(
		31	+ r'throughput per GPU \(TFLOP/s/GPU\):\s([0-9\.]+).*?(?:abs_pg_loss:\|vf_loss:)\s([0-9\.]+)'
		32	+ )
		33	+
29	memory_pattern = re.compile(	34	memory_pattern = re.compile(
30	r"\[Rank (\d+)\] \(after \d+ iterations\) memory \(MB\) \\| allocated: ([0-9.]+) \\| max allocated: ([0-9.]+)"	35	r"\[Rank (\d+)\] \(after \d+ iterations\) memory \(MB\) \\| allocated: ([0-9.]+) \\| max allocated: ([0-9.]+)"
31	)	36	)