新增Ray_PPO的ST测试-MindSpeed-LLM-AtomGit

Atests/st/baseline_results/ray_ppo_full_llama32_1b_tp1pp1.json+23-0

@@ -0,0 +1,23 @@
	1	+{
	2	+ "lm loss": [
	3	+ 0.4065,
	4	+ 0.4323,
	5	+ 0.3794,
	6	+ 0.3497,
	7	+ 0.3427,
	8	+ 0.4128,
	9	+ 0.3027,
	10	+ 0.4638,
	11	+ 0.3945,
	12	+ 0.3411,
	13	+ 0.4205,
	14	+ 0.4161,
	15	+ 0.3589,
	16	+ 0.3458,
	17	+ 0.4575
	18	+ ],
	19	+ "throughput": [
	20	+ ],
	21	+ "memo info": [
	22	+ ]
	23	+}

Atests/st/configs/model/llama32-1b.yaml+35-0

@@ -0,0 +1,35 @@
	1	+llama32-1b:
	2	+ use_mcore_models: true
	3	+ sequence_parallel: true
	4	+ use_mc2: true
	5	+ use_flash_attn: true
	6	+ use_rotary_position_embeddings: true
	7	+ use_fused_rmsnorm: true
	8	+ use_fused_swiglu: true
	9	+ rope_scaling_type: llama3
	10	+ rope_scaling_factor: 32.0
	11	+ low_freq_factor: 1.0
	12	+ high_freq_factor: 4.0
	13	+ original_max_position_embeddings: 8192
	14	+ max_position_embeddings: 8192
	15	+ num_layers: 16
	16	+ hidden_size: 2048
	17	+ ffn_hidden_size: 8192
	18	+ num_attention_heads: 32
	19	+ group_query_attention: true
	20	+ num_query_groups: 8
	21	+ make_vocab_size_divisible_by: 1
	22	+ padded_vocab_size: 128256
	23	+ disable_bias_linear: true
	24	+ attention_dropout: 0.0
	25	+ init_method_std: 0.01
	26	+ hidden_dropout: 0.0
	27	+ position_embedding_type: rope
	28	+ rotary_base: 500000
	29	+ normalization: RMSNorm
	30	+ norm_epsilon: 1e-5
	31	+ swiglu: true
	32	+ no_masked_softmax_fusion: true
	33	+ attention_softmax_in_fp32: true
	34	+ no_gradient_accumulation_fusion: true
	35	+ bf16: true

Atests/st/configs/ray_ppo_full_llama32_1b_tp1pp1.yaml+112-0

@@ -0,0 +1,112 @@
	1	+defaults:
	2	+ - model:
	3	+ - llama32-1b
	4	+
	5	+training:
	6	+ global_batch_size: 8
	7	+ seq_length: 309
	8	+ tokenizer_type: PretrainedFromHF
	9	+ tokenizer_name_or_path: /data/ppo/llama-3.2-1b-instruct/
	10	+ train_iters: 15
	11	+ distributed_backend: nccl
	12	+ no_shared_storage: true
	13	+ save_interval: 10000
	14	+ no_load_optim: true
	15	+ no_load_rng: true
	16	+ bf16: false
	17	+ is_instruction_dataset: true
	18	+ variable_seq_lengths: true
	19	+ no_shuffle: true
	20	+ stage: ray_ppo
	21	+
	22	+
	23	+actor_rollout_ref:
	24	+ actor_rollout:
	25	+ model: llama32-1b
	26	+ do_sample: false
	27	+ micro_batch_size: 4
	28	+ ppo_mini_batch_size: 4
	29	+ num_samples_per_step: 2
	30	+ max_prompt_length: 256
	31	+ ppo_epochs: 1
	32	+ clip_ratio: 0.2
	33	+ entropy_coeff: 0.001
	34	+ shuffle_minibatch: false
	35	+ use_kv_cache: false
	36	+ tensor_model_parallel_size: 1
	37	+ pipeline_model_parallel_size: 1
	38	+ lr: 1e-7
	39	+ lr_decay_style: constant
	40	+ min_lr: 0.0
	41	+ weight_decay: 0.0
	42	+ lr_warmup_fraction: 0.0
	43	+ clip_grad: 10000.0
	44	+ adam_beta1: 0.9
	45	+ adam_beta2: 0.999
	46	+ initial_loss_scale: 4096
	47	+ finetune: true
	48	+ load: /data/ppo/llama-3.2-1b-instruct-tp1-pp1
	49	+ save: ./ckpt
	50	+ num_gpus_for_train: 1
	51	+ num_gpus_for_infer: 1
	52	+ pad_to_multiple_of: 1
	53	+ data_path: /data/ppo/llama32-ppo-trl/alpaca
	54	+ split: 100,0,0
	55	+
	56	+ ref:
	57	+ model: llama32-1b
	58	+ tensor_model_parallel_size: 1
	59	+ pipeline_model_parallel_size: 1
	60	+ micro_batch_size: 8
	61	+ load: /data/ppo/llama-3.2-1b-instruct-tp1-pp1
	62	+
	63	+critic:
	64	+ model: llama32-1b
	65	+ tensor_model_parallel_size: 1
	66	+ pipeline_model_parallel_size: 1
	67	+ use_mcore_models: True
	68	+ micro_batch_size: 4
	69	+ sequence_parallel: False
	70	+ lr: 1e-7
	71	+ lr_decay_style: constant
	72	+ min_lr: 0.0
	73	+ weight_decay: 0.0
	74	+ lr_warmup_fraction: 0.0
	75	+ use_distributed_optimizer: true
	76	+ clip_grad: 10000.0
	77	+ adam_beta1: 0.9
	78	+ adam_beta2: 0.999
	79	+ initial_loss_scale: 4096
	80	+ no_load_optim: True
	81	+ no_load_rng: True
	82	+ is_instruction_dataset: true
	83	+ variable_seq_lengths: true
	84	+ load: /data/ppo/llama-3.2-1b-rm-mcore-tp1-pp1
	85	+ save: ./ckpt
	86	+ cliprange_value: 0.2
	87	+ critic_mini_batch_size: 4
	88	+ critic_update_epochs: 1
	89	+
	90	+reward:
	91	+ model: llama32-1b
	92	+ tensor_model_parallel_size: 1
	93	+ pipeline_model_parallel_size: 1
	94	+ micro_batch_size: 8
	95	+ sequence_parallel: false
	96	+ load: /data/ppo/llama-3.2-1b-rm-mcore-tp1-pp1
	97	+
	98	+algorithm:
	99	+ gamma: 1.0
	100	+ lam: 0.95
	101	+ adv_estimator: gae
	102	+ kl_penalty: kl
	103	+ kl_ctrl:
	104	+ type: fixed
	105	+ kl_coef: 0.05
	106	+ missing_eos_penalty: 0.0
	107	+
	108	+resource_pool:
	109	+ actor_rollout: [2]
	110	+ ref: [1]
	111	+ critic: [1]
	112	+ reward: [1]

Atests/st/shell_scripts/ray_ppo_full_llama32_1b_tp1pp1.sh+9-0

@@ -0,0 +1,9 @@
	1	+#!/bin/bash
	2	+export CUDA_DEVICE_MAX_CONNECTIONS=1
	3	+export HCCL_DETERMINISTIC=True
	4	+
	5	+
	6	+basepath=$(cd `dirname $0`; cd ../../../; pwd)
	7	+
	8	+
	9	+python $basepath/ray_gpt.py --config-dir=$basepath/tests/st/configs --config-name=ray_ppo_full_llama32_1b_tp1pp1

Mtests/test_tools/acquire_json.py+8-5

	@@ -17,13 +17,13 @@ def transfer_logs_as_json(log_file, output_json_file):
17	Args:	17	Args:
18	log_file: str, path to the dir where the logs are located.	18	log_file: str, path to the dir where the logs are located.
19	output_json_file: str, path of the json file transferred from the logs.	19	output_json_file: str, path of the json file transferred from the logs.
20	-	20	+
21	Returns:	21	Returns:
22	data: json, the values parsed from the log, formatted as a json file.	22	data: json, the values parsed from the log, formatted as a json file.
23	"""	23	"""
24	-	24	+
25	log_pattern = re.compile(	25	log_pattern = re.compile(
26	- r"throughput per GPU $TFLOP/s/GPU$:\s+([0-9.]+)\s+\\|.*?lm loss:\s+([0-9.]+E[+-][0-9]+)"	26	+ r"throughput per GPU $TFLOP/s/GPU$:\s+([0-9.]+)\s+\\|.?lm loss:\s+([0-9.]+E[+-][0-9]+) \| . critic/vf_loss : ([0-9.]+)"
27	)	27	)
28		28
29	memory_pattern = re.compile(	29	memory_pattern = re.compile(
	@@ -41,8 +41,11 @@ def transfer_logs_as_json(log_file, output_json_file):
41	memory_matches = memory_pattern.findall(log_content)	41	memory_matches = memory_pattern.findall(log_content)
42		42
43	if log_matches:	43	if log_matches:
44	- data["lm loss"] = [float(match[1]) for match in log_matches]	44	+ if log_matches[0][1] != "":
45	- data["throughput"] = [float(match[0]) for match in log_matches]	45	+ data["lm loss"] = [float(match[1]) for match in log_matches]
		46	+ data["throughput"] = [float(match[0]) for match in log_matches]
		47	+ else:
		48	+ data["lm loss"] = [float(match[2]) for match in log_matches]
46		49
47	if memory_matches:	50	if memory_matches:
48	memo_info = [	51	memo_info = [

Mtests/test_tools/test_ci_st.py+3-1

	@@ -55,8 +55,10 @@ class TestCIST:
55	}	55	}
56		56
57	if test_obj in comparison_selection:	57	if test_obj in comparison_selection:
58	- print(f"===================== Begin comparing {test_obj} ===================")
59	expected_list = self.expected[test_obj]	58	expected_list = self.expected[test_obj]
		59	+ if not expected_list:
		60	+ return
		61	+ print(f"===================== Begin comparing {test_obj} ===================")
60	actual_list = self.actual[test_obj]	62	actual_list = self.actual[test_obj]
61	print(f"The list of expected values: {expected_list}")	63	print(f"The list of expected values: {expected_list}")
62	print(f"The list of actual values: {actual_list}")	64	print(f"The list of actual values: {actual_list}")