已合并
增加verl qwen3+dapo+A3单机脚本，优化了A3单机性能，平均单卡吞吐290TPS #794
imzhengwei创建于 2025年12月5日
增加verl qwen3+dapo+A3单机脚本，优化了A3单机性能，平均单卡吞吐290TPS #794
已合并
共 2 个文件变更+217-0
| @@ -0,0 +1,138 @@ | |||
| 1 | +set -xeuo pipefail | ||
张 | |||
| 2 | + | ||
Z  贴一下精度和性能的验证报告   精度和性能链接打不开，请开通访问权限！ | |||
| 3 | +project_name='DAPO' | ||
| 4 | +exp_name='DAPO-Qwen3-32B' | ||
| 5 | + | ||
| 6 | +adv_estimator=grpo | ||
| 7 | + | ||
| 8 | +use_kl_in_reward=False | ||
| 9 | +kl_coef=0.0 | ||
| 10 | +use_kl_loss=False | ||
| 11 | +kl_loss_coef=0.0 | ||
| 12 | + | ||
| 13 | +clip_ratio_low=0.2 | ||
| 14 | +clip_ratio_high=0.28 | ||
| 15 | + | ||
Z 目前正在版本迭代，这两天verl和vllm会升级到新版本，在新版本上验证完直接上新版本吧 | |||
| 16 | +max_prompt_length=$((1024 * 2)) | ||
| 17 | +max_response_length=$((1024 * 10)) | ||
| 18 | +enable_overlong_buffer=True | ||
| 19 | +overlong_buffer_len=$((1024 * 4)) | ||
| 20 | +overlong_penalty_factor=1.0 | ||
| 21 | + | ||
| 22 | +loss_agg_mode="token-mean" | ||
| 23 | + | ||
| 24 | +use_token_level_loss=True | ||
| 25 | +enable_filter_groups=False | ||
| 26 | +filter_groups_metric=acc | ||
| 27 | +max_num_gen_batches=10 | ||
| 28 | +train_prompt_bsz=32 | ||
| 29 | +gen_prompt_bsz=$((train_prompt_bsz * 1)) | ||
| 30 | +n_resp_per_prompt=16 | ||
| 31 | +train_prompt_mini_bsz=32 | ||
| 32 | + | ||
| 33 | +# Ray | ||
| 34 | +WORKING_DIR=${WORKING_DIR:-"${PWD}"} | ||
| 35 | +RUNTIME_ENV=${RUNTIME_ENV:-"${WORKING_DIR}/verl/trainer/runtime_env.yaml"} | ||
这两个参数后续没有用到，建议删除。 | |||
| 36 | +NNODES={NNODES:-1} | ||
| 37 | +NPUS_PER_NODE={NPUS_PER_NODE:-16} | ||
| 38 | + | ||
| 39 | +# Paths | ||
| 40 | +MODEL_PATH=./ckpt/Qwen3-32B | ||
| 41 | +CKPTS_DIR=./ckpt/Qwen3-32B-save | ||
| 42 | +TRAIN_FILE=./data/dapo-math-17k.parquet | ||
| 43 | +TEST_FILE=./data/dapo-math-17k.parquet | ||
| 44 | + | ||
| 45 | +# Algorithm | ||
| 46 | +temperature=1.0 | ||
| 47 | +top_p=1.0 | ||
| 48 | +top_k=-1 # 0 for HF rollout, -1 for vLLM rollout | ||
| 49 | +val_top_p=0.7 | ||
| 50 | + | ||
| 51 | +# Performance Related Parameter | ||
| 52 | +sp_size=4 | ||
| 53 | +use_dynamic_bsz=True | ||
| 54 | +actor_ppo_max_token_len=$(((max_prompt_length + max_response_length) / sp_size)) | ||
| 55 | +infer_ppo_max_token_len=$(((max_prompt_length + max_response_length) / sp_size)) | ||
| 56 | +offload=True | ||
| 57 | +gen_tp=4 | ||
| 58 | + | ||
| 59 | +python3 -m recipe.dapo.main_dapo \ | ||
| 60 | + data.train_files="${TRAIN_FILE}" \ | ||
| 61 | + data.val_files="${TEST_FILE}" \ | ||
| 62 | + data.prompt_key=prompt \ | ||
| 63 | + data.truncation='left' \ | ||
| 64 | + data.max_prompt_length=${max_prompt_length} \ | ||
| 65 | + data.max_response_length=${max_response_length} \ | ||
| 66 | + data.gen_batch_size=${gen_prompt_bsz} \ | ||
| 67 | + data.train_batch_size=${train_prompt_bsz} \ | ||
| 68 | + actor_rollout_ref.rollout.n=${n_resp_per_prompt} \ | ||
| 69 | + algorithm.adv_estimator=${adv_estimator} \ | ||
| 70 | + algorithm.use_kl_in_reward=${use_kl_in_reward} \ | ||
| 71 | + algorithm.kl_ctrl.kl_coef=${kl_coef} \ | ||
| 72 | + actor_rollout_ref.actor.use_kl_loss=${use_kl_loss} \ | ||
| 73 | + actor_rollout_ref.actor.kl_loss_coef=${kl_loss_coef} \ | ||
| 74 | + actor_rollout_ref.actor.clip_ratio_low=${clip_ratio_low} \ | ||
| 75 | + actor_rollout_ref.actor.clip_ratio_high=${clip_ratio_high} \ | ||
| 76 | + actor_rollout_ref.actor.clip_ratio_c=10.0 \ | ||
| 77 | + algorithm.filter_groups.enable=${enable_filter_groups} \ | ||
| 78 | + algorithm.filter_groups.max_num_gen_batches=${max_num_gen_batches} \ | ||
| 79 | + algorithm.filter_groups.metric=${filter_groups_metric} \ | ||
| 80 | + actor_rollout_ref.model.use_remove_padding=True \ | ||
| 81 | + actor_rollout_ref.actor.use_dynamic_bsz=${use_dynamic_bsz} \ | ||
| 82 | + actor_rollout_ref.ref.log_prob_use_dynamic_bsz=${use_dynamic_bsz} \ | ||
| 83 | + actor_rollout_ref.rollout.log_prob_use_dynamic_bsz=${use_dynamic_bsz} \ | ||
| 84 | + actor_rollout_ref.actor.ppo_max_token_len_per_gpu=${actor_ppo_max_token_len} \ | ||
| 85 | + actor_rollout_ref.ref.log_prob_max_token_len_per_gpu=${infer_ppo_max_token_len} \ | ||
| 86 | + actor_rollout_ref.rollout.log_prob_max_token_len_per_gpu=${infer_ppo_max_token_len} \ | ||
| 87 | + actor_rollout_ref.model.path="${MODEL_PATH}" \ | ||
| 88 | + actor_rollout_ref.model.enable_gradient_checkpointing=True \ | ||
| 89 | + actor_rollout_ref.actor.optim.lr=1e-6 \ | ||
| 90 | + actor_rollout_ref.actor.optim.lr_warmup_steps=10 \ | ||
| 91 | + actor_rollout_ref.actor.optim.weight_decay=0.1 \ | ||
| 92 | + actor_rollout_ref.actor.ppo_mini_batch_size=${train_prompt_mini_bsz} \ | ||
| 93 | + actor_rollout_ref.actor.fsdp_config.param_offload=${offload} \ | ||
| 94 | + actor_rollout_ref.actor.fsdp_config.optimizer_offload=${offload} \ | ||
| 95 | + actor_rollout_ref.actor.entropy_coeff=0 \ | ||
| 96 | + actor_rollout_ref.actor.grad_clip=1.0 \ | ||
| 97 | + actor_rollout_ref.rollout.name=vllm \ | ||
| 98 | + actor_rollout_ref.actor.loss_agg_mode=${loss_agg_mode} \ | ||
| 99 | + actor_rollout_ref.actor.ulysses_sequence_parallel_size=${sp_size} \ | ||
| 100 | + actor_rollout_ref.rollout.gpu_memory_utilization=0.85 \ | ||
| 101 | + actor_rollout_ref.rollout.tensor_model_parallel_size=${gen_tp} \ | ||
| 102 | + actor_rollout_ref.rollout.enable_chunked_prefill=True \ | ||
| 103 | + actor_rollout_ref.rollout.max_num_batched_tokens=$((max_prompt_length + max_response_length)) \ | ||
| 104 | + actor_rollout_ref.rollout.temperature=${temperature} \ | ||
| 105 | + actor_rollout_ref.rollout.top_p=${top_p} \ | ||
| 106 | + actor_rollout_ref.rollout.top_k="${top_k}" \ | ||
| 107 | + actor_rollout_ref.rollout.val_kwargs.temperature=${temperature} \ | ||
| 108 | + actor_rollout_ref.rollout.val_kwargs.top_p=${val_top_p} \ | ||
| 109 | + actor_rollout_ref.rollout.val_kwargs.top_k=${top_k} \ | ||
| 110 | + actor_rollout_ref.rollout.val_kwargs.do_sample=True \ | ||
| 111 | + actor_rollout_ref.rollout.val_kwargs.n=1 \ | ||
| 112 | + actor_rollout_ref.ref.fsdp_config.param_offload=${offload} \ | ||
| 113 | + actor_rollout_ref.ref.ulysses_sequence_parallel_size=${sp_size} \ | ||
| 114 | + actor_rollout_ref.actor.fsdp_config.fsdp_size=-1 \ | ||
| 115 | + actor_rollout_ref.ref.strategy=fsdp2 \ | ||
| 116 | + actor_rollout_ref.actor.strategy=fsdp2 \ | ||
| 117 | + reward_model.reward_manager=dapo \ | ||
| 118 | + reward_model.overlong_buffer.enable=${enable_overlong_buffer} \ | ||
| 119 | + reward_model.overlong_buffer.len=${overlong_buffer_len} \ | ||
| 120 | + reward_model.overlong_buffer.penalty_factor=${overlong_penalty_factor} \ | ||
| 121 | + trainer.logger='["console"]' \ | ||
| 122 | + trainer.project_name="${project_name}" \ | ||
| 123 | + trainer.experiment_name="${exp_name}" \ | ||
| 124 | + trainer.n_gpus_per_node="${NPUS_PER_NODE}" \ | ||
| 125 | + trainer.nnodes="${NNODES}" \ | ||
| 126 | + trainer.val_before_train=False \ | ||
| 127 | + trainer.save_freq=100 \ | ||
| 128 | + trainer.test_freq=100 \ | ||
| 129 | + trainer.total_epochs=10 \ | ||
| 130 | + trainer.default_local_dir="${CKPTS_DIR}" \ | ||
| 131 | + trainer.resume_mode=auto \ | ||
| 132 | + trainer.balance_batch=True \ | ||
| 133 | + actor_rollout_ref.rollout.enforce_eager=False \ | ||
| 134 | + actor_rollout_ref.actor.use_torch_compile=False \ | ||
| 135 | + actor_rollout_ref.ref.use_torch_compile=False \ | ||
| 136 | + actor_rollout_ref.actor.fsdp_config.forward_prefetch=True \ | ||
| 137 | + actor_rollout_ref.actor.entropy_from_logits_with_chunking=True \ | ||
| 138 | + trainer.device=npu 2>&1 | tee "logs/verl_qwen3_32b_$(date +%Y%m%d_%H%M).log" | ||
| @@ -0,0 +1,79 @@ | |||
| 1 | +pkill -9 python | ||
| 2 | +ray stop --force | ||
| 3 | +rm -rf /tmp/ray | ||
| 4 | + | ||
| 5 | +export RAY_DEDUP_LOGS=0 | ||
| 6 | +export HYDRA_FULL_ERROR=1 | ||
| 7 | + | ||
| 8 | +# ------------- HCCL优化 -------------# | ||
| 9 | +export HCCL_ASYNC_ERROR_HANDLING=0 | ||
| 10 | +export HCCL_EXEC_TIMEOUT=3600 | ||
| 11 | +export HCCL_CONNECT_TIMEOUT=3600 | ||
| 12 | +export HCCL_OPEXPANSION_MODE=AIV | ||
| 13 | +export HCCL_BUFFSIZE=256 | ||
| 14 | +export ATB_LLM_HCCL_ENABLE=1 | ||
| 15 | + | ||
| 16 | +# TASK_QUEUE_ENABLE，下发优化，图模式设置为1，非图模式设置为2 | ||
| 17 | +export TASK_QUEUE_ENABLE=1 | ||
| 18 | +# 使能vllm V1 Engine | ||
| 19 | +export VLLM_USE_V1=1 | ||
| 20 | +# 请确认该文件存在(可通过 find /usr -name libjemalloc.so.2 确认) | ||
| 21 | +export LD_PRELOAD=/usr/lib/aarch64-linux-gnu/libjemalloc.so.2 | ||
| 22 | + | ||
| 23 | +#修改为当前需要跑的用例路径 | ||
| 24 | +DEFAULT_SH="./test_dapo_qwen3_32b_fsdp2_A3_1_node.sh" | ||
| 25 | +echo "Use $DEFAULT_SH" | ||
| 26 | + | ||
| 27 | +ulimit -n 32768 | ||
| 28 | +mkdir logs | ||
| 29 | + | ||
| 30 | +NNODES=1 | ||
| 31 | +NPUS_PER_NODE=16 | ||
| 32 | +#修改为对应主节点IP | ||
| 33 | +MASTER_ADDR="IP FOR MASTER NODE" | ||
| 34 | +#修改为当前节点的通信网卡 | ||
| 35 | +SOCKET_IFNAME="Your SOCKET IFNAME" | ||
| 36 | +export HCCL_SOCKET_IFNAME="SOCKET IFNAME FOR CURRENT NODE" | ||
| 37 | +export GLOO_SOCKET_IFNAME="SOCKET IFNAME FOR CURRENT NODE" | ||
| 38 | +#获取当前节点IP | ||
| 39 | +CURRENT_IP=$(ifconfig $SOCKET_IFNAME | grep -Eo 'inet (addr:)?([0-9]{1,3}\.){3}[0-9]{1,3}' | awk '{print $NF}') | ||
| 40 | +if [ "$MASTER_ADDR" = "$CURRENT_IP" ]; then | ||
| 41 | + # 主节点启动 | ||
| 42 | + ray start --head --port 6766 --dashboard-host=$MASTER_ADDR --node-ip-address=$CURRENT_IP --dashboard-port=8260 --resources='{"NPU": '$NPUS_PER_NODE'}' | ||
| 43 | + | ||
| 44 | + while true; do | ||
| 45 | + ray_status_output=$(ray status) | ||
| 46 | + npu_count=$(echo "$ray_status_output" | grep -oP '(?<=/)\d+\.\d+(?=\s*NPU)' | head -n 1) | ||
| 47 | + npu_count_int=$(echo "$npu_count" | awk '{print int($1)}') | ||
| 48 | + device_count=$((npu_count_int / $NPUS_PER_NODE)) | ||
| 49 | + | ||
| 50 | + # 判断 device_count 是否与 NNODES 相等 | ||
| 51 | + if [ "$device_count" -eq "$NNODES" ]; then | ||
| 52 | + echo "Ray cluster is ready with $device_count devices (from $npu_count NPU resources), starting Python script." | ||
| 53 | + ray status | ||
| 54 | + bash $DEFAULT_SH | ||
| 55 | + break | ||
| 56 | + else | ||
| 57 | + echo "Waiting for Ray to allocate $NNODES devices. Current device count: $device_count" | ||
| 58 | + sleep 5 | ||
| 59 | + fi | ||
| 60 | + done | ||
| 61 | +else | ||
| 62 | + # 子节点尝试往主节点注册ray直到成功 | ||
| 63 | + while true; do | ||
| 64 | + # 尝试连接 Ray 集群 | ||
| 65 | + ray start --address="$MASTER_ADDR:6766" --resources='{"NPU": '$NPUS_PER_NODE'}' --node-ip-address=$CURRENT_IP | ||
| 66 | + | ||
| 67 | + # 检查连接是否成功 | ||
| 68 | + ray status | ||
| 69 | + if [ $? -eq 0 ]; then | ||
| 70 | + echo "Successfully connected to the Ray cluster!" | ||
| 71 | + break | ||
| 72 | + else | ||
| 73 | + echo "Failed to connect to the Ray cluster. Retrying in 5 seconds..." | ||
| 74 | + sleep 5 | ||
| 75 | + fi | ||
| 76 | + done | ||
| 77 | +fi | ||
| 78 | + | ||
| 79 | +sleep 600 | ||
建议脚本和配置文件的名称可以带上节点数量, 例如: test_dapo_qwen3_32b_fsdp2_A3_1_node.sh