已开启
test ci fail for verl #863
yezhibei创建于 2025年12月23日
test ci fail for verl #863
已开启
共 1 个文件变更+22-0
| @@ -0,0 +1,22 @@ | |||
| 1 | +diff --git a/recipe/dapo/dapo_ray_trainer.py b/recipe/dapo/dapo_ray_trainer.py | ||
| 2 | +index 18995f67..db209482 100644 | ||
| 3 | +--- a/recipe/dapo/dapo_ray_trainer.py | ||
| 4 | ++++ b/recipe/dapo/dapo_ray_trainer.py | ||
| 5 | + class RayDAPOTrainer(RayPPOTrainer): | ||
| 6 | + non_tensor_batch_keys=["raw_prompt_ids"], | ||
| 7 | + ) | ||
| 8 | + gen_batch_output = gen_batch.repeat( | ||
| 9 | +- repeat_times=self.config.actor_rollout_ref.rollout.n, interleave=True | ||
| 10 | ++ repeat_times=self.config.actor_rollout_ref.rollout.n, interleave=False | ||
| 11 | + ) | ||
| 12 | + | ||
| 13 | + is_last_step = self.global_steps >= self.total_training_steps | ||
| 14 | + class RayDAPOTrainer(RayPPOTrainer): | ||
| 15 | + [str(uuid.uuid4()) for _ in range(len(new_batch.batch))], dtype=object | ||
| 16 | + ) | ||
| 17 | + # repeat to align with repeated responses in rollout | ||
| 18 | +- new_batch = new_batch.repeat(repeat_times=self.config.actor_rollout_ref.rollout.n, interleave=True) | ||
| 19 | ++ new_batch = new_batch.repeat(repeat_times=self.config.actor_rollout_ref.rollout.n, interleave=False) | ||
| 20 | + new_batch = new_batch.union(gen_batch_output) | ||
| 21 | + | ||
| 22 | + if self.config.algorithm.use_kl_in_reward: | ||