已合并
fix: Disallow use-global-aux-loss when moe-alltoall-overlap-comm is enabled #4389
zzyyjj012创建于 4月8日
fix: Disallow use-global-aux-loss when moe-alltoall-overlap-comm is enabled #4389
已合并
共 1 个文件变更+6-0
| @@ -37,6 +37,12 @@ class Qwen3NextFeature(MindSpeedFeature): | |||
| 37 | if args.use_global_aux_loss: | 37 | if args.use_global_aux_loss: |
| 38 | if int(args.pipeline_model_parallel_size) > 1: | 38 | if int(args.pipeline_model_parallel_size) > 1: |
| 39 | raise AssertionError('use-global-aux-loss is not support for pipeline-model-parallel-size > 1, please use FSDP2') | 39 | raise AssertionError('use-global-aux-loss is not support for pipeline-model-parallel-size > 1, please use FSDP2') |
| 40 | + if args.moe_alltoall_overlap_comm: | ||
| 41 | + raise AssertionError('`--use-global-aux-loss` is not support for `--moe-alltoall-overlap-comm`') | ||
| 42 | + if args.moe_allgather_overlap_comm: | ||
| 43 | + raise AssertionError('`--use-global-aux-loss` is not support for `--moe-allgather-overlap-comm`') | ||
| 44 | + if args.moe_fb_overlap: | ||
| 45 | + raise AssertionError('`--use-global-aux-loss` is not support for `--moe-fb-overlap`') | ||
| 40 | 46 | ||
| 41 | patch_manager.register_patch('megatron.core.transformer.moe.router.TopKRouter.forward', | 47 | patch_manager.register_patch('megatron.core.transformer.moe.router.TopKRouter.forward', |
| 42 | global_aux_loss_topk_router_forward) | 48 | global_aux_loss_topk_router_forward) |