已合并
refactor: modify default qwen3.5 mocre config #2879
gaojie创建于 7月24日
refactor: modify default qwen3.5 mocre config #2879
已合并
共 1 个文件变更+4-32
| @@ -7,7 +7,7 @@ gpt_args: | |||
| 7 | context_parallel_algo: ulysses_cp_algo | 7 | context_parallel_algo: ulysses_cp_algo |
| 8 | tensor_model_parallel_size: 2 | 8 | tensor_model_parallel_size: 2 |
| 9 | sequence_parallel: true | 9 | sequence_parallel: true |
| 10 | - expert_model_parallel_size: 4 | 10 | + expert_model_parallel_size: 8 |
| 11 | pipeline_model_parallel_size: 1 | 11 | pipeline_model_parallel_size: 1 |
| 12 | 12 | ||
| 13 | ## training: | 13 | ## training: |
| @@ -91,7 +91,7 @@ model: | |||
| 91 | text_decoder: | 91 | text_decoder: |
| 92 | model_id: qwen3_5_moe | 92 | model_id: qwen3_5_moe |
| 93 | # parallel | 93 | # parallel |
| 94 | - expert_model_parallel_size: 4 | 94 | + expert_model_parallel_size: 8 |
| 95 | tensor_model_parallel_size: 2 | 95 | tensor_model_parallel_size: 2 |
| 96 | pipeline_model_parallel_size: 1 | 96 | pipeline_model_parallel_size: 1 |
| 97 | sequence_parallel: true | 97 | sequence_parallel: true |
| @@ -151,7 +151,7 @@ model: | |||
| 151 | apply_rotary_pos_emb_in_fp32: false | 151 | apply_rotary_pos_emb_in_fp32: false |
| 152 | max_position_embeddings: 8192 | 152 | max_position_embeddings: 8192 |
| 153 | vocab_size: 248320 | 153 | vocab_size: 248320 |
| 154 | - recompute_num_layers: 0 | 154 | + recompute_num_layers: 40 |
| 155 | recompute_granularity: full | 155 | recompute_granularity: full |
| 156 | recompute_method: block | 156 | recompute_method: block |
| 157 | patch: | 157 | patch: |
| @@ -210,35 +210,7 @@ data: | |||
| 210 | dataloader_mode: sampler | 210 | dataloader_mode: sampler |
| 211 | drop_last: true | 211 | drop_last: true |
| 212 | sampler_type: BaseRandomBatchSampler | 212 | sampler_type: BaseRandomBatchSampler |
| 213 | - num_workers: 0 | 213 | + num_workers: 8 |
| 214 | collate_param: | 214 | collate_param: |
| 215 | model_name: qwen3vl | 215 | model_name: qwen3vl |
| 216 | ignore_pad_token_for_loss: true | 216 | ignore_pad_token_for_loss: true |
| 217 | - | ||
| 218 | - | ||
| 219 | -# 工具配置 | ||
| 220 | -tools: | ||
| 221 | - profile: | ||
| 222 | - enable: false | ||
| 223 | - profile_type: static | ||
| 224 | - ranks: [0] | ||
| 225 | - static_param: | ||
| 226 | - level: level1 | ||
| 227 | - with_stack: false | ||
| 228 | - with_memory: false | ||
| 229 | - record_shapes: false | ||
| 230 | - with_cpu: true | ||
| 231 | - save_path: ./profiling | ||
| 232 | - start_step: 10 | ||
| 233 | - end_step: 11 | ||
| 234 | - data_simplification: false | ||
| 235 | - aic_metrics_type: PipeUtilization | ||
| 236 | - memory_profile: | ||
| 237 | - enable: false | ||
| 238 | - start_step: 1 | ||
| 239 | - end_step: 2 | ||
| 240 | - save_path: ./memory_snapshot | ||
| 241 | - dump_ranks: [0] | ||
| 242 | - stacks: all | ||
| 243 | - max_entries: null | ||
| 244 | - mem_info: false | ||