checkpoint: async_save: false enable_save: false load_balanced: false load_path: '' load_worker_number: 1 no_load_optim: true no_save_optim: false prefix: custom remove_redundancy: false save_interleaved_steps: 100 save_max: 1 save_path: '' training: deterministic: false global_batch_size: 1024 local_batch_size: 1 max_norm: 1.0 seed: 42 steps: 8000000000 optimizer: adamw_betas: - 0.9 - 0.95 adamw_eps: 1.0e-08 type: Muon use_fused_adamw: true weight_decay: 0.1 adamw_include: - '*word_embeddings*' - '*output_layer*' matched_adamw_rms: 0.2 momentum: 0.95 nesterov: true ns_coefficients: [[[3.4445, -4.7750, 2.0315], 8], [[2.0, -1.5, 0.5], 2]] ns_steps: 8 comm_strategy: allgather_deredundency use_fused_adamw: True phase0_batch_memory_gb: 1 # 动量预处理 → 主要影响 reserved 高水位 ns_batch_memory_gb: 1 # Newton-Schulz → 主要影响 allocated 峰值 phase4_batch_memory_gb: 4 qk_clip_enabled: true qk_clip_threshold: 100.0 lr_scheduler: type: ConstantWarmUpLR learning_rate: 0.0001 warmup_ratio: 0 parallelism: context_parallel: 1 context_parallel_method: colossal data_parallel_shard: -1 data_parallel_shard_strategy: optim_grads_params enable_loss_parallel: true enable_mc2: true expert_parallel: 64 sharded_accumulated_grad: false expert_parallel_async_d2h: true expert_parallel_use_safe_tokens: false moe_token_dispatcher_type: alltoall pipeline_parallel: 8 pipeline_parallel_interleave_num: 2 pipeline_parallel_layers_per_stage: - 0-2,31-34 - 3-6,35-37 - 7-10,38-41 - 11-14,42-45 - 15-18,46-49 - 19-22,50-53 - 23-26,54-58 - 27-30,59-60 pipeline_parallel_overlap_b_f: true pipeline_parallel_overlap_p2p: true pipeline_parallel_p2p_transport: "edge" #"multi_stream" pipeline_parallel_schedule: 1f1b sequence_parallel: true tensor_parallel: 4 model: add_bias_linear: false apply_rope_fusion: true architectures: DeepseekV4ForCausalLM attention_dropout: 0.0 chunk_loss_num: 4 compute_dtype: bfloat16 hidden_act: "fusedswiglu" #silu swiglu_limit: 10.0 hidden_dropout: 0.0 hidden_size: 7168 initializer_range: 0.02 kv_lora_rank: 512 layernorm_compute_dtype: float32 max_position_embeddings: 4096 model_type: deepseek_v4 moe_aux_loss_coeff: 0.0006 moe_grouped_gemm: true moe_intermediate_size: 2048 moe_permute_fusion: true moe_router_bias_update_rate: 0.002 moe_router_enable_expert_bias: true moe_router_force_expert_balance: true moe_router_load_balancing_type: seq_aux_loss moe_shared_expert_intermediate_size: 2048 mtp_loss_scaling_factor: 0.3 multi_latent_attention: true n_routed_experts: 384 n_shared_experts: 1 num_attention_heads: 128 num_experts_per_tok: 8 num_hidden_layers: 61 num_nextn_predict_layers: 1 params_dtype: bfloat16 position_embedding_type: yarn q_lora_rank: 1536 qk_layernorm: true qk_nope_head_dim: 128 qk_rope_head_dim: 64 rms_norm_eps: 1.0e-06 rope_theta: 10000 rotary_dtype: float32 routed_scaling_factor: 2.5 scoring_func: sigmoid seq_length: 4096 softmax_compute_dtype: float32 use_flash_attention: true v_head_dim: 128 vocab_size: 131072 enable_hc_head: false enable_hyper_connections: true use_fused_mhc: True experimental_attention_variant: mla hc_eps: 1.0e-06 hc_mult: 4 hc_sinkhorn_iters: 20 mla_qkv_concat: true moe_router_pre_softmax: true moe_router_score_function: sigmoid moe_router_topk_scaling_factor: 2.5 moe_token_dispatcher_type: alltoall moe_token_drop_policy: probs mtp_loss_factor: 0.3 normalization: RMSNorm num_hash_layers: 3 qk_clip: true recompute: mode: select #select full_recompute_layer: [0-2, 3-5, 7-9, 11-13, 15-17, 19-21, 23-25, 27-28, 31-32, 35-36, 38-40, 42-44, 46-47, 50-51, 54-55] #full_recompute_layer: [0-2, 3-4, 7-8, 11-12, 15-16, 19-20, 23-24, 27-28, 31-32, 35-36, 38-41, 42-44, 46-47, 50-51, 54-55] #full_recompute_layer: [0-2, 3-5, 7-9, 11-13, 15-17, 19-21, 23-25, 27-29, 31-33, 35-36, 38-40, 42-44, 46-48, 50-52, 54-56, 59] #full_recompute_layer: [0-61] #exclude_op: ['AllToAll'] select_module: mlp.experts.activation_func: ["0-61"] mlp.shared_experts.activation_func: ["0-61"] transformer_layer.mlp.experts.activation_func: ["62"] transformer_layer.mlp.shared_experts.activation_func: ["62"] #recompute: # mode: full #full_recompute_layer: [0-3, 4-7, 8-11, 12-15, 16-19, 20-23, 24-27, 28-31, 32-35, 36-39, 40-43, 44-47, 48-51, 52-54, 56-58, 60] #full_recompute_layer: [0-2, 3-5, 7-9, 11-13, 15-17, 19-21, 23-25, 27-28, 31-32, 35-36, 38-40, 42-44, 46-47, 50-51, 54-55] #full_recompute_layer: [0-2, 4-6, 8-10, 12-14, 16-18, 20-22, 24-26, 28-30, 32-34, 35-37, 38-40, 42-44, 46-47, 50-52, 54] #full_recompute_layer: [0-61] #full_recompute_layer: #[0-2,4-6,8-10,12-14,16-18,20-22,24-26,28-30,32-34,36-38,40-42,44-46,48-50,52-54,56-57,60-61] #[0-1,4-5,8-9,12-13,16-17,20-21,24-25,28-29,32-33,36-37,40-41,44-45,48-49,52-53,556-57,60] #[0-1,4-5,8-9,12-13,16-17,20-21,24-25,28] #[0-31] #exclude_op: ['AllToAll'] train_dataset: dataloader: column_names: - input_ids - labels - loss_mask - position_ids type: BlendedMegatronDatasetDataLoader config: create_attention_mask: false create_compressed_eod_mask: false data_path: - '1' - /hpfs/huawei/public/deepseek4_flash/datasets/mmap_deepseekv4_datasets_text_document eod: 1 eod_mask_loss: false eod_pad_length: 128 pad: -1 reset_attention_mask: false reset_position_ids: false seed: 1234 seq_length: 4096 split: 1, 0, 0 datasets_type: GPTDataset sizes: - 512000 - 0 - 0 profiler: enable_profiling: True #True end_step: 5 output_path: /hpfs/huawei/0727lc/prof_lc0728_1513_level1 profile_cpu: False profile_memory: False profiler_level: 1 profiler_rank: [0, 128, 256, 384, 512, 640, 768, 896] #profiler_rank: [768,769,770,771,772,773,774,775,776,777,778,779,780,781,782,783,784,785,786,787,788,789,790,791,792,793,794,795,796,797,798,799,800] #profiler_rank: [768,774,784,785,787,788,792,800,808,816,824,830] start_step: 5 with_stack: False
checkpoint: async_save: false enable_save: false load_balanced: false load_path: '' load_worker_number: 1 no_load_optim: true no_save_optim: false prefix: custom remove_redundancy: false save_interleaved_steps: 100 save_max: 1 save_path: '' training: deterministic: false global_batch_size: 1024 local_batch_size: 1 max_norm: 1.0 seed: 42 steps: 8000000000 optimizer: adamw_betas: - 0.9 - 0.95 adamw_eps: 1.0e-08 type: Muon use_fused_adamw: true weight_decay: 0.1 adamw_include: - '*word_embeddings*' - '*output_layer*' matched_adamw_rms: 0.2 momentum: 0.95 nesterov: true ns_coefficients: [[[3.4445, -4.7750, 2.0315], 8], [[2.0, -1.5, 0.5], 2]] ns_steps: 8 comm_strategy: allgather_deredundency use_fused_adamw: True phase0_batch_memory_gb: 1 # 动量预处理 → 主要影响 reserved 高水位 ns_batch_memory_gb: 1 # Newton-Schulz → 主要影响 allocated 峰值 phase4_batch_memory_gb: 4 qk_clip_enabled: true qk_clip_threshold: 100.0 lr_scheduler: type: ConstantWarmUpLR learning_rate: 0.0001 warmup_ratio: 0 parallelism: context_parallel: 1 context_parallel_method: colossal data_parallel_shard: -1 data_parallel_shard_strategy: optim_grads_params enable_loss_parallel: true enable_mc2: true expert_parallel: 64 sharded_accumulated_grad: false expert_parallel_async_d2h: true expert_parallel_use_safe_tokens: false moe_token_dispatcher_type: alltoall pipeline_parallel: 8 pipeline_parallel_interleave_num: 2 pipeline_parallel_layers_per_stage: - 0-2,31-34 - 3-6,35-37 - 7-10,38-41 - 11-14,42-45 - 15-18,46-49 - 19-22,50-53 - 23-26,54-58 - 27-30,59-60 pipeline_parallel_overlap_b_f: true pipeline_parallel_overlap_p2p: true pipeline_parallel_p2p_transport: "edge" #"multi_stream" pipeline_parallel_schedule: 1f1b sequence_parallel: true tensor_parallel: 4 model: add_bias_linear: false apply_rope_fusion: true architectures: DeepseekV4ForCausalLM attention_dropout: 0.0 chunk_loss_num: 4 compute_dtype: bfloat16 hidden_act: "fusedswiglu" #silu swiglu_limit: 10.0 hidden_dropout: 0.0 hidden_size: 7168 initializer_range: 0.02 kv_lora_rank: 512 layernorm_compute_dtype: float32 max_position_embeddings: 4096 model_type: deepseek_v4 moe_aux_loss_coeff: 0.0006 moe_grouped_gemm: true moe_intermediate_size: 2048 moe_permute_fusion: true moe_router_bias_update_rate: 0.002 moe_router_enable_expert_bias: true moe_router_force_expert_balance: true moe_router_load_balancing_type: seq_aux_loss moe_shared_expert_intermediate_size: 2048 mtp_loss_scaling_factor: 0.3 multi_latent_attention: true n_routed_experts: 384 n_shared_experts: 1 num_attention_heads: 128 num_experts_per_tok: 8 num_hidden_layers: 61 num_nextn_predict_layers: 1 params_dtype: bfloat16 position_embedding_type: yarn q_lora_rank: 1536 qk_layernorm: true qk_nope_head_dim: 128 qk_rope_head_dim: 64 rms_norm_eps: 1.0e-06 rope_theta: 10000 rotary_dtype: float32 routed_scaling_factor: 2.5 scoring_func: sigmoid seq_length: 4096 softmax_compute_dtype: float32 use_flash_attention: true v_head_dim: 128 vocab_size: 131072 enable_hc_head: false enable_hyper_connections: true use_fused_mhc: True experimental_attention_variant: mla hc_eps: 1.0e-06 hc_mult: 4 hc_sinkhorn_iters: 20 mla_qkv_concat: true moe_router_pre_softmax: true moe_router_score_function: sigmoid moe_router_topk_scaling_factor: 2.5 moe_token_dispatcher_type: alltoall moe_token_drop_policy: probs mtp_loss_factor: 0.3 normalization: RMSNorm num_hash_layers: 3 qk_clip: true recompute: mode: select #select full_recompute_layer: [0-2, 3-5, 7-9, 11-13, 15-17, 19-21, 23-25, 27-28, 31-32, 35-36, 38-40, 42-44, 46-47, 50-51, 54-55] #full_recompute_layer: [0-2, 3-4, 7-8, 11-12, 15-16, 19-20, 23-24, 27-28, 31-32, 35-36, 38-41, 42-44, 46-47, 50-51, 54-55] #full_recompute_layer: [0-2, 3-5, 7-9, 11-13, 15-17, 19-21, 23-25, 27-29, 31-33, 35-36, 38-40, 42-44, 46-48, 50-52, 54-56, 59] #full_recompute_layer: [0-61] #exclude_op: ['AllToAll'] select_module: mlp.experts.activation_func: ["0-61"] mlp.shared_experts.activation_func: ["0-61"] transformer_layer.mlp.experts.activation_func: ["62"] transformer_layer.mlp.shared_experts.activation_func: ["62"] #recompute: # mode: full #full_recompute_layer: [0-3, 4-7, 8-11, 12-15, 16-19, 20-23, 24-27, 28-31, 32-35, 36-39, 40-43, 44-47, 48-51, 52-54, 56-58, 60] #full_recompute_layer: [0-2, 3-5, 7-9, 11-13, 15-17, 19-21, 23-25, 27-28, 31-32, 35-36, 38-40, 42-44, 46-47, 50-51, 54-55] #full_recompute_layer: [0-2, 4-6, 8-10, 12-14, 16-18, 20-22, 24-26, 28-30, 32-34, 35-37, 38-40, 42-44, 46-47, 50-52, 54] #full_recompute_layer: [0-61] #full_recompute_layer: #[0-2,4-6,8-10,12-14,16-18,20-22,24-26,28-30,32-34,36-38,40-42,44-46,48-50,52-54,56-57,60-61] #[0-1,4-5,8-9,12-13,16-17,20-21,24-25,28-29,32-33,36-37,40-41,44-45,48-49,52-53,556-57,60] #[0-1,4-5,8-9,12-13,16-17,20-21,24-25,28] #[0-31] #exclude_op: ['AllToAll'] train_dataset: dataloader: column_names: - input_ids - labels - loss_mask - position_ids type: BlendedMegatronDatasetDataLoader config: create_attention_mask: false create_compressed_eod_mask: false data_path: - '1' - /hpfs/huawei/public/deepseek4_flash/datasets/mmap_deepseekv4_datasets_text_document eod: 1 eod_mask_loss: false eod_pad_length: 128 pad: -1 reset_attention_mask: false reset_position_ids: false seed: 1234 seq_length: 4096 split: 1, 0, 0 datasets_type: GPTDataset sizes: - 512000 - 0 - 0 profiler: enable_profiling: True #True end_step: 5 output_path: /hpfs/huawei/0727lc/prof_lc0728_1513_level1 profile_cpu: False profile_memory: False profiler_level: 1 profiler_rank: [0, 128, 256, 384, 512, 640, 768, 896] #profiler_rank: [768,769,770,771,772,773,774,775,776,777,778,779,780,781,782,783,784,785,786,787,788,789,790,791,792,793,794,795,796,797,798,799,800] #profiler_rank: [768,774,784,785,787,788,792,800,808,816,824,830] start_step: 5 with_stack: False