Mindformer侧验证yaml配置

checkpoint:
  enable_save: false
  save_path: output/checkpoint/tc-act-002
  save_max: 1
  save_interleaved_steps: 100
  no_save_optim: false
  async_save: false
  prefix: custom
  remove_redundancy: false
  load_path: ''
  load_balanced: false
  no_load_optim: true
  load_worker_number: 1
training:
  steps: 100
  local_batch_size: 1
  global_batch_size: 8
  max_norm: 1.0
  seed: 42
  deterministic: true
optimizer:
  type: AdamW
  betas:
  - 0.9
  - 0.95
  eps: 1.0e-08
  weight_decay: 0.01
lr_scheduler:
  type: ConstantWarmUpLR
  learning_rate: 1.0e-05
  warmup_ratio: 0
parallelism:
  data_parallel_shard: 8
  data_parallel_shard_strategy: optim_grads_params
  tensor_parallel: 2
  context_parallel: 1
  context_parallel_method: colossal
  pipeline_parallel: 1
  pipeline_parallel_schedule: 1f1b
  pipeline_parallel_microbatch_size: 1
  pipeline_parallel_interleave_num: 1
  pipeline_parallel_layers_per_stage: null
  sequence_parallel: true
  expert_parallel: 1
recompute:
  mode: full
  full_recompute_layer:
  - 0-3
  select_module: null
train_dataset:
  dataloader:
    type: BlendedMegatronDatasetDataLoader
    datasets_type: GPTDataset
    sizes:
    - 8000
    - 0
    - 0
    config:
      split: 1, 0, 0
      seq_length: 4096
      eod_mask_loss: false
      reset_position_ids: false
      create_attention_mask: false
      reset_attention_mask: false
      create_compressed_eod_mask: false
      eod_pad_length: 128
      eod: 1
      pad: -1
      data_path:
      - '1'
      - /home/data/datasets/deepseek-datasets/mmap_deepseekv3_datasets_text_document
    column_names:
    - input_ids
    - labels
    - loss_mask
    - position_ids
    shuffle: false
    python_multiprocessing: false
  drop_remainder: true
  num_parallel_workers: 8
  prefetch_size: 1
  numa_enable: false
model:
  model_type: deepseek_v3
  architectures: DeepseekV3ForCausalLM
  offset: 0
  vocab_size: 129280
  seq_length: 4096
  hidden_size: 3584
  intermediate_size: 9216
  num_hidden_layers: 4
  max_position_embeddings: 163840
  hidden_act: silu
  num_attention_heads: 32
  rms_norm_eps: 1.0e-06
  add_bias_linear: false
  use_flash_attention: true
  multi_latent_attention: true
  mla_qkv_concat: true
  kv_lora_rank: 512
  q_lora_rank: 768
  qk_rope_head_dim: 64
  qk_nope_head_dim: 128
  v_head_dim: 128
  qk_layernorm: true
  attention_dropout: 0.0
  hidden_dropout: 0.0
  params_dtype: float32
  compute_dtype: bfloat16
  layernorm_compute_dtype: float32
  softmax_compute_dtype: float32
  rotary_dtype: float32
  initializer_range: 0.01
  num_nextn_predict_layers: 0
  mtp_loss_scaling_factor: 0.3
  position_embedding_type: rope
  scaling_factor: 40
  beta_fast: 32
  beta_slow: 1
  mscale: 1
  mscale_all_dim: 1
  rope_theta: 10000
  enable_hyper_connections: true
  num_residual_streams: 4
  mhc_sinkhorn_iterations: 20
  mhc_init_gating_factor: 0.01
  router_dense_type: float32
  gated_linear_unit: true
  moe_intermediate_size: 1024
  routed_scaling_factor: 2.5
  first_k_dense_replace: 2
  n_routed_experts: 64
  num_experts_per_tok: 4
  n_shared_experts: 1
  moe_shared_expert_intermediate_size: 1024
  moe_token_dispatcher_type: alltoall
  moe_grouped_gemm: true
  moe_router_load_balancing_type: seq_aux_loss
  moe_aux_loss_coeff: 0.001
  scoring_func: sigmoid
  norm_topk_prob: true
  moe_token_drop_policy: probs
  moe_router_enable_expert_bias: true
  moe_router_bias_update_rate: 0.001
  use_pad_tokens: true
  topk_group: 2
  n_group: 4

数据集
wget https://atp-modelzoo-wlcb-pai.oss-cn-wulanchabu.aliyuncs.com/release/models/pai-megatron-patch/deepseek-datasets/mmap_deepseekv3_datasets_text_document.bin
wget https://atp-modelzoo-wlcb-pai.oss-cn-wulanchabu.aliyuncs.com/release/models/pai-megatron-patch/deepseek-datasets/mmap_deepseekv3_datasets_text_document.idx