checkpoint:
enable_save: False
save_path: "output/checkpoint"
save_max: 1
save_interleaved_steps: 10000
no_save_optim: False
async_save: False
prefix: "custom"
remove_redundancy: False

load_path: "" #"/hpfs/huawei/zqh/hp_0519/py_unified_40L1mtp200Step/iteration_00000200" #"/hpfs/huawei/zqh/py_unified_40L1mtp/iteration_00000010/" #"/hpfs/huawei/zqh/hp_0519/py_1200100_1_unified/iteration_01200100/" #"/hpfs/huawei/zqh/hp_0519/py_0601_1500_step/iteration_00001500/" #"/hpfs/huawei/zqh/hp_0519/ckpt2_0_convert_from_graph_new/" #"/hpfs/huawei/zqh/TeleProjects/mindformers/output_0527/iteration_01201200/" #"/hpfs/huawei/wh/hp_0520/unified_safe/1-1201200_1_ckpt_convert/unified_safe/"
load_balanced: False
no_load_optim: True
load_worker_number: 1

training: # name refers to torchtitan
steps: 4000 #1000 # 1,230,028,800 # total number of training steps
local_batch_size: 2 # per-rank batch size (corresponds to runner_config: batch_size)
global_batch_size: 512 # total number of samples processed per step across all ranks

这里对过了

max_norm: 1.0 # enable gradient clipping if value > 0
seed: 42 # random seed for training
deterministic: True # enable deterministic training behavior

优化器配置

#optimizer:

type: AdamW

betas:

- 0.9

- 0.95

eps: 1.e-8

weight_decay: 0.01

Muon优化器

optimizer:
type: Muon
adamw_betas: [0.9, 0.95]
adamw_eps: 1.e-8
weight_decay: 0.1
matched_adamw_rms: 0.2
momentum: 0.95
nesterov: True
ns_steps: 5
qk_clip_threshold: 100
qk_clip_enabled: False
#comm_strategy: allgather_deredundency
#use_fused_adamw: True

lr_scheduler:
type: ConstantWarmUpLR #CosineWithWarmUpLR #ConstantWarmUpLR
learning_rate: 3.0e-5
#lr_end: 1.0e-4
warmup_lr_init: 0
warmup_steps: 0 #500 #124000 #500 #2000
total_steps: 165000

lr_scheduler:

type: CosineWithWarmUpLR

learning_rate: 1.0e-6

#warmup_ratio: 0

#lr_end: 1.0e-5

warmup_lr_init: 0

warmup_steps: 0 #2000

total_steps: 223517

parallelism: # name refers to torchtitan

Data Parallelism & FSDP

data_parallel_shard: -1
data_parallel_shard_strategy: "optim_grads_params"

tensor_parallel: 1 # tensor parallelism degree (TP size)
context_parallel: 1 # context parallelism degree (CP size)
context_parallel_method: colossal # implementation method for context parallelism
pipeline_parallel: 1 # pipeline parallelism degree (number of pipeline stages)
pipeline_parallel_schedule: "1f1b" # pipeline execution schedule (e.g., 1F1B)
pipeline_parallel_microbatch_size: 1 # number of micro-batches per pipeline step
pipeline_parallel_interleave_num: 1 # number of interleaved model chunks per pipeline stage

number of layers assigned to each pipeline stage

supports uneven or interleaved layer placement

pipeline_parallel_layers_per_stage: null
sequence_parallel: true # enable sequence parallelism
expert_parallel: 1

#重计算配置
recompute:
mode: full
full_recompute_layer: [0-39]

#swap:
#  enable: True
#  default_prefetch: 1
#  layer_swap:
#    - layers: [0-28] #

train_dataset:
dataloader:
type: OrderedIndexDataLoader
data_path: "/hpfs/nlp/yangzhihao/data/pretrain_data/ML7.2.1_6.2T_6.4T_256K/" #"/hpfs/nlp/yangzhihao/data/pretrain_data/V9.0.0/" #"/hpfs/nlp/yangzhihao/data/pretrain_data/M6.0.0_5T_5.5T"
shuffle: False
sequence_length: 4096
reset_position_ids: True
eod_mask_loss: True
create_attention_mask: True
reset_attention_mask: True
create_compressed_eod_mask: True
compressed_eod_mask_length: 128
eod_token: 2
pad_token: 3
create_seq_len_vector: False
column_names: ["input_ids", "labels", "loss_mask", "position_ids", "actual_seq_len"]
python_multiprocessing: False
use_cache: True
cache_dir: "/hpfs/huawei/zqh/cached_datasets"
drop_remainder: True
num_parallel_workers: 8
prefetch_size: 1
numa_enable: False

#train_dataset: &train_dataset

data_loader:

balance_config:

data_parallel: 8

model_parallel: 64

global_batch_size: 256

micro_batch_num: 32

type: OrderedIndexDataLoader

data_path: "/hpfs/nlp/yangzhihao/data/pretrain_data/ML7.2.1_6.2T_6.4T_256K/" #"/hpfs/huawei/lilei/MM6.1.0_5.5T_6T_100B/" # "/hpfs/huawei/lilei/mhc_test/test_dataset"

shuffle: False

sequence_length: 4096 #262144 #4096

reset_position_ids: False

eod_mask_loss: True

create_attention_mask: True

reset_attention_mask: True

create_compressed_eod_mask: True

compressed_eod_mask_length: 384 #128

eod_token: 2

pad_token: 3

create_seq_len_vector: False

input_columns: ["input_ids", "labels", "loss_mask", "position_ids", "actual_seq_len"]

construct_args_key: ["input_ids", "labels", "loss_mask", "position_ids", "actual_seq_len"]

num_parallel_workers: 16 #8

python_multiprocessing: False

drop_remainder: True

numa_enable: False

prefetch_size: 1

model: # 已对完
#chunk_loss_num: 4
input_layout: "TND"
model_type: deepseek_v3
architectures: DeepseekV3ForCausalLM
offset: 0
vocab_size: 131072
seq_length: 4096 #*seq_len
hidden_size: 3584
intermediate_size: 9216
num_hidden_layers: 40
max_position_embeddings: 4096
hidden_act: "fusedswiglu" # 'fusedswiglu' # 'silu' # 存疑
num_attention_heads: 32
rms_norm_eps: 1.e-6
add_bias_linear: False
use_flash_attention: True
multi_latent_attention: True
mla_qkv_concat: True
kv_lora_rank: 512
q_lora_rank: 768
qk_rope_head_dim: 64
qk_nope_head_dim: 128
v_head_dim: 128
qk_layernorm: True

attention_dropout: 0.0
hidden_dropout: 0.0

params_dtype: "float32" #"bfloat16" #"float32"
compute_dtype: "bfloat16"
layernorm_compute_dtype: "float32"
softmax_compute_dtype: "float32"
rotary_dtype: "float32"
initializer_range: 0.02

num_nextn_predict_layers: 1 # 1
mtp_loss_scaling_factor: 0.1
mtp_enable_hyper_connections: False

position_embedding_type: "rope"
rotary_scaling_factor: 1.0 #scaling_factor: 1.0 # 原40 存疑
beta_fast: 32
beta_slow: 1
mscale: 1
mscale_all_dim: 1
rope_theta: 10000.0

mhc config

enable_hyper_connections: True
num_residual_streams: 4
mhc_sinkhorn_iterations: 20
mhc_init_gating_factor: 0.01
use_fused_mhc: True
hc_sinkhorn_clamp: [-30, 30]

moe config

router_dense_type: "float32"
gated_linear_unit: True
moe_intermediate_size: 1024
routed_scaling_factor: 2.0
first_k_dense_replace: 2
n_routed_experts: 64
num_experts_per_tok: 4
n_shared_experts: 1
moe_shared_expert_intermediate_size: 1024
norm_topk_prob: True
scoring_func: "sigmoid"
moe_grouped_gemm: True
topk_group: 0
n_group: 0
use_pad_tokens: True
moe_token_dispatcher_type: "alltoall"
router_jitter_noise: 0.00 # 静态图/动态图共用配置；>0 时启用门控分数乘性抖动噪声 U(1-eps, 1+eps)

moe_router_load_balancing_type: "seq_aux_loss"
moe_aux_loss_coeff: 0 #6.0e-4

moe_router_enable_expert_bias: False #True
moe_router_bias_update_rate: 0.0 #0.0001
moe_token_drop_policy: probs
moe_router_force_expert_balance: False

dsa config

experimental_attention_variant: dsa
dsa_indexer_n_heads: 64
dsa_indexer_head_dim: 128
dsa_indexer_topk: 2048
dsa_indexer_loss_coeff: 1.0
dsa_indexer_use_sparse_loss: True