[core-llm][dskv3]master同步2.0.0的deepseek3 A3脚本-MindSpeed-LLM-AtomGit

Mexamples/mcore/deepseek3/pretrain_deepseek3_671b_4k_A3_ptd.sh+31-18

	@@ -1,6 +1,12 @@
1	-#!/bin/bash	1	+#!/bin/bash
		2	+
		3	+# 需要切换MindSpeed版本
		4	+# git checkout f2acbe71a47b9307c2425bc51f9565802bd901cf # checkout commit from MindSpeed core_r0.8.0 in 2025.03.30
		5	+
2	export CUDA_DEVICE_MAX_CONNECTIONS=1	6	export CUDA_DEVICE_MAX_CONNECTIONS=1
3	-export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True	7	+export TASK_QUEUE_ENABLE=2
		8	+export HCCL_OP_BASE_FFTS_MODE=TRUE
		9	+export PYTORCH_NPU_ALLOC_CONF="expandable_segments:True"
4	export HCCL_CONNECT_TIMEOUT=3600	10	export HCCL_CONNECT_TIMEOUT=3600
5		11
6	NPUS_PER_NODE=16	12	NPUS_PER_NODE=16
	@@ -15,10 +21,8 @@ DATA_PATH="your data path"
15	TOKENIZER_PATH="your tokenizer path"	21	TOKENIZER_PATH="your tokenizer path"
16	CKPT_LOAD_DIR="your model ckpt path"	22	CKPT_LOAD_DIR="your model ckpt path"
17		23
18	-
19	TP=2	24	TP=2
20	PP=8	25	PP=8
21	-VPP=2
22	EP=32	26	EP=32
23	CP=1	27	CP=1
24	CP_TYPE='ulysses_cp_algo'	28	CP_TYPE='ulysses_cp_algo'
	@@ -52,7 +56,6 @@ MOE_ARGS="
52	--moe-permutation-async-comm \	56	--moe-permutation-async-comm \
53	--use-fused-moe-token-permute-and-unpermute \	57	--use-fused-moe-token-permute-and-unpermute \
54	--moe-token-dispatcher-type alltoall \	58	--moe-token-dispatcher-type alltoall \
55	- --moe-alltoall-overlap-comm \
56	--first-k-dense-replace 3 \	59	--first-k-dense-replace 3 \
57	--moe-layer-freq 1 \	60	--moe-layer-freq 1 \
58	--n-shared-experts 1 \	61	--n-shared-experts 1 \
	@@ -63,18 +66,29 @@ MOE_ARGS="
63	--n-group 8 \	66	--n-group 8 \
64	--topk-group 4 \	67	--topk-group 4 \
65	--routed-scaling-factor 2.5 \	68	--routed-scaling-factor 2.5 \
		69	+ --moe-aux-loss-coeff 0.0001 \
66	--seq-aux \	70	--seq-aux \
67	--norm-topk-prob \	71	--norm-topk-prob \
68	--moe-router-score-function sigmoid \	72	--moe-router-score-function sigmoid \
69	--moe-router-enable-expert-bias \	73	--moe-router-enable-expert-bias \
70	- --moe-tp-extend-ep	74	+ --moe-tp-extend-ep \
71	"	75	"
72		76
73	-
74	MTP_ARGS="	77	MTP_ARGS="
75	--num-nextn-predict-layers 1 \	78	--num-nextn-predict-layers 1 \
76	--share-mtp-embedding-and-output-weight \	79	--share-mtp-embedding-and-output-weight \
77	- --recompute-mtp-norm \	80	+ --mtp-mem-efficient-logits \
		81	+"
		82	+
		83	+DUALPIPE_ARGS="
		84	+ --moe-fb-overlap \
		85	+ --schedules-method dualpipev \
		86	+"
		87	+
		88	+RECOMPUTE_ARGS="
		89	+ --recompute-granularity full \
		90	+ --recompute-method block \
		91	+ --recompute-num-layers 4 \
78	"	92	"
79		93
80	ROPE_ARGS="	94	ROPE_ARGS="
	@@ -90,19 +104,17 @@ ROPE_ARGS="
90	GPT_ARGS="	104	GPT_ARGS="
91	--spec mindspeed_llm.tasks.models.spec.deepseek_spec layer_spec \	105	--spec mindspeed_llm.tasks.models.spec.deepseek_spec layer_spec \
92	--reset-position-ids \	106	--reset-position-ids \
		107	+ --router-gating-in-fp32 \
93	--gemm-gradient-accumulation-fusion \	108	--gemm-gradient-accumulation-fusion \
94	- --noop-layers 47,62,63 \	109	+ --noop-layers 61,62,63 \
95	- --recompute-granularity full \	110	+ --manual-gc \
96	- --recompute-method uniform \	111	+ --manual-gc-interval 50 \
97	- --recompute-num-layers 1 \
98	--no-shared-storage \	112	--no-shared-storage \
99	--use-distributed-optimizer \	113	--use-distributed-optimizer \
100	- --reuse-fp32-param \
101	--use-flash-attn \	114	--use-flash-attn \
102	--use-mcore-models \	115	--use-mcore-models \
103	--tensor-model-parallel-size ${TP} \	116	--tensor-model-parallel-size ${TP} \
104	--pipeline-model-parallel-size ${PP} \	117	--pipeline-model-parallel-size ${PP} \
105	- --num-layers-per-virtual-pipeline-stage ${VPP} \
106	--expert-model-parallel-size ${EP} \	118	--expert-model-parallel-size ${EP} \
107	--sequence-parallel \	119	--sequence-parallel \
108	--context-parallel-size ${CP} \	120	--context-parallel-size ${CP} \
	@@ -149,7 +161,7 @@ GPT_ARGS="
149	--no-load-optim \	161	--no-load-optim \
150	--no-load-rng \	162	--no-load-rng \
151	--bf16 \	163	--bf16 \
152	- --distributed-timeout-minutes 120	164	+ --distributed-timeout-minutes 120 \
153	"	165	"
154		166
155	DATA_ARGS="	167	DATA_ARGS="
	@@ -171,10 +183,11 @@ python -m torch.distributed.launch $DISTRIBUTED_ARGS pretrain_gpt.py \
171	$DATA_ARGS \	183	$DATA_ARGS \
172	$OUTPUT_ARGS \	184	$OUTPUT_ARGS \
173	$MLA_ARGS \	185	$MLA_ARGS \
		186	+ $DUALPIPE_ARGS \
		187	+ $RECOMPUTE_ARGS \
174	$ROPE_ARGS \	188	$ROPE_ARGS \
175	$MOE_ARGS \	189	$MOE_ARGS \
176	$MTP_ARGS \	190	$MTP_ARGS \
177	- --distributed-backend nccl \
178	--save $CKPT_SAVE_DIR \	191	--save $CKPT_SAVE_DIR \
179	- --load $CKPT_LOAD_DIR \	192	+ --load $CKPT_LOAD_DIR \
180	- \| tee logs/pretrain_deepseek3_671b_4k_A3_ptd.log	193	+ --distributed-backend nccl \| tee logs/pretrain_deepseek3_671b_4k_A3_ptd.log

Rexamples/mcore/deepseek3/pretrain_deepseek3_60b_4k_A3_ptd.sh→tests/poc/deepseek3/pretrain_deepseek3_60b_4k_128die_A3_ptd.sh+1-0

	@@ -66,6 +66,7 @@ MOE_ARGS="
66	--moe-router-score-function sigmoid \	66	--moe-router-score-function sigmoid \
67	--moe-router-enable-expert-bias \	67	--moe-router-enable-expert-bias \
68	--moe-tp-extend-ep \	68	--moe-tp-extend-ep \
		69	+ --fix-router \
69	"	70	"
70		71
71	MTP_ARGS="	72	MTP_ARGS="

Atests/poc/deepseek3/pretrain_deepseek3_671b_4k_512die_A3_ptd.sh+195-0

@@ -0,0 +1,195 @@
	1	+#!/bin/bash
	2	+
	3	+# 需要切换MindSpeed版本
	4	+# git checkout f2acbe71a47b9307c2425bc51f9565802bd901cf # checkout commit from MindSpeed core_r0.8.0 in 2025.03.30
	5	+
	6	+export CUDA_DEVICE_MAX_CONNECTIONS=1
	7	+export TASK_QUEUE_ENABLE=2
	8	+export HCCL_OP_BASE_FFTS_MODE=TRUE
	9	+export PYTORCH_NPU_ALLOC_CONF="expandable_segments:True"
	10	+export HCCL_CONNECT_TIMEOUT=3600
	11	+
	12	+NPUS_PER_NODE=16
	13	+MASTER_ADDR=localhost #主节点IP
	14	+MASTER_PORT=6000
	15	+NNODES=32
	16	+NODE_RANK=0
	17	+WORLD_SIZE=$(($NPUS_PER_NODE*$NNODES))
	18	+
	19	+CKPT_SAVE_DIR="your model save ckpt path"
	20	+DATA_PATH="your data path"
	21	+TOKENIZER_PATH="your tokenizer path"
	22	+CKPT_LOAD_DIR="your model ckpt path"
	23	+
	24	+TP=2
	25	+PP=8
	26	+EP=32
	27	+CP=1
	28	+CP_TYPE='ulysses_cp_algo'
	29	+NUM_LAYERS=64
	30	+SEQ_LEN=4096
	31	+MBS=1
	32	+GBS=3840
	33	+
	34	+DISTRIBUTED_ARGS="
	35	+ --nproc_per_node $NPUS_PER_NODE \
	36	+ --nnodes $NNODES \
	37	+ --node_rank $NODE_RANK \
	38	+ --master_addr $MASTER_ADDR \
	39	+ --master_port $MASTER_PORT
	40	+"
	41	+
	42	+MLA_ARGS="
	43	+ --multi-head-latent-attention \
	44	+ --qk-rope-head-dim 64 \
	45	+ --qk-nope-head-dim 128 \
	46	+ --q-lora-rank 1536 \
	47	+ --kv-lora-rank 512 \
	48	+ --v-head-dim 128 \
	49	+ --qk-layernorm \
	50	+ --mla-mm-split \
	51	+ --mla-fa-without-pad \
	52	+"
	53	+
	54	+MOE_ARGS="
	55	+ --moe-grouped-gemm \
	56	+ --moe-permutation-async-comm \
	57	+ --use-fused-moe-token-permute-and-unpermute \
	58	+ --moe-token-dispatcher-type alltoall \
	59	+ --first-k-dense-replace 3 \
	60	+ --moe-layer-freq 1 \
	61	+ --n-shared-experts 1 \
	62	+ --num-experts 256 \
	63	+ --moe-router-topk 8 \
	64	+ --moe-intermediate-size 2048 \
	65	+ --moe-router-load-balancing-type noaux_tc \
	66	+ --n-group 8 \
	67	+ --topk-group 4 \
	68	+ --routed-scaling-factor 2.5 \
	69	+ --moe-aux-loss-coeff 0.0001 \
	70	+ --seq-aux \
	71	+ --norm-topk-prob \
	72	+ --moe-router-score-function sigmoid \
	73	+ --moe-router-enable-expert-bias \
	74	+ --moe-tp-extend-ep \
	75	+"
	76	+
	77	+MTP_ARGS="
	78	+ --num-nextn-predict-layers 1 \
	79	+ --share-mtp-embedding-and-output-weight \
	80	+ --mtp-mem-efficient-logits \
	81	+"
	82	+
	83	+DUALPIPE_ARGS="
	84	+ --moe-fb-overlap \
	85	+ --schedules-method dualpipev \
	86	+"
	87	+
	88	+MEM_ARGS="
	89	+ --mla-zero-memory \
	90	+ --moe-zero-memory level0 \
	91	+ --recompute-activation-function \
	92	+ --swap-optimizer \
	93	+"
	94	+
	95	+ROPE_ARGS="
	96	+ --rope-scaling-beta-fast 32 \
	97	+ --rope-scaling-beta-slow 1 \
	98	+ --rope-scaling-factor 40 \
	99	+ --rope-scaling-mscale 1.0 \
	100	+ --rope-scaling-mscale-all-dim 1.0 \
	101	+ --rope-scaling-original-max-position-embeddings 4096 \
	102	+ --rope-scaling-type yarn
	103	+"
	104	+
	105	+GPT_ARGS="
	106	+ --spec mindspeed_llm.tasks.models.spec.deepseek_spec layer_spec \
	107	+ --reset-position-ids \
	108	+ --router-gating-in-fp32 \
	109	+ --gemm-gradient-accumulation-fusion \
	110	+ --noop-layers 61,62,63 \
	111	+ --manual-gc \
	112	+ --manual-gc-interval 50 \
	113	+ --no-shared-storage \
	114	+ --use-distributed-optimizer \
	115	+ --use-flash-attn \
	116	+ --use-mcore-models \
	117	+ --tensor-model-parallel-size ${TP} \
	118	+ --pipeline-model-parallel-size ${PP} \
	119	+ --expert-model-parallel-size ${EP} \
	120	+ --sequence-parallel \
	121	+ --context-parallel-size ${CP} \
	122	+ --context-parallel-algo ${CP_TYPE} \
	123	+ --num-layers ${NUM_LAYERS} \
	124	+ --hidden-size 7168 \
	125	+ --ffn-hidden-size 18432 \
	126	+ --num-attention-heads 128 \
	127	+ --tokenizer-type PretrainedFromHF \
	128	+ --tokenizer-name-or-path ${TOKENIZER_PATH} \
	129	+ --seq-length ${SEQ_LEN} \
	130	+ --max-position-embeddings 163840 \
	131	+ --micro-batch-size ${MBS} \
	132	+ --global-batch-size ${GBS} \
	133	+ --make-vocab-size-divisible-by 1 \
	134	+ --lr 1.0e-5 \
	135	+ --train-iters 30 \
	136	+ --lr-decay-style cosine \
	137	+ --untie-embeddings-and-output-weights \
	138	+ --disable-bias-linear \
	139	+ --attention-dropout 0.0 \
	140	+ --init-method-std 0.02 \
	141	+ --hidden-dropout 0.0 \
	142	+ --position-embedding-type rope \
	143	+ --normalization RMSNorm \
	144	+ --use-fused-rotary-pos-emb \
	145	+ --use-rotary-position-embeddings \
	146	+ --use-fused-swiglu \
	147	+ --use-fused-rmsnorm \
	148	+ --swiglu \
	149	+ --no-masked-softmax-fusion \
	150	+ --attention-softmax-in-fp32 \
	151	+ --min-lr 1.0e-7 \
	152	+ --weight-decay 1e-2 \
	153	+ --lr-warmup-iters 0 \
	154	+ --clip-grad 1.0 \
	155	+ --adam-beta1 0.9 \
	156	+ --adam-beta2 0.999 \
	157	+ --initial-loss-scale 65536 \
	158	+ --vocab-size 129280 \
	159	+ --padded-vocab-size 129280 \
	160	+ --rotary-base 10000 \
	161	+ --norm-epsilon 1e-6 \
	162	+ --no-load-optim \
	163	+ --no-load-rng \
	164	+ --bf16 \
	165	+ --fix-router \
	166	+ --distributed-timeout-minutes 120 \
	167	+"
	168	+
	169	+DATA_ARGS="
	170	+ --data-path $DATA_PATH \
	171	+ --split 100,0,0
	172	+"
	173	+
	174	+OUTPUT_ARGS="
	175	+ --log-interval 1 \
	176	+ --save-interval 2000 \
	177	+ --eval-interval 2000 \
	178	+ --eval-iters 0 \
	179	+ --no-save-optim \
	180	+ --no-save-rng
	181	+"
	182	+
	183	+python -m torch.distributed.launch $DISTRIBUTED_ARGS pretrain_gpt.py \
	184	+ $GPT_ARGS \
	185	+ $DATA_ARGS \
	186	+ $OUTPUT_ARGS \
	187	+ $MLA_ARGS \
	188	+ $DUALPIPE_ARGS \
	189	+ $MEM_ARGS \
	190	+ $ROPE_ARGS \
	191	+ $MOE_ARGS \
	192	+ $MTP_ARGS \
	193	+ --save $CKPT_SAVE_DIR \
	194	+ --load $CKPT_LOAD_DIR \
	195	+ --distributed-backend nccl \| tee logs/pretrain_deepseek3_671b_4k_512die_A3_ptd.log