poc脚本修改-MindSpeed-LLM-AtomGit

Mtests/poc/deepseek_500b/pretrain_deepseek_500b_128k_256die_A2.sh+8-6

	@@ -34,7 +34,7 @@ TP=1
34	PP=8	34	PP=8
35	CP=16	35	CP=16
36	EP=32	36	EP=32
37	-NUM_LAYERS=32	37	+NUM_LAYERS=64
38	CP_TYPE='megatron_cp_algo'	38	CP_TYPE='megatron_cp_algo'
39	SEQ_LEN=131072	39	SEQ_LEN=131072
40	MBS=1	40	MBS=1
	@@ -54,7 +54,8 @@ MOE_ARGS="
54	--moe-token-dispatcher-type alltoall \	54	--moe-token-dispatcher-type alltoall \
55	--moe-alltoall-overlap-comm \	55	--moe-alltoall-overlap-comm \
56	--moe-router-topk 5 \	56	--moe-router-topk 5 \
57	- --moe-permutation-async-comm	57	+ --moe-permutation-async-comm \
		58	+ --use-fused-moe-token-permute-and-unpermute \
58	"	59	"
59		60
60	GPT_ARGS="	61	GPT_ARGS="
	@@ -93,7 +94,7 @@ GPT_ARGS="
93	--position-embedding-type rope \	94	--position-embedding-type rope \
94	--normalization RMSNorm \	95	--normalization RMSNorm \
95	--swiglu \	96	--swiglu \
96	- --use-fused-rotary-pos-emb \	97	+ --use-fused-rotary-pos-emb-new \
97	--use-fused-swiglu \	98	--use-fused-swiglu \
98	--use-fused-rmsnorm \	99	--use-fused-rmsnorm \
99	--use-flash-attn \	100	--use-flash-attn \
	@@ -112,11 +113,12 @@ GPT_ARGS="
112	--expert-model-parallel-size ${EP} \	113	--expert-model-parallel-size ${EP} \
113	--lr-warmup-fraction 0.01 \	114	--lr-warmup-fraction 0.01 \
114	--swap-attention \	115	--swap-attention \
115	- --recompute-num-layers 4 \	116	+ --recompute-method block \
		117	+ --recompute-num-layers 8 \
116	--enable-recompute-layers-per-pp-rank \	118	--enable-recompute-layers-per-pp-rank \
117	--use-fused-ring-attention-update \	119	--use-fused-ring-attention-update \
118	- --use-fused-moe-token-permute-and-unpermute \	120	+ --recompute-in-advance \
119	- --recompute-in-advance \	121	+ --fix-router \
120	--bf16	122	--bf16
121	"	123	"
122		124

Mtests/poc/deepseek_500b/pretrain_deepseek_500b_128k_256die_A3.sh+9-10

	@@ -1,7 +1,4 @@
1	#!/bin/bash	1	#!/bin/bash
2	-export HCCL_OP_EXPANSION_MODE="AI_CPU"
3	-export PYTORCH_NPU_ALLOC_CONF="expandable_segments:True"
4	-
5	export CUDA_DEVICE_MAX_CONNECTIONS=1	2	export CUDA_DEVICE_MAX_CONNECTIONS=1
6	export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True	3	export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
7		4
	@@ -37,7 +34,7 @@ TP=1
37	PP=8	34	PP=8
38	CP=16	35	CP=16
39	EP=32	36	EP=32
40	-NUM_LAYERS=32	37	+NUM_LAYERS=64
41	CP_TYPE='megatron_cp_algo'	38	CP_TYPE='megatron_cp_algo'
42	SEQ_LEN=131072	39	SEQ_LEN=131072
43	MBS=1	40	MBS=1
	@@ -57,7 +54,8 @@ MOE_ARGS="
57	--moe-token-dispatcher-type alltoall \	54	--moe-token-dispatcher-type alltoall \
58	--moe-alltoall-overlap-comm \	55	--moe-alltoall-overlap-comm \
59	--moe-router-topk 5 \	56	--moe-router-topk 5 \
60	- --moe-permutation-async-comm	57	+ --moe-permutation-async-comm \
		58	+ --use-fused-moe-token-permute-and-unpermute \
61	"	59	"
62		60
63	GPT_ARGS="	61	GPT_ARGS="
	@@ -96,7 +94,7 @@ GPT_ARGS="
96	--position-embedding-type rope \	94	--position-embedding-type rope \
97	--normalization RMSNorm \	95	--normalization RMSNorm \
98	--swiglu \	96	--swiglu \
99	- --use-fused-rotary-pos-emb \	97	+ --use-fused-rotary-pos-emb-new \
100	--use-fused-swiglu \	98	--use-fused-swiglu \
101	--use-fused-rmsnorm \	99	--use-fused-rmsnorm \
102	--use-flash-attn \	100	--use-flash-attn \
	@@ -115,11 +113,12 @@ GPT_ARGS="
115	--expert-model-parallel-size ${EP} \	113	--expert-model-parallel-size ${EP} \
116	--lr-warmup-fraction 0.01 \	114	--lr-warmup-fraction 0.01 \
117	--swap-attention \	115	--swap-attention \
118	- --recompute-num-layers 4 \	116	+ --recompute-method block \
		117	+ --recompute-num-layers 8 \
		118	+ --recompute-in-advance \
119	--enable-recompute-layers-per-pp-rank \	119	--enable-recompute-layers-per-pp-rank \
120	--use-fused-ring-attention-update \	120	--use-fused-ring-attention-update \
121	- --use-fused-moe-token-permute-and-unpermute \	121	+ --fix-router \
122	- --recompute-in-advance \
123	--bf16	122	--bf16
124	"	123	"
125		124
	@@ -151,4 +150,4 @@ python -m torch.distributed.launch $DISTRIBUTED_ARGS pretrain_gpt.py \
151	${DATA_ARGS} \	150	${DATA_ARGS} \
152	${OUTPUT_ARGS} \	151	${OUTPUT_ARGS} \
153	--distributed-backend nccl \	152	--distributed-backend nccl \
154	- \| tee logs/pretrain_deepseek_500b_128k_256die_mcore_A3.log	153	+ \| tee logs/pretrain_deepseek_500b_128k_256die_mcore_A3.log

Mtests/poc/deepseek_500b/pretrain_deepseek_500b_32k_256die_A2.sh+6-4

	@@ -14,7 +14,6 @@ for i in "${!IPs[@]}";
14	do	14	do
15	if [ "$LOCAL_HOST" == "${IPs[$i]}" ];	15	if [ "$LOCAL_HOST" == "${IPs[$i]}" ];
16	then	16	then
17	-# echo "${IPs[$i]}"
18	echo "Node Rank : ${i}"	17	echo "Node Rank : ${i}"
19	NODE_RANK=$i	18	NODE_RANK=$i
20	break	19	break
	@@ -55,7 +54,8 @@ MOE_ARGS="
55	--moe-token-dispatcher-type alltoall \	54	--moe-token-dispatcher-type alltoall \
56	--moe-alltoall-overlap-comm \	55	--moe-alltoall-overlap-comm \
57	--moe-router-topk 5 \	56	--moe-router-topk 5 \
58	- --moe-permutation-async-comm	57	+ --moe-permutation-async-comm \
		58	+ --use-fused-moe-token-permute-and-unpermute \
59	"	59	"
60		60
61	GPT_ARGS="	61	GPT_ARGS="
	@@ -98,6 +98,8 @@ GPT_ARGS="
98	--use-fused-swiglu \	98	--use-fused-swiglu \
99	--use-fused-rmsnorm \	99	--use-fused-rmsnorm \
100	--use-flash-attn \	100	--use-flash-attn \
		101	+ --use-fused-ring-attention-update \
		102	+ --use-fused-rotary-pos-emb-new \
101	--no-masked-softmax-fusion \	103	--no-masked-softmax-fusion \
102	--attention-softmax-in-fp32 \	104	--attention-softmax-in-fp32 \
103	--min-lr 1.0e-7 \	105	--min-lr 1.0e-7 \
	@@ -115,9 +117,9 @@ GPT_ARGS="
115	--swap-attention \	117	--swap-attention \
116	--recompute-num-layers 8 \	118	--recompute-num-layers 8 \
117	--enable-recompute-layers-per-pp-rank \	119	--enable-recompute-layers-per-pp-rank \
118	- --use-fused-ring-attention-update \
119	- --use-fused-moe-token-permute-and-unpermute \
120	--recompute-in-advance \	120	--recompute-in-advance \
		121	+ --fix-router \
		122	+ --distributed-timeout-minutes 120 \
121	--bf16	123	--bf16
122	"	124	"
123		125

Mtests/poc/deepseek_500b/pretrain_deepseek_500b_32k_256die_A3.sh+8-8

	@@ -1,7 +1,4 @@
1	#!/bin/bash	1	#!/bin/bash
2	-export HCCL_OP_EXPANSION_MODE="AI_CPU"
3	-export PYTORCH_NPU_ALLOC_CONF="expandable_segments:True"
4	-
5	export CUDA_DEVICE_MAX_CONNECTIONS=1	2	export CUDA_DEVICE_MAX_CONNECTIONS=1
6	export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True	3	export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
7		4
	@@ -57,7 +54,8 @@ MOE_ARGS="
57	--moe-token-dispatcher-type alltoall \	54	--moe-token-dispatcher-type alltoall \
58	--moe-alltoall-overlap-comm \	55	--moe-alltoall-overlap-comm \
59	--moe-router-topk 5 \	56	--moe-router-topk 5 \
60	- --moe-permutation-async-comm	57	+ --moe-permutation-async-comm \
		58	+ --use-fused-moe-token-permute-and-unpermute \
61	"	59	"
62		60
63	GPT_ARGS="	61	GPT_ARGS="
	@@ -96,9 +94,10 @@ GPT_ARGS="
96	--position-embedding-type rope \	94	--position-embedding-type rope \
97	--normalization RMSNorm \	95	--normalization RMSNorm \
98	--swiglu \	96	--swiglu \
99	- --use-fused-rotary-pos-emb \
100	--use-fused-swiglu \	97	--use-fused-swiglu \
101	--use-fused-rmsnorm \	98	--use-fused-rmsnorm \
		99	+ --use-fused-ring-attention-update \
		100	+ --use-fused-rotary-pos-emb-new \
102	--use-flash-attn \	101	--use-flash-attn \
103	--no-masked-softmax-fusion \	102	--no-masked-softmax-fusion \
104	--attention-softmax-in-fp32 \	103	--attention-softmax-in-fp32 \
	@@ -115,11 +114,12 @@ GPT_ARGS="
115	--expert-model-parallel-size ${EP} \	114	--expert-model-parallel-size ${EP} \
116	--lr-warmup-fraction 0.01 \	115	--lr-warmup-fraction 0.01 \
117	--swap-attention \	116	--swap-attention \
		117	+ --recompute-method block \
118	--recompute-num-layers 8 \	118	--recompute-num-layers 8 \
119	--enable-recompute-layers-per-pp-rank \	119	--enable-recompute-layers-per-pp-rank \
120	- --use-fused-ring-attention-update \	120	+ --recompute-in-advance \
121	- --use-fused-moe-token-permute-and-unpermute \	121	+ --fix-router \
122	- --recompute-in-advance \	122	+ --distributed-timeout-minutes 120 \
123	--bf16	123	--bf16
124	"	124	"
125		125

Mtests/poc/deepseek_500b/pretrain_deepseek_500b_8k_256die_A2.sh+6-4

	@@ -54,7 +54,8 @@ MOE_ARGS="
54	--moe-token-dispatcher-type alltoall \	54	--moe-token-dispatcher-type alltoall \
55	--moe-alltoall-overlap-comm \	55	--moe-alltoall-overlap-comm \
56	--moe-router-topk 5 \	56	--moe-router-topk 5 \
57	- --moe-permutation-async-comm	57	+ --moe-permutation-async-comm \
		58	+ --use-fused-moe-token-permute-and-unpermute \
58	"	59	"
59		60
60	GPT_ARGS="	61	GPT_ARGS="
	@@ -93,10 +94,11 @@ GPT_ARGS="
93	--position-embedding-type rope \	94	--position-embedding-type rope \
94	--normalization RMSNorm \	95	--normalization RMSNorm \
95	--swiglu \	96	--swiglu \
96	- --use-fused-rotary-pos-emb \
97	--use-fused-swiglu \	97	--use-fused-swiglu \
98	--use-fused-rmsnorm \	98	--use-fused-rmsnorm \
99	--use-flash-attn \	99	--use-flash-attn \
		100	+ --use-fused-rotary-pos-emb-new \
		101	+ --use-fused-ring-attention-update \
100	--no-masked-softmax-fusion \	102	--no-masked-softmax-fusion \
101	--attention-softmax-in-fp32 \	103	--attention-softmax-in-fp32 \
102	--min-lr 1.0e-7 \	104	--min-lr 1.0e-7 \
	@@ -115,9 +117,9 @@ GPT_ARGS="
115	--recompute-method block \	117	--recompute-method block \
116	--recompute-num-layers 4 \	118	--recompute-num-layers 4 \
117	--enable-recompute-layers-per-pp-rank \	119	--enable-recompute-layers-per-pp-rank \
118	- --use-fused-ring-attention-update \
119	- --use-fused-moe-token-permute-and-unpermute \
120	--recompute-in-advance \	120	--recompute-in-advance \
		121	+ --fix-router \
		122	+ --distributed-timeout-minutes 120 \
121	--bf16	123	--bf16
122	"	124	"
123		125

Mtests/poc/deepseek_500b/pretrain_deepseek_500b_8k_256die_A3.sh+9-10

	@@ -1,7 +1,4 @@
1	#!/bin/bash	1	#!/bin/bash
2	-export HCCL_ALGO="level0:fast_double_ring"
3	-export HCCL_OP_EXPANSION_MODE="AI_CPU"
4	-
5	export CUDA_DEVICE_MAX_CONNECTIONS=1	2	export CUDA_DEVICE_MAX_CONNECTIONS=1
6	export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True	3	export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
7		4
	@@ -34,9 +31,9 @@ DATA_PATH="your data path"
34	TOKENIZER_MODEL="your tokenizer path"	31	TOKENIZER_MODEL="your tokenizer path"
35		32
36	TP=1	33	TP=1
37	-PP=16	34	+PP=8
38	CP=1	35	CP=1
39	-EP=16	36	+EP=32
40	NUM_LAYERS=64	37	NUM_LAYERS=64
41	CP_TYPE='megatron_cp_algo'	38	CP_TYPE='megatron_cp_algo'
42	SEQ_LEN=8192	39	SEQ_LEN=8192
	@@ -57,7 +54,8 @@ MOE_ARGS="
57	--moe-token-dispatcher-type alltoall \	54	--moe-token-dispatcher-type alltoall \
58	--moe-alltoall-overlap-comm \	55	--moe-alltoall-overlap-comm \
59	--moe-router-topk 5 \	56	--moe-router-topk 5 \
60	- --moe-permutation-async-comm	57	+ --moe-permutation-async-comm \
		58	+ --use-fused-moe-token-permute-and-unpermute \
61	"	59	"
62		60
63	GPT_ARGS="	61	GPT_ARGS="
	@@ -96,7 +94,7 @@ GPT_ARGS="
96	--position-embedding-type rope \	94	--position-embedding-type rope \
97	--normalization RMSNorm \	95	--normalization RMSNorm \
98	--swiglu \	96	--swiglu \
99	- --use-fused-rotary-pos-emb \	97	+ --use-fused-rotary-pos-emb-new \
100	--use-fused-swiglu \	98	--use-fused-swiglu \
101	--use-fused-rmsnorm \	99	--use-fused-rmsnorm \
102	--use-flash-attn \	100	--use-flash-attn \
	@@ -114,12 +112,13 @@ GPT_ARGS="
114	--num-query-groups 4 \	112	--num-query-groups 4 \
115	--expert-model-parallel-size ${EP} \	113	--expert-model-parallel-size ${EP} \
116	--lr-warmup-fraction 0.01 \	114	--lr-warmup-fraction 0.01 \
117	- --recompute-granularity full \	115	+ --swap-attention \
118	--recompute-method block \	116	--recompute-method block \
119	- --recompute-num-layers 4 \	117	+ --recompute-num-layers 8 \
120	--enable-recompute-layers-per-pp-rank \	118	--enable-recompute-layers-per-pp-rank \
121	- --use-fused-moe-token-permute-and-unpermute \
122	--recompute-in-advance \	119	--recompute-in-advance \
		120	+ --fix-router \
		121	+ --distributed-timeout-minutes 120 \
123	--bf16	122	--bf16
124	"	123	"
125		124

Mtests/poc/llama2_70b/pretrain_llama2_70b_128k_256die_A2_pack_ptd.sh+1-1

	@@ -90,7 +90,7 @@ GPT_ARGS="
90	--bf16 \	90	--bf16 \
91	--swap-attention \	91	--swap-attention \
92	--recompute-activation-function \	92	--recompute-activation-function \
93	- --num-layers-per-virtual-pipeline-stage 10 \	93	+ --num-layers-per-virtual-pipeline-stage 2 \
94	--reset-position-ids \	94	--reset-position-ids \
95	--no-shared-storage \	95	--no-shared-storage \
96	"	96	"

Mtests/poc/llama2_70b/pretrain_llama2_70b_128k_256die_A3_pack_ptd.sh+10-0

	@@ -23,6 +23,14 @@ SEQ_LEN=131072
23	MBS=1	23	MBS=1
24	GBS=8	24	GBS=8
25		25
		26	+DISTRIBUTED_ARGS="
		27	+ --nproc_per_node $NPUS_PER_NODE \
		28	+ --nnodes $NNODES \
		29	+ --master_addr $MASTER_ADDR \
		30	+ --master_port $MASTER_PORT \
		31	+ --node_rank $NODE_RANK
		32	+"
		33	+
26	GPT_ARGS="	34	GPT_ARGS="
27	--use-mcore-models \	35	--use-mcore-models \
28	--rope-scaling-type llama3 \	36	--rope-scaling-type llama3 \
	@@ -79,6 +87,8 @@ GPT_ARGS="
79	--lr-warmup-fraction 0.01 \	87	--lr-warmup-fraction 0.01 \
80	--sequence-parallel \	88	--sequence-parallel \
81	--use-fused-ring-attention-update \	89	--use-fused-ring-attention-update \
		90	+ --use-ascend-coc \
		91	+ --coc-fused-kernel \
82	--bf16 \	92	--bf16 \
83	--swap-attention \	93	--swap-attention \
84	--recompute-activation-function \	94	--recompute-activation-function \

Mtests/poc/llama2_moe_500b/pretrain_llama2_moe_500b_128k_A3_ptd.sh+1-0

	@@ -86,6 +86,7 @@ GPT_ARGS="
86	--recompute-in-advance \	86	--recompute-in-advance \
87	--enable-recompute-layers-per-pp-rank \	87	--enable-recompute-layers-per-pp-rank \
88	--use-fused-ring-attention-update \	88	--use-fused-ring-attention-update \
		89	+ --fix-router \
89	--bf16	90	--bf16
90	"	91	"
91		92

Mtests/poc/llama2_moe_500b/pretrain_llama2_moe_500b_32k_A3_ptd.sh+1-0

	@@ -92,6 +92,7 @@ GPT_ARGS="
92	--recompute-in-advance \	92	--recompute-in-advance \
93	--use-fused-ring-attention-update \	93	--use-fused-ring-attention-update \
94	--use-fused-moe-token-permute-and-unpermute \	94	--use-fused-moe-token-permute-and-unpermute \
		95	+ --fix-router \
95	--bf16	96	--bf16
96	"	97	"
97		98

Mtests/poc/llama2_moe_500b/pretrain_llama2_moe_500b_8k_A3_ptd.sh+1-0

	@@ -92,6 +92,7 @@ GPT_ARGS="
92	--enable-recompute-layers-per-pp-rank \	92	--enable-recompute-layers-per-pp-rank \
93	--use-fused-moe-token-permute-and-unpermute \	93	--use-fused-moe-token-permute-and-unpermute \
94	--recompute-in-advance \	94	--recompute-in-advance \
		95	+ --fix-router \
95	--bf16	96	--bf16
96	"	97	"
97		98