Star206
303
代码
代码
Issues71
Pull Requests95
流水线
项目讨论
Wiki
分析
Star206
303
已合并
【mindspore】【sh】add deepseek2_lite/llama2/llama3/glm4 #2805
nomiz创建于 2025年6月10日
【mindspore】【sh】add deepseek2_lite/llama2/llama3/glm4 #2805
已合并
nomiz创建于 2025年6月10日
从refs/pull/2805/head合入到master
共 5 个文件变更+606-0
Aexamples/mindspore/deepseek2_lite/pretrain_deepseek2_lite_16b_8p_ms.sh+155-0
@@ -0,0 +1,155 @@
1+#!/bin/bash
2+export CUDA_DEVICE_MAX_CONNECTIONS=1
3+export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
4+ 
5+NPUS_PER_NODE=8
6+MASTER_ADDR=localhost
7+MASTER_PORT=6000
8+NNODES=1
9+NODE_RANK=0
10+WORLD_SIZE=$(($NPUS_PER_NODE*$NNODES))
11+ 
12+CKPT_SAVE_DIR="your model save ckpt path"
13+DATA_PATH="your data path"
14+TOKENIZER_PATH="your tokenizer path"
15+CKPT_LOAD_DIR="your model ckpt path"
16+ 
17+TP=1
18+PP=1
19+EP=8
20+ 
21+DISTRIBUTED_ARGS="
22+ --worker_num $WORLD_SIZE \
23+ --local_worker_num $NPUS_PER_NODE \
24+ --log_dir="msrun_log" \
25+ --join=True \
26+ --cluster_time_out=300 \
27+ --master_port $MASTER_PORT
28+"
29+ 
30+MLA_ARGS="
31+ --spec mindspeed_llm.tasks.models.spec.deepseek_spec layer_spec \
32+ --multi-head-latent-attention \
33+ --qk-rope-head-dim 64 \
34+ --qk-nope-head-dim 128 \
35+ --kv-lora-rank 512 \
36+ --v-head-dim 128 \
37+ --qk-layernorm \
38+"
39+ 
40+MOE_ARGS="
41+ --moe-grouped-gemm \
42+ --moe-alltoall-overlap-comm \
43+ --moe-permutation-async-comm \
44+ --moe-token-dispatcher-type alltoall \
45+ --use-fused-moe-token-permute-and-unpermute \
46+ --first-k-dense-replace 1 \
47+ --moe-layer-freq 1 \
48+ --n-shared-experts 2 \
49+ --num-experts 64 \
50+ --moe-router-topk 6 \
51+ --moe-intermediate-size 1408 \
52+ --moe-router-load-balancing-type pai_megatron_aux_loss \
53+ --topk-group 1 \
54+ --moe-aux-loss-coeff 0.01 \
55+ --routed-scaling-factor 1.0 \
56+ --seq-aux
57+"
58+ 
59+ROPE_ARGS="
60+ --rope-scaling-beta-fast 32 \
61+ --rope-scaling-beta-slow 1 \
62+ --rope-scaling-factor 40 \
63+ --rope-scaling-mscale 0.707 \
64+ --rope-scaling-mscale-all-dim 0.707 \
65+ --rope-scaling-original-max-position-embeddings 4096 \
66+ --rope-scaling-type yarn
67+"
68+ 
69+GPT_ARGS="
70+ --shape-order BNSD \
71+ --reuse-fp32-param \
72+ --load $CKPT_LOAD_DIR \
73+ --use-distributed-optimizer \
74+ --use-flash-attn \
75+ --use-mcore-models \
76+ --tensor-model-parallel-size ${TP} \
77+ --pipeline-model-parallel-size ${PP} \
78+ --expert-model-parallel-size ${EP} \
79+ --sequence-parallel \
80+ --num-layers 27 \
81+ --recompute-granularity full \
82+ --recompute-method uniform \
83+ --recompute-num-layers 1 \
84+ --hidden-size 2048 \
85+ --ffn-hidden-size 10944 \
86+ --num-attention-heads 16 \
87+ --tokenizer-type PretrainedFromHF \
88+ --tokenizer-name-or-path ${TOKENIZER_PATH} \
89+ --finetune \
90+ --num-workers 8 \
91+ --seq-length 4096 \
92+ --max-position-embeddings 163840 \
93+ --micro-batch-size 1 \
94+ --global-batch-size 8 \
95+ --make-vocab-size-divisible-by 1 \
96+ --lr 2e-5 \
97+ --train-iters 2000 \
98+ --lr-decay-iters 2000 \
99+ --lr-decay-style cosine \
100+ --untie-embeddings-and-output-weights \
101+ --disable-bias-linear \
102+ --attention-dropout 0.0 \
103+ --init-method-std 0.02 \
104+ --hidden-dropout 0.0 \
105+ --position-embedding-type rope \
106+ --normalization RMSNorm \
107+ --use-fused-rotary-pos-emb \
108+ --use-rotary-position-embeddings \
109+ --use-fused-swiglu \
110+ --use-fused-rmsnorm \
111+ --swiglu \
112+ --no-masked-softmax-fusion \
113+ --attention-softmax-in-fp32 \
114+ --min-lr 1.0e-8 \
115+ --weight-decay 1e-1 \
116+ --lr-warmup-iters 1920 \
117+ --clip-grad 1.0 \
118+ --adam-beta1 0.9 \
119+ --adam-beta2 0.95 \
120+ --initial-loss-scale 65536 \
121+ --vocab-size 102400 \
122+ --padded-vocab-size 102400 \
123+ --rotary-base 10000 \
124+ --no-gradient-accumulation-fusion \
125+ --norm-epsilon 1e-6 \
126+ --no-load-optim \
127+ --no-load-rng \
128+ --bf16
129+"
130+ 
131+DATA_ARGS="
132+ --data-path $DATA_PATH \
133+ --split 99,1,0
134+"
135+ 
136+OUTPUT_ARGS="
137+ --log-interval 1 \
138+ --save-interval 1000 \
139+ --eval-interval 10000 \
140+ --eval-iters 10 \
141+ --no-save-optim \
142+ --no-save-rng
143+"
144+ 
145+msrun ${DISTRIBUTED_ARGS} pretrain_gpt.py \
146+ $GPT_ARGS \
147+ $DATA_ARGS \
148+ $OUTPUT_ARGS \
149+ $MLA_ARGS \
150+ $ROPE_ARGS \
151+ $MOE_ARGS \
152+ --ai-framework mindspore \
153+ --distributed-backend nccl \
154+ --save $CKPT_SAVE_DIR \
155+ | tee logs/pretrain_deepseek2_lite_8p_ms.log
Aexamples/mindspore/glm4/pretrain_glm4_9b_32k_ms.sh+110-0
@@ -0,0 +1,110 @@
1+#!/bin/bash
2+export CUDA_DEVICE_MAX_CONNECTIONS=1
3+ 
4+NPUS_PER_NODE=8
5+MASTER_ADDR="your master node IP"
6+MASTER_PORT=6000
7+NNODES=2
8+NODE_RANK=0
9+WORLD_SIZE=$(($NPUS_PER_NODE*$NNODES))
10+ 
11+CKPT_SAVE_DIR="your model save ckpt path"
12+DATA_PATH="your data path"
13+TOKENIZER_PATH="your tokenizer path"
14+CKPT_LOAD_DIR="your model ckpt path"
15+ 
16+TP=2
17+PP=2
18+CP=4
19+ 
20+DISTRIBUTED_ARGS="
21+ --worker_num $WORLD_SIZE \
22+ --local_worker_num $NPUS_PER_NODE \
23+ --log_dir="msrun_log" \
24+ --join=True \
25+ --cluster_time_out=300 \
26+ --master_port $MASTER_PORT
27+"
28+ 
29+GPT_ARGS="
30+ --tensor-model-parallel-size ${TP} \
31+ --pipeline-model-parallel-size ${PP} \
32+ --context-parallel-algo megatron_cp_algo \
33+ --context-parallel-size ${CP} \
34+ --sequence-parallel \
35+ --use-mcore-models \
36+ --use-flash-attn \
37+ --use-fused-swiglu \
38+ --use-fused-rmsnorm \
39+ --overlap-grad-reduce \
40+ --overlap-param-gather \
41+ --use-cp-send-recv-overlap \
42+ --use-distributed-optimizer \
43+ --num-layers 40 \
44+ --hidden-size 4096 \
45+ --ffn-hidden-size 13696 \
46+ --num-attention-heads 32 \
47+ --seq-length 32768 \
48+ --micro-batch-size 1 \
49+ --global-batch-size 64 \
50+ --max-position-embeddings 32768 \
51+ --padded-vocab-size 151552 \
52+ --make-vocab-size-divisible-by 1 \
53+ --group-query-attention \
54+ --num-query-groups 2 \
55+ --disable-bias-linear \
56+ --add-qkv-bias \
57+ --position-embedding-type rope \
58+ --use-glm-rope \
59+ --rotary-percent 0.5 \
60+ --rotary-base 5000000 \
61+ --normalization RMSNorm \
62+ --swiglu \
63+ --tokenizer-type PretrainedFromHF \
64+ --tokenizer-name-or-path ${TOKENIZER_PATH} \
65+ --norm-epsilon 1.5625e-07 \
66+ --lr 1.25e-6 \
67+ --train-iters 2000 \
68+ --lr-decay-style cosine \
69+ --untie-embeddings-and-output-weights \
70+ --attention-dropout 0.0 \
71+ --init-method-std 0.01 \
72+ --hidden-dropout 0.0 \
73+ --no-masked-softmax-fusion \
74+ --no-create-attention-mask-in-dataloader \
75+ --attention-softmax-in-fp32 \
76+ --min-lr 1.25e-7 \
77+ --weight-decay 1e-1 \
78+ --lr-warmup-fraction 0.01 \
79+ --clip-grad 1.0 \
80+ --adam-beta1 0.9 \
81+ --adam-beta2 0.95 \
82+ --initial-loss-scale 4096 \
83+ --no-gradient-accumulation-fusion \
84+ --no-load-optim \
85+ --no-load-rng \
86+ --no-shared-storage \
87+ --bf16
88+"
89+ 
90+DATA_ARGS="
91+ --data-path ${DATA_PATH} \
92+ --split 100,0,0
93+"
94+ 
95+OUTPUT_ARGS="
96+ --log-interval 1 \
97+ --save-interval 2000 \
98+ --eval-interval 1000 \
99+ --eval-iters 0 \
100+"
101+ 
102+msrun ${DISTRIBUTED_ARGS} pretrain_gpt.py \
103+ ${GPT_ARGS} \
104+ ${DATA_ARGS} \
105+ ${OUTPUT_ARGS} \
106+ --ai-framework mindspore \
107+ --distributed-backend nccl \
108+ --load ${CKPT_LOAD_DIR} \
109+ --save ${CKPT_SAVE_DIR} \
110+ | tee logs/pretrain_glm4_9b_32k_ms.log
Aexamples/mindspore/llama2/tune_llama2_7b_full_pack_16k_ms.sh+115-0
@@ -0,0 +1,115 @@
1+#!/bin/bash
2+ 
3+export CUDA_DEVICE_MAX_CONNECTIONS=1
4+export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
5+ 
6+NPUS_PER_NODE=8
7+MASTER_ADDR=localhost
8+MASTER_PORT=6039
9+NNODES=1
10+NODE_RANK=0
11+WORLD_SIZE=$(($NPUS_PER_NODE*$NNODES))
12+ 
13+CKPT_SAVE_DIR="your model save ckpt path"
14+DATA_PATH="your data path"
15+TOKENIZER_PATH="your tokenizer path"
16+CKPT_LOAD_DIR="your model ckpt path"
17+ 
18+TP=1
19+PP=2
20+CP=2
21+ 
22+DISTRIBUTED_ARGS="
23+ --worker_num $WORLD_SIZE \
24+ --local_worker_num $NPUS_PER_NODE \
25+ --log_dir="msrun_log" \
26+ --join=True \
27+ --cluster_time_out=300 \
28+ --master_port $MASTER_PORT
29+"
30+ 
31+ 
32+GPT_ARGS="
33+ --no-check-for-nan-in-loss-and-grad \
34+ --use-mcore-models \
35+ --reuse-fp32-param \
36+ --tensor-model-parallel-size ${TP} \
37+ --pipeline-model-parallel-size ${PP} \
38+ --context-parallel-size ${CP} \
39+ --load ${CKPT_LOAD_DIR} \
40+ --context-parallel-algo megatron_cp_algo \
41+ --sequence-parallel \
42+ --cp-attention-mask-type general \
43+ --swap-attention \
44+ --num-workers 16 \
45+ --cp-window-size 1 \
46+ --use-fused-rotary-pos-emb \
47+ --use-fused-swiglu \
48+ --use-fused-rmsnorm \
49+ --use-cp-send-recv-overlap \
50+ --log-throughput \
51+ --overlap-grad-reduce \
52+ --overlap-param-gather \
53+ --sequence-parallel \
54+ --use-distributed-optimizer \
55+ --num-layers 32 \
56+ --hidden-size 4096 \
57+ --ffn-hidden-size 11008 \
58+ --num-attention-heads 32 \
59+ --tokenizer-type PretrainedFromHF \
60+ --tokenizer-name-or-path ${TOKENIZER_PATH} \
61+ --seq-length 16384 \
62+ --max-position-embeddings 16384 \
63+ --micro-batch-size 1 \
64+ --global-batch-size 128 \
65+ --make-vocab-size-divisible-by 1 \
66+ --lr 1.0e-6 \
67+ --train-iters 2000 \
68+ --lr-decay-style cosine \
69+ --untie-embeddings-and-output-weights \
70+ --attention-dropout 0.0 \
71+ --init-method-std 0.01 \
72+ --hidden-dropout 0.0 \
73+ --position-embedding-type rope \
74+ --normalization RMSNorm \
75+ --swiglu \
76+ --use-flash-attn \
77+ --no-masked-softmax-fusion \
78+ --attention-softmax-in-fp32 \
79+ --weight-decay 1e-1 \
80+ --clip-grad 1.0 \
81+ --adam-beta1 0.9 \
82+ --initial-loss-scale 1 \
83+ --adam-beta2 0.999 \
84+ --adam-eps 1e-5 \
85+ --disable-bias-linear \
86+ --group-query-attention \
87+ --num-query-groups 8 \
88+ --bf16 \
89+ --is-instruction-dataset \
90+ --finetune \
91+ --stage sft \
92+ 
93+"
94+ 
95+DATA_ARGS="
96+ --data-path $DATA_PATH \
97+ --split 100,0,0 \
98+ --reset-position-ids \
99+"
100+ 
101+OUTPUT_ARGS="
102+ --log-interval 1 \
103+ --save-interval 2000 \
104+ --eval-interval 1000 \
105+ --eval-iters 0 \
106+"
107+ 
108+msrun $DISTRIBUTED_ARGS posttrain_gpt.py \
109+ $GPT_ARGS \
110+ $DATA_ARGS \
111+ $OUTPUT_ARGS \
112+ --ai-framework mindspore \
113+ --distributed-backend nccl \
114+ --save $CKPT_SAVE_DIR \
115+ | tee logs/tune_llama2_7b_full_16K_ms.log
Aexamples/mindspore/llama2/tune_llama2_7b_full_pack_32k_ms.sh+115-0
@@ -0,0 +1,115 @@
1+#!/bin/bash
2+ 
3+export CUDA_DEVICE_MAX_CONNECTIONS=1
4+ 
5+ 
6+NPUS_PER_NODE=8
7+MASTER_ADDR=localhost
8+MASTER_PORT=6065
9+NNODES=1
10+NODE_RANK=0
11+WORLD_SIZE=$(($NPUS_PER_NODE*$NNODES))
12+ 
13+CKPT_SAVE_DIR="your model save ckpt path"
14+DATA_PATH="your data path"
15+TOKENIZER_PATH="your tokenizer path"
16+CKPT_LOAD_DIR="your model ckpt path"
17+ 
18+TP=2
19+PP=1
20+CP=4
21+ 
22+DISTRIBUTED_ARGS="
23+ --worker_num $WORLD_SIZE \
24+ --local_worker_num $NPUS_PER_NODE \
25+ --log_dir="msrun_log" \
26+ --join=True \
27+ --cluster_time_out=300 \
28+ --master_port $MASTER_PORT
29+"
30+ 
31+ 
32+GPT_ARGS="
33+ --no-check-for-nan-in-loss-and-grad \
34+ --use-mcore-models \
35+ --reuse-fp32-param \
36+ --tensor-model-parallel-size ${TP} \
37+ --pipeline-model-parallel-size ${PP} \
38+ --context-parallel-size ${CP} \
39+ --load ${CKPT_LOAD_DIR} \
40+ --context-parallel-algo ulysses_cp_algo \
41+ --ulysses-degree-in-cp 2 \
42+ --sequence-parallel \
43+ --cp-attention-mask-type causal \
44+ --num-workers 16 \
45+ --cp-window-size 1 \
46+ --use-fused-rotary-pos-emb \
47+ --use-fused-swiglu \
48+ --use-fused-rmsnorm \
49+ --use-cp-send-recv-overlap \
50+ --log-throughput \
51+ --overlap-grad-reduce \
52+ --overlap-param-gather \
53+ --sequence-parallel \
54+ --use-distributed-optimizer \
55+ --num-layers 32 \
56+ --hidden-size 4096 \
57+ --ffn-hidden-size 11008 \
58+ --num-attention-heads 32 \
59+ --tokenizer-type PretrainedFromHF \
60+ --tokenizer-name-or-path ${TOKENIZER_PATH} \
61+ --seq-length 32768 \
62+ --max-position-embeddings 32768 \
63+ --micro-batch-size 1 \
64+ --global-batch-size 64 \
65+ --make-vocab-size-divisible-by 1 \
66+ --lr 1.0e-6 \
67+ --train-iters 2000 \
68+ --lr-decay-style cosine \
69+ --untie-embeddings-and-output-weights \
70+ --attention-dropout 0.0 \
71+ --init-method-std 0.01 \
72+ --hidden-dropout 0.0 \
73+ --position-embedding-type rope \
74+ --normalization RMSNorm \
75+ --swiglu \
76+ --use-flash-attn \
77+ --no-masked-softmax-fusion \
78+ --attention-softmax-in-fp32 \
79+ --weight-decay 1e-1 \
80+ --clip-grad 1.0 \
81+ --adam-beta1 0.9 \
82+ --initial-loss-scale 1 \
83+ --adam-beta2 0.999 \
84+ --adam-eps 1e-5 \
85+ --disable-bias-linear \
86+ --group-query-attention \
87+ --num-query-groups 8 \
88+ --bf16 \
89+ --is-instruction-dataset \
90+ --finetune \
91+ --stage sft \
92+ 
93+"
94+ 
95+DATA_ARGS="
96+ --data-path $DATA_PATH \
97+ --split 100,0,0 \
98+ --reset-position-ids \
99+"
100+ 
101+OUTPUT_ARGS="
102+ --log-interval 1 \
103+ --save-interval 2000 \
104+ --eval-interval 1000 \
105+ --eval-iters 0 \
106+"
107+ 
108+msrun $DISTRIBUTED_ARGS posttrain_gpt.py \
109+ $GPT_ARGS \
110+ $DATA_ARGS \
111+ $OUTPUT_ARGS \
112+ --ai-framework mindspore \
113+ --distributed-backend nccl \
114+ --save $CKPT_SAVE_DIR \
115+ | tee logs/tune_llama2_7b_full_32k_ms.log
Aexamples/mindspore/llama3/tune_llama3_8b_full_ms.sh+111-0
@@ -0,0 +1,111 @@
1+#!/bin/bash
2+ 
3+export CUDA_DEVICE_MAX_CONNECTIONS=1
4+ 
5+ 
6+NPUS_PER_NODE=8
7+MASTER_ADDR=localhost
8+MASTER_PORT=6024
9+NNODES=1
10+NODE_RANK=0
11+WORLD_SIZE=$(($NPUS_PER_NODE*$NNODES))
12+ 
13+CKPT_SAVE_DIR="your model save ckpt path"
14+DATA_PATH="your data path"
15+TOKENIZER_PATH="your tokenizer path"
16+CKPT_LOAD_DIR="your model ckpt path"
17+ 
18+TP=1
19+PP=8
20+ 
21+DISTRIBUTED_ARGS="
22+ --worker_num $WORLD_SIZE \
23+ --local_worker_num $NPUS_PER_NODE \
24+ --log_dir="msrun_log" \
25+ --join=True \
26+ --cluster_time_out=300 \
27+ --master_port $MASTER_PORT
28+"
29+ 
30+GPT_ARGS="
31+ --stage sft \
32+ --tensor-model-parallel-size ${TP} \
33+ --pipeline-model-parallel-size ${PP} \
34+ --use-mcore-models \
35+ --micro-batch-size 1 \
36+ --global-batch-size 8 \
37+ --sequence-parallel \
38+ --use-flash-attn \
39+ --prompt-type llama3 \
40+ --variable-seq-lengths \
41+ --use-rotary-position-embeddings \
42+ --rope-scaling-type llama3 \
43+ --rope-scaling-factor 8.0 \
44+ --rotary-percent 1.0 \
45+ --low-freq-factor 1.0 \
46+ --high-freq-factor 4.0 \
47+ --original-max-position-embeddings 8192 \
48+ --tokenizer-type PretrainedFromHF \
49+ --tokenizer-name-or-path ${TOKENIZER_PATH} \
50+ --tokenizer-not-use-fast \
51+ --num-layers 32 \
52+ --hidden-size 4096 \
53+ --ffn-hidden-size 14336 \
54+ --num-attention-heads 32 \
55+ --group-query-attention \
56+ --num-query-groups 8 \
57+ --seq-length 8192 \
58+ --max-position-embeddings 8192 \
59+ --make-vocab-size-divisible-by 1 \
60+ --padded-vocab-size 128256 \
61+ --untie-embeddings-and-output-weights \
62+ --disable-bias-linear \
63+ --attention-dropout 0.0 \
64+ --init-method-std 0.02 \
65+ --hidden-dropout 0.0 \
66+ --position-embedding-type rope \
67+ --rotary-base 500000 \
68+ --normalization RMSNorm \
69+ --norm-epsilon 1e-5 \
70+ --swiglu \
71+ --no-masked-softmax-fusion \
72+ --attention-softmax-in-fp32 \
73+ --lr 1e-7 \
74+ --train-iters 2000 \
75+ --lr-decay-style constant \
76+ --weight-decay 0.0 \
77+ --clip-grad 1.0 \
78+ --adam-beta1 0.9 \
79+ --adam-beta2 0.999 \
80+ --initial-loss-scale 4096 \
81+ --finetune \
82+ --is-instruction-dataset \
83+ --no-gradient-accumulation-fusion \
84+ --no-load-optim \
85+ --no-load-rng \
86+ --bf16 \
87+ --seed 42 \
88+ --vocab-size 128256 \
89+"
90+ 
91+DATA_ARGS="
92+ --data-path $DATA_PATH \
93+ --split 100,0,0
94+"
95+ 
96+OUTPUT_ARGS="
97+ --log-interval 1 \
98+ --save-interval 2000 \
99+ --eval-interval 2000 \
100+ --eval-iters 10 \
101+ --load ${CKPT_LOAD_DIR} \
102+ --save ${CKPT_SAVE_DIR} \
103+"
104+ 
105+msrun $DISTRIBUTED_ARGS posttrain_gpt.py \
106+ $GPT_ARGS \
107+ $DATA_ARGS \
108+ $OUTPUT_ARGS \
109+ --ai-framework mindspore \
110+ --distributed-backend nccl \
111+ | tee logs/tune_llama31_8b_full_ms.log