Star206
310
代码
代码
Issues70
Pull Requests98
流水线
项目讨论
Wiki
分析
Star206
310
已合并
添加Qwen2.5 0.5B/1.5B/3B LoRA微调,权重合并及chat脚本 Qwen2.5其他微调脚本添加padded-samples参数 #2325
AtomGit-Bot创建于 2025年3月3日
添加Qwen2.5 0.5B/1.5B/3B LoRA微调,权重合并及chat脚本 Qwen2.5其他微调脚本添加padded-samples参数 #2325
已合并
AtomGit-Bot创建于 2025年3月3日
从refs/pull/2325/head合入到master
共 14 个文件变更+465-0
Aexamples/mcore/qwen25/chat_qwen25_7b_lora_ptd.sh+74-0
@@ -0,0 +1,74 @@
1+#!/bin/bash
2+export CUDA_DEVICE_MAX_CONNECTIONS=1
3+ 
4+# please fill these path configurations
5+TOKENIZER_PATH="your tokenizer directory path"
6+CHECKPOINT="your model directory path"
7+CHECKPOINT_LORA="your lora model directory path"
8+ 
9+# Change for multinode config
10+MASTER_ADDR=localhost
11+MASTER_PORT=6000
12+NNODES=1
13+NODE_RANK=0
14+NPUS_PER_NODE=1
15+WORLD_SIZE=$(($NPUS_PER_NODE*$NNODES))
16+ 
17+TP=1
18+PP=1
19+SEQ_LENGTH=4096
20+ 
21+DISTRIBUTED_ARGS="
22+ --nproc_per_node $NPUS_PER_NODE \
23+ --nnodes $NNODES \
24+ --node_rank $NODE_RANK \
25+ --master_addr $MASTER_ADDR \
26+ --master_port $MASTER_PORT
27+"
28+ 
29+LORA_ARGS="
30+ --lora-load ${CHECKPOINT_LORA} \
31+ --lora-r 8 \
32+ --lora-alpha 16 \
33+ --lora-fusion \
34+ --lora-target-modules linear_qkv linear_proj linear_fc1 linear_fc2
35+"
36+ 
37+torchrun $DISTRIBUTED_ARGS inference.py \
38+ --task chat \
39+ --prompt-type qwen \
40+ --use-mcore-models \
41+ --tensor-model-parallel-size ${TP} \
42+ --pipeline-model-parallel-size ${PP} \
43+ --load ${CHECKPOINT} \
44+ --num-layers 28 \
45+ --hidden-size 3584 \
46+ --num-attention-heads 28 \
47+ --ffn-hidden-size 18944 \
48+ --max-position-embeddings ${SEQ_LENGTH} \
49+ --seq-length ${SEQ_LENGTH} \
50+ --make-vocab-size-divisible-by 1 \
51+ --padded-vocab-size 152064 \
52+ --rotary-base 1000000 \
53+ --untie-embeddings-and-output-weights \
54+ --micro-batch-size 1 \
55+ --swiglu \
56+ --disable-bias-linear \
57+ --tokenizer-type PretrainedFromHF \
58+ --tokenizer-name-or-path ${TOKENIZER_PATH} \
59+ --normalization RMSNorm \
60+ --position-embedding-type rope \
61+ --norm-epsilon 1e-6 \
62+ --hidden-dropout 0 \
63+ --attention-dropout 0 \
64+ --tokenizer-not-use-fast \
65+ --add-qkv-bias \
66+ --max-new-tokens 256 \
67+ --no-gradient-accumulation-fusion \
68+ --exit-on-missing-checkpoint \
69+ --attention-softmax-in-fp32 \
70+ --seed 42 \
71+ --group-query-attention \
72+ --num-query-groups 4 \
73+ ${LORA_ARGS} \
74+ | tee logs/chat_mcore_qwen25_7b_lora.log
Aexamples/mcore/qwen25/ckpt_convert_qwen25_mcore2hf_lora.sh+19-0
@@ -0,0 +1,19 @@
1+# 修改 ascend-toolkit 路径
2+source /usr/local/Ascend/ascend-toolkit/set_env.sh
3+ 
4+# 设置并行策略
5+python convert_ckpt.py \
6+ --use-mcore-models \
7+ --model-type GPT \
8+ --model-type-hf llama2 \
9+ --load-model-type mg \
10+ --save-model-type hf \
11+ --target-tensor-parallel-size 1 \
12+ --target-pipeline-parallel-size 1 \
13+ --add-qkv-bias \
14+ --lora-r 8 \
15+ --lora-alpha 16 \
16+ --lora-target-modules linear_qkv linear_proj linear_fc1 linear_fc2 \
17+ --load-dir ./model_weights/qwen25_mcore/ \
18+ --lora-load ./ckpt/qwen25_7b_lora \
19+ --save-dir ./model_from_hf/qwen25_7b_hf/ # 需要填入原始HF模型路径，新权重会存于./model_from_hf/qwen25_7b_hf/mg2hg/
Aexamples/mcore/qwen25/tune_qwen25_0point5b_4k_lora_ptd.sh+121-0
@@ -0,0 +1,121 @@
1+#!/bin/bash
2+export CUDA_DEVICE_MAX_CONNECTIONS=1
3+ 
4+NPUS_PER_NODE=8
5+MASTER_ADDR=localhost
6+MASTER_PORT=6011
7+NNODES=1
8+NODE_RANK=0
9+WORLD_SIZE=$(($NPUS_PER_NODE*$NNODES))
10+ 
11+# please fill these path configurations
12+CKPT_SAVE_DIR="your model save ckpt path"
13+DATA_PATH="your data path"
14+TOKENIZER_PATH="your tokenizer path"
15+CKPT_LOAD_DIR="your model ckpt path"
16+ 
17+TP=1
18+PP=1
19+MBS=1
20+GBS=8
21+SEQ_LEN=4096
22+CP_ALGO=megatron_cp_algo
23+ 
24+DISTRIBUTED_ARGS="
25+ --nproc_per_node $NPUS_PER_NODE \
26+ --nnodes $NNODES \
27+ --node_rank $NODE_RANK \
28+ --master_addr $MASTER_ADDR \
29+ --master_port $MASTER_PORT
30+"
31+ 
32+TUNE_ARGS="
33+ --finetune \
34+ --stage sft \
35+ --is-instruction-dataset \
36+ --tokenizer-not-use-fast \
37+ --prompt-type qwen \
38+ --variable-seq-lengths \
39+ --padded-samples \
40+ --lora-r 8 \
41+ --lora-alpha 16 \
42+ --lora-fusion \
43+ --lora-target-modules linear_qkv linear_proj linear_fc1 linear_fc2
44+"
45+ 
46+GPT_ARGS="
47+ --use-mcore-models \
48+ --tensor-model-parallel-size ${TP} \
49+ --pipeline-model-parallel-size ${PP} \
50+ --use-distributed-optimizer \
51+ --tokenizer-type PretrainedFromHF \
52+ --tokenizer-name-or-path ${TOKENIZER_PATH} \
53+ --seq-length ${SEQ_LEN} \
54+ --max-position-embeddings ${SEQ_LEN} \
55+ --micro-batch-size ${MBS} \
56+ --global-batch-size ${GBS} \
57+ --group-query-attention \
58+ --num-query-groups 2 \
59+ --num-layers 24 \
60+ --hidden-size 896 \
61+ --ffn-hidden-size 4864 \
62+ --num-attention-heads 14 \
63+ --rotary-base 1000000 \
64+ --normalization RMSNorm \
65+ --norm-epsilon 1e-06 \
66+ --swiglu \
67+ --add-qkv-bias \
68+ --disable-bias-linear \
69+ --attention-dropout 0.0 \
70+ --hidden-dropout 0.0 \
71+ --make-vocab-size-divisible-by 1 \
72+ --padded-vocab-size 151936 \
73+ --lr 7.75e-7 \
74+ --train-iters 2000 \
75+ --lr-decay-style cosine \
76+ --lr-warmup-fraction 0.01 \
77+ --init-method-std 0.01 \
78+ --position-embedding-type rope \
79+ --use-fused-rmsnorm \
80+ --use-fused-rotary-pos-emb \
81+ --use-rotary-position-embeddings \
82+ --use-fused-swiglu \
83+ --overlap-grad-reduce \
84+ --use-flash-attn \
85+ --no-masked-softmax-fusion \
86+ --attention-softmax-in-fp32 \
87+ --min-lr 7.75e-8 \
88+ --weight-decay 1e-1 \
89+ --clip-grad 1.0 \
90+ --adam-beta1 0.9 \
91+ --adam-beta2 0.95 \
92+ --initial-loss-scale 4096 \
93+ --no-gradient-accumulation-fusion \
94+ --no-load-optim \
95+ --no-load-rng \
96+ --seed 42 \
97+ --bf16
98+"
99+ 
100+DATA_ARGS="
101+ --data-path $DATA_PATH \
102+ --split 100,0,0
103+"
104+ 
105+OUTPUT_ARGS="
106+ --log-interval 1 \
107+ --save-interval 1000 \
108+ --eval-interval 1000 \
109+ --eval-iters 0 \
110+"
111+ 
112+torchrun $DISTRIBUTED_ARGS postrain_gpt.py \
113+ $GPT_ARGS \
114+ $DATA_ARGS \
115+ $OUTPUT_ARGS \
116+ $TUNE_ARGS \
117+ --distributed-backend nccl \
118+ --log-throughput \
119+ --load ${CKPT_LOAD_DIR} \
120+ --save ${CKPT_SAVE_DIR} \
121+ | tee logs/tune_mcore_qwen25_0point5b_4k_lora.log
Mexamples/mcore/qwen25/tune_qwen25_14b_4K_lora_pack.sh+2-0
@@ -95,6 +95,8 @@ TUNE_ARGS="
95 --is-instruction-dataset \95 --is-instruction-dataset \
96 --tokenizer-not-use-fast \96 --tokenizer-not-use-fast \
97 --prompt-type qwen \97 --prompt-type qwen \
98+ --padded-samples \
99+ --neat-pack \
98 --lora-r 8 \100 --lora-r 8 \
99 --lora-alpha 16 \101 --lora-alpha 16 \
100 --lora-fusion \102 --lora-fusion \
Mexamples/mcore/qwen25/tune_qwen25_14b_4k_full_ptd.sh+1-0
@@ -35,6 +35,7 @@ TUNE_ARGS="
35 --is-instruction-dataset \35 --is-instruction-dataset \
36 --tokenizer-not-use-fast \36 --tokenizer-not-use-fast \
37 --prompt-type qwen \37 --prompt-type qwen \
38+ --padded-samples \
38 --variable-seq-lengths \39 --variable-seq-lengths \
39"40"
40 41 
Mexamples/mcore/qwen25/tune_qwen25_14b_4k_lora_ptd.sh+1-0
@@ -35,6 +35,7 @@ TUNE_ARGS="
35 --is-instruction-dataset \35 --is-instruction-dataset \
36 --tokenizer-not-use-fast \36 --tokenizer-not-use-fast \
37 --prompt-type qwen \37 --prompt-type qwen \
38+ --padded-samples \
38 --variable-seq-lengths \39 --variable-seq-lengths \
39 --lora-r 8 \40 --lora-r 8 \
40 --lora-alpha 16 \41 --lora-alpha 16 \
Aexamples/mcore/qwen25/tune_qwen25_1point5b_4k_lora_ptd.sh+120-0
@@ -0,0 +1,120 @@
1+#!/bin/bash
2+export CUDA_DEVICE_MAX_CONNECTIONS=1
3+ 
4+# Change for multinode config
5+MASTER_ADDR=localhost
6+MASTER_PORT=6000
7+NNODES=1
8+NODE_RANK=0
9+NPUS_PER_NODE=8
10+WORLD_SIZE=$(($NPUS_PER_NODE*$NNODES))
11+ 
12+# please fill these path configurations
13+CKPT_LOAD_DIR="your model ckpt path"
14+CKPT_SAVE_DIR="your model save ckpt path"
15+DATA_PATH="your data path"
16+TOKENIZER_PATH="your tokenizer path"
17+ 
18+TP=1
19+PP=1
20+MBS=1
21+GBS=8
22+SEQ_LEN=4096
23+ 
24+DISTRIBUTED_ARGS="
25+ --nproc_per_node $NPUS_PER_NODE \
26+ --nnodes $NNODES \
27+ --node_rank $NODE_RANK \
28+ --master_addr $MASTER_ADDR \
29+ --master_port $MASTER_PORT
30+"
31+ 
32+TUNE_ARGS="
33+ --finetune \
34+ --stage sft \
35+ --is-instruction-dataset \
36+ --tokenizer-not-use-fast \
37+ --prompt-type qwen \
38+ --padded-samples \
39+ --variable-seq-lengths \
40+ --lora-r 8 \
41+ --lora-alpha 16 \
42+ --lora-fusion \
43+ --lora-target-modules linear_qkv linear_proj linear_fc1 linear_fc2
44+"
45+ 
46+GPT_ARGS="
47+ --use-mcore-models \
48+ --tensor-model-parallel-size ${TP} \
49+ --pipeline-model-parallel-size ${PP}
50+ --num-layers 28 \
51+ --hidden-size 1536 \
52+ --ffn-hidden-size 8960 \
53+ --num-attention-heads 12 \
54+ --group-query-attention \
55+ --num-query-groups 2 \
56+ --tokenizer-type PretrainedFromHF \
57+ --tokenizer-name-or-path ${TOKENIZER_PATH} \
58+ --seq-length ${SEQ_LEN} \
59+ --max-position-embeddings ${SEQ_LEN} \
60+ --micro-batch-size ${MBS} \
61+ --global-batch-size ${GBS} \
62+ --make-vocab-size-divisible-by 1 \
63+ --padded-vocab-size 151936 \
64+ --rotary-base 1000000 \
65+ --train-iters 2000 \
66+ --lr 7.75e-7 \
67+ --min-lr 7.75e-8 \
68+ --weight-decay 1e-1 \
69+ --lr-decay-style cosine \
70+ --lr-warmup-fraction 0.01 \
71+ --clip-grad 1.0 \
72+ --adam-beta1 0.9 \
73+ --adam-beta2 0.95 \
74+ --add-qkv-bias \
75+ --disable-bias-linear \
76+ --attention-dropout 0.0 \
77+ --init-method-std 0.01 \
78+ --hidden-dropout 0.0 \
79+ --position-embedding-type rope \
80+ --normalization RMSNorm \
81+ --norm-epsilon 1e-06 \
82+ --swiglu \
83+ --use-distributed-optimizer \
84+ --use-flash-attn \
85+ --use-fused-rotary-pos-emb \
86+ --use-rotary-position-embeddings \
87+ --use-fused-swiglu \
88+ --use-fused-rmsnorm \
89+ --overlap-grad-reduce \
90+ --no-masked-softmax-fusion \
91+ --attention-softmax-in-fp32 \
92+ --initial-loss-scale 4096 \
93+ --no-gradient-accumulation-fusion \
94+ --no-load-optim \
95+ --no-load-rng \
96+ --seed 42 \
97+ --bf16
98+"
99+ 
100+DATA_ARGS="
101+ --data-path $DATA_PATH \
102+ --split 100,0,0
103+"
104+ 
105+OUTPUT_ARGS="
106+ --log-interval 1 \
107+ --save-interval 2000 \
108+ --eval-interval 2000 \
109+ --eval-iters 0 \
110+"
111+ 
112+torchrun $DISTRIBUTED_ARGS posttrain_gpt.py \
113+ $GPT_ARGS \
114+ $DATA_ARGS \
115+ $OUTPUT_ARGS \
116+ $TUNE_ARGS \
117+ --distributed-backend nccl \
118+ --load ${CKPT_LOAD_DIR} \
119+ --save ${CKPT_SAVE_DIR} \
120+ | tee logs/tune_mcore_qwen25_1point5b_4k_lora.log
Mexamples/mcore/qwen25/tune_qwen25_32b_4k_full_ptd.sh+1-0
@@ -44,6 +44,7 @@ TUNE_ARGS="
44 --is-instruction-dataset \44 --is-instruction-dataset \
45 --tokenizer-not-use-fast \45 --tokenizer-not-use-fast \
46 --prompt-type qwen \46 --prompt-type qwen \
47+ --padded-samples \
47 --variable-seq-lengths \48 --variable-seq-lengths \
48"49"
49 50 
Mexamples/mcore/qwen25/tune_qwen25_32b_4k_lora_ptd.sh+1-0
@@ -35,6 +35,7 @@ TUNE_ARGS="
35 --is-instruction-dataset \35 --is-instruction-dataset \
36 --tokenizer-not-use-fast \36 --tokenizer-not-use-fast \
37 --prompt-type qwen \37 --prompt-type qwen \
38+ --padded-samples \
38 --variable-seq-lengths \39 --variable-seq-lengths \
39 --lora-r 8 \40 --lora-r 8 \
40 --lora-alpha 16 \41 --lora-alpha 16 \
Aexamples/mcore/qwen25/tune_qwen25_3b_4k_lora_ptd.sh+121-0
@@ -0,0 +1,121 @@
1+#!/bin/bash
2+export CUDA_DEVICE_MAX_CONNECTIONS=1
3+ 
4+# Change for multinode config
5+MASTER_ADDR=localhost
6+MASTER_PORT=6000
7+NNODES=1
8+NODE_RANK=0
9+NPUS_PER_NODE=8
10+WORLD_SIZE=$(($NPUS_PER_NODE*$NNODES))
11+ 
12+# please fill these path configurations
13+CKPT_LOAD_DIR="your model ckpt path"
14+CKPT_SAVE_DIR="your model save ckpt path"
15+DATA_PATH="your data path"
16+TOKENIZER_PATH="your tokenizer path"
17+ 
18+TP=1
19+PP=1
20+MBS=1
21+GBS=8
22+SEQ_LEN=4096
23+ 
24+DISTRIBUTED_ARGS="
25+ --nproc_per_node $NPUS_PER_NODE \
26+ --nnodes $NNODES \
27+ --node_rank $NODE_RANK \
28+ --master_addr $MASTER_ADDR \
29+ --master_port $MASTER_PORT
30+"
31+ 
32+TUNE_ARGS="
33+ --finetune \
34+ --stage sft \
35+ --is-instruction-dataset \
36+ --tokenizer-not-use-fast \
37+ --prompt-type qwen \
38+ --padded-samples \
39+ --variable-seq-lengths \
40+ --lora-r 8 \
41+ --lora-alpha 16 \
42+ --lora-fusion \
43+ --lora-target-modules linear_qkv linear_proj linear_fc1 linear_fc2
44+"
45+ 
46+GPT_ARGS="
47+ --use-mcore-models \
48+ --tensor-model-parallel-size ${TP} \
49+ --pipeline-model-parallel-size ${PP} \
50+ --num-layers 36 \
51+ --hidden-size 2048 \
52+ --ffn-hidden-size 11008 \
53+ --num-attention-heads 16 \
54+ --group-query-attention \
55+ --num-query-groups 2 \
56+ --tokenizer-type PretrainedFromHF \
57+ --tokenizer-name-or-path ${TOKENIZER_PATH} \
58+ --seq-length ${SEQ_LEN} \
59+ --max-position-embeddings ${SEQ_LEN} \
60+ --micro-batch-size ${MBS} \
61+ --global-batch-size ${GBS} \
62+ --make-vocab-size-divisible-by 1 \
63+ --padded-vocab-size 151936 \
64+ --rotary-base 1000000 \
65+ --train-iters 2000 \
66+ --lr 7.75e-7 \
67+ --min-lr 7.75e-8 \
68+ --weight-decay 1e-1 \
69+ --lr-decay-style cosine \
70+ --lr-warmup-fraction 0.01 \
71+ --clip-grad 1.0 \
72+ --adam-beta1 0.9 \
73+ --adam-beta2 0.95 \
74+ --add-qkv-bias \
75+ --disable-bias-linear \
76+ --attention-dropout 0.0 \
77+ --init-method-std 0.01 \
78+ --hidden-dropout 0.0 \
79+ --position-embedding-type rope \
80+ --normalization RMSNorm \
81+ --norm-epsilon 1e-06 \
82+ --swiglu \
83+ --use-distributed-optimizer \
84+ --use-flash-attn \
85+ --use-fused-rotary-pos-emb \
86+ --use-rotary-position-embeddings \
87+ --use-fused-swiglu \
88+ --use-fused-rmsnorm \
89+ --use-mc2 \
90+ --overlap-grad-reduce \
91+ --no-masked-softmax-fusion \
92+ --attention-softmax-in-fp32 \
93+ --initial-loss-scale 4096 \
94+ --no-gradient-accumulation-fusion \
95+ --no-load-optim \
96+ --no-load-rng \
97+ --seed 42 \
98+ --bf16
99+"
100+ 
101+DATA_ARGS="
102+ --data-path $DATA_PATH \
103+ --split 100,0,0
104+"
105+ 
106+OUTPUT_ARGS="
107+ --log-interval 1 \
108+ --save-interval 2000 \
109+ --eval-interval 2000 \
110+ --eval-iters 0 \
111+"
112+ 
113+torchrun $DISTRIBUTED_ARGS posttrain_gpt.py \
114+ $GPT_ARGS \
115+ $DATA_ARGS \
116+ $OUTPUT_ARGS \
117+ $TUNE_ARGS \
118+ --distributed-backend nccl \
119+ --load ${CKPT_LOAD_DIR} \
120+ --save ${CKPT_SAVE_DIR} \
121+ | tee logs/tune_mcore_qwen25_3b_4k_lora.log
Mexamples/mcore/qwen25/tune_qwen25_72b_4k_full_ptd.sh+1-0
@@ -44,6 +44,7 @@ TUNE_ARGS="
44 --is-instruction-dataset \44 --is-instruction-dataset \
45 --tokenizer-not-use-fast \45 --tokenizer-not-use-fast \
46 --prompt-type qwen \46 --prompt-type qwen \
47+ --padded-samples \
47 --variable-seq-lengths \48 --variable-seq-lengths \
48"49"
49 50 
Mexamples/mcore/qwen25/tune_qwen25_72b_4k_lora_ptd.sh+1-0
@@ -35,6 +35,7 @@ TUNE_ARGS="
35 --is-instruction-dataset \35 --is-instruction-dataset \
36 --tokenizer-not-use-fast \36 --tokenizer-not-use-fast \
37 --prompt-type qwen \37 --prompt-type qwen \
38+ --padded-samples \
38 --variable-seq-lengths \39 --variable-seq-lengths \
39 --lora-r 8 \40 --lora-r 8 \
40 --lora-alpha 16 \41 --lora-alpha 16 \
Mexamples/mcore/qwen25/tune_qwen25_7b_4k_full_ptd.sh+1-0
@@ -35,6 +35,7 @@ TUNE_ARGS="
35 --is-instruction-dataset \35 --is-instruction-dataset \
36 --tokenizer-not-use-fast \36 --tokenizer-not-use-fast \
37 --prompt-type qwen \37 --prompt-type qwen \
38+ --padded-samples \
38 --variable-seq-lengths \39 --variable-seq-lengths \
39"40"
40 41 
Mexamples/mcore/qwen25/tune_qwen25_7b_4k_lora_ptd.sh+1-0
@@ -36,6 +36,7 @@ TUNE_ARGS="
36 --tokenizer-not-use-fast \36 --tokenizer-not-use-fast \
37 --prompt-type qwen \37 --prompt-type qwen \
38 --variable-seq-lengths \38 --variable-seq-lengths \
39+ --padded-samples \
39 --lora-r 8 \40 --lora-r 8 \
40 --lora-alpha 16 \41 --lora-alpha 16 \
41 --lora-fusion \42 --lora-fusion \