Star213
324
代码
代码
Issues58
Pull Requests95
项目讨论
Wiki
分析
Star213
324
已合并
新增MiniCPM-2B微调 #1813
AtomGit-Bot创建于 2024年10月25日
新增MiniCPM-2B微调 #1813
已合并
AtomGit-Bot创建于 2024年10月25日
从refs/pull/1813/head合入到master
共 8 个文件变更+355-2
Aexamples/mcore/minicpm/data_convert_minicpm_instruction.sh+13-0
@@ -0,0 +1,13 @@
1+source /usr/local/Ascend/ascend-toolkit/set_env.sh
2+mkdir ./funetune_dataset
3+ 
4+python ./preprocess_data.py \
5+ --input ./dataset/train-00000-of-00042-d964455e17e96d5a.parquet \
6+ --tokenizer-name-or-path ./model_from_hf/MiniCPM-2B-sft-bf16 \
7+ --output-prefix ./finetune_dataset/alpaca \
8+ --workers 4 \
9+ --log-interval 1000 \
10+ --tokenizer-type PretrainedFromHF \
11+ --handler-name AlpacaStyleInstructionHandler \
12+ --prompt-type cpm \
13+ --overwrite-cache
Aexamples/mcore/minicpm/evaluate_minicpm_2b_lora_ptd.sh+59-0
@@ -0,0 +1,59 @@
1+#!/bin/bash
2+export CUDA_DEVICE_MAX_CONNECTIONS=1
3+export HCCL_CONNECT_TIMEOUT=1200
4+ 
5+# modify script model path and tokenizer path
6+CHECKPOINT="your lora ckpt path"
7+TOKENIZER_PATH="your tokenizer path"
8+ 
9+# configure task and data path
10+DATA_PATH="your eval data path"
11+TASK="your task name"
12+ 
13+# distributed config
14+MASTER_ADDR=localhost
15+MASTER_PORT=6011
16+NNODES=1
17+NODE_RANK=0
18+NPUS_PER_NODE=1
19+DISTRIBUTED_ARGS="--nproc_per_node $NPUS_PER_NODE --nnodes $NNODES --node_rank $NODE_RANK --master_addr $MASTER_ADDR --master_port $MASTER_PORT"
20+ 
21+# configure generation parameters
22+python -m torch.distributed.launch $DISTRIBUTED_ARGS evaluation.py \
23+ --task-data-path $DATA_PATH \
24+ --task $TASK\
25+ --seq-length 4096 \
26+ --max-new-tokens 1 \
27+ --evaluation-batch-size 1 \
28+ --max-position-embeddings 4096 \
29+ --tensor-model-parallel-size 1 \
30+ --pipeline-model-parallel-size 1 \
31+ --num-layers 40 \
32+ --hidden-size 2304 \
33+ --ffn-hidden-size 5760 \
34+ --num-attention-heads 36 \
35+ --swiglu \
36+ --disable-bias-linear \
37+ --load ${CHECKPOINT} \
38+ --normalization RMSNorm \
39+ --tokenizer-type PretrainedFromHF \
40+ --tokenizer-name-or-path ${TOKENIZER_PATH} \
41+ --tokenizer-not-use-fast \
42+ --bf16 \
43+ --micro-batch-size 1 \
44+ --use-fused-rmsnorm \
45+ --position-embedding-type rope \
46+ --exit-on-missing-checkpoint \
47+ --no-load-rng \
48+ --no-load-optim \
49+ --no-masked-softmax-fusion \
50+ --make-vocab-size-divisible-by 1 \
51+ --use-mcore-models \
52+ --scale-emb 12 \
53+ --dim-model-base 256 \
54+ --scale-depth 1.4 \
55+ --lora-r 8 \
56+ --lora-alpha 16 \
57+ --lora-fusion \
58+ --lora-target-modules linear_qkv linear_proj linear_fc1 linear_fc2 \
59+ --seed 42 | tee logs/evaluation_minicpm_2b_${TASK}.log
Aexamples/mcore/minicpm/generate_minicpm_2b_lora_ptd.sh+59-0
@@ -0,0 +1,59 @@
1+#!/bin/bash
2+ 
3+# The number of parameters is not aligned
4+export HCCL_CONNECT_TIMEOUT=1200
5+export CUDA_DEVICE_MAX_CONNECTIONS=1
6+ 
7+# please fill these path configurations
8+CHECKPOINT="your lora ckpt path"
9+TOKENIZER_PATH="your tokenizer path"
10+ 
11+# Change for multinode config
12+MASTER_ADDR=localhost
13+MASTER_PORT=6001
14+NNODES=1
15+NODE_RANK=0
16+NPUS_PER_NODE=1
17+WORLD_SIZE=$(($NPUS_PER_NODE*$NNODES))
18+ 
19+DISTRIBUTED_ARGS="--nproc_per_node $NPUS_PER_NODE --nnodes $NNODES --node_rank $NODE_RANK --master_addr $MASTER_ADDR --master_port $MASTER_PORT"
20+ 
21+python -m torch.distributed.launch $DISTRIBUTED_ARGS inference.py \
22+ --tensor-model-parallel-size 1 \
23+ --pipeline-model-parallel-size 1 \
24+ --num-layers 40 \
25+ --hidden-size 2304 \
26+ --ffn-hidden-size 5760 \
27+ --position-embedding-type rope \
28+ --norm-epsilon 1e-5 \
29+ --seq-length 4096 \
30+ --max-new-tokens 256 \
31+ --micro-batch-size 4 \
32+ --global-batch-size 16 \
33+ --num-attention-heads 36 \
34+ --max-position-embeddings 4096 \
35+ --swiglu \
36+ --load "${CHECKPOINT}" \
37+ --tokenizer-type PretrainedFromHF \
38+ --tokenizer-name-or-path "${TOKENIZER_PATH}" \
39+ --tokenizer-not-use-fast \
40+ --bf16 \
41+ --normalization RMSNorm \
42+ --disable-bias-linear \
43+ --attention-softmax-in-fp32 \
44+ --no-load-optim \
45+ --no-load-rng \
46+ --no-masked-softmax-fusion \
47+ --no-gradient-accumulation-fusion \
48+ --exit-on-missing-checkpoint \
49+ --make-vocab-size-divisible-by 1 \
50+ --use-mcore-models \
51+ --scale-emb 12 \
52+ --dim-model-base 256 \
53+ --scale-depth 1.4 \
54+ --lora-r 8 \
55+ --lora-alpha 16 \
56+ --lora-fusion \
57+ --lora-target-modules linear_qkv linear_proj linear_fc1 linear_fc2 \
58+ | tee logs/generate_minicpm_2b.log
59+ 
Aexamples/mcore/minicpm/tune_minicpm_2b_full_ptd.sh+104-0
@@ -0,0 +1,104 @@
1+#!/bin/bash
2+ 
3+export CUDA_DEVICE_MAX_CONNECTIONS=1
4+export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
5+ 
6+GPUS_PER_NODE=8
7+MASTER_ADDR=localhost
8+MASTER_PORT=6009
9+NNODES=1
10+NODE_RANK=0
11+WORLD_SIZE=$(($GPUS_PER_NODE*$NNODES))
12+ 
13+CKPT_SAVE_DIR="your model save ckpt path"
14+DATA_PATH="your data path"
15+TOKENIZER_PATH="your tokenizer path"
16+CKPT_LOAD_DIR="your model ckpt path"
17+TP=1
18+PP=1
19+ 
20+DISTRIBUTED_ARGS="
21+ --nproc_per_node $GPUS_PER_NODE \
22+ --nnodes $NNODES \
23+ --node_rank $NODE_RANK \
24+ --master_addr $MASTER_ADDR \
25+ --master_port $MASTER_PORT
26+"
27+ 
28+GPT_ARGS="
29+ --use-mcore-models \
30+ --tensor-model-parallel-size ${TP} \
31+ --pipeline-model-parallel-size ${PP} \
32+ --sequence-parallel \
33+ --num-layers 40 \
34+ --hidden-size 2304 \
35+ --ffn-hidden-size 5760 \
36+ --num-attention-heads 36 \
37+ --tokenizer-type PretrainedFromHF \
38+ --tokenizer-name-or-path "${TOKENIZER_PATH}" \
39+ --tokenizer-not-use-fast \
40+ --vocab-size 122753 \
41+ --seq-length 4096 \
42+ --max-position-embeddings 4096 \
43+ --micro-batch-size 2 \
44+ --global-batch-size 16 \
45+ --make-vocab-size-divisible-by 1 \
46+ --lr 1e-7 \
47+ --train-iters 2000 \
48+ --lr-decay-style constant \
49+ --disable-bias-linear \
50+ --attention-dropout 0.0 \
51+ --init-method-std 0.01 \
52+ --hidden-dropout 0.0 \
53+ --position-embedding-type rope \
54+ --normalization RMSNorm \
55+ --norm-epsilon 1e-5 \
56+ --use-fused-rmsnorm \
57+ --swiglu \
58+ --use-flash-attn \
59+ --no-masked-softmax-fusion \
60+ --attention-softmax-in-fp32 \
61+ --min-lr 1e-8 \
62+ --weight-decay 1e-1 \
63+ --lr-warmup-fraction 0.01 \
64+ --clip-grad 1.0 \
65+ --adam-beta1 0.9 \
66+ --initial-loss-scale 65536 \
67+ --adam-beta2 0.999 \
68+ --no-gradient-accumulation-fusion \
69+ --no-load-optim \
70+ --no-load-rng \
71+ --use-distributed-optimizer \
72+ --use-fused-swiglu \
73+ --use-fused-rotary-pos-emb \
74+ --overlap-grad-reduce \
75+ --scale-emb 12 \
76+ --dim-model-base 256 \
77+ --scale-depth 1.4 \
78+ --bf16 \
79+"
80+ 
81+DATA_ARGS="
82+ --data-path $DATA_PATH \
83+ --split 100,0,0 \
84+ --finetune \
85+ --is-instruction-dataset \
86+ --prompt-type cpm \
87+ --no-shuffle \
88+"
89+ 
90+OUTPUT_ARGS="
91+ --log-interval 1 \
92+ --save-interval 1000 \
93+ --eval-interval 1000 \
94+ --eval-iters 10 \
95+"
96+ 
97+torchrun $DISTRIBUTED_ARGS pretrain_gpt.py \
98+ $GPT_ARGS \
99+ $DATA_ARGS \
100+ $OUTPUT_ARGS \
101+ --distributed-backend nccl \
102+ --load $CKPT_LOAD_DIR \
103+ --save $CKPT_SAVE_DIR \
104+ | tee logs/tune_minicpm_2b_full_npu.log
Aexamples/mcore/minicpm/tune_minicpm_2b_lora_ptd.sh+109-0
@@ -0,0 +1,109 @@
1+#!/bin/bash
2+ 
3+export CUDA_DEVICE_MAX_CONNECTIONS=1
4+export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
5+ 
6+GPUS_PER_NODE=8
7+MASTER_ADDR=localhost
8+MASTER_PORT=6009
9+NNODES=1
10+NODE_RANK=0
11+WORLD_SIZE=$(($GPUS_PER_NODE*$NNODES))
12+ 
13+CKPT_SAVE_DIR="your model save ckpt path"
14+DATA_PATH="your data path"
15+TOKENIZER_PATH="your tokenizer path"
16+CKPT_LOAD_DIR="your model ckpt path"
17+TP=1
18+PP=1
19+ 
20+DISTRIBUTED_ARGS="
21+ --nproc_per_node $GPUS_PER_NODE \
22+ --nnodes $NNODES \
23+ --node_rank $NODE_RANK \
24+ --master_addr $MASTER_ADDR \
25+ --master_port $MASTER_PORT
26+"
27+ 
28+GPT_ARGS="
29+ --use-mcore-models \
30+ --tensor-model-parallel-size ${TP} \
31+ --pipeline-model-parallel-size ${PP} \
32+ --sequence-parallel \
33+ --num-layers 40 \
34+ --hidden-size 2304 \
35+ --ffn-hidden-size 5760 \
36+ --num-attention-heads 36 \
37+ --tokenizer-type PretrainedFromHF \
38+ --tokenizer-name-or-path "${TOKENIZER_PATH}" \
39+ --tokenizer-not-use-fast \
40+ --vocab-size 122753 \
41+ --seq-length 4096 \
42+ --max-position-embeddings 4096 \
43+ --micro-batch-size 1 \
44+ --global-batch-size 8 \
45+ --make-vocab-size-divisible-by 1 \
46+ --lr 1e-7 \
47+ --train-iters 2000 \
48+ --lr-decay-style constant \
49+ --disable-bias-linear \
50+ --attention-dropout 0.0 \
51+ --init-method-std 0.01 \
52+ --hidden-dropout 0.0 \
53+ --position-embedding-type rope \
54+ --normalization RMSNorm \
55+ --norm-epsilon 1e-5 \
56+ --use-fused-rmsnorm \
57+ --swiglu \
58+ --use-flash-attn \
59+ --no-masked-softmax-fusion \
60+ --attention-softmax-in-fp32 \
61+ --min-lr 1e-8 \
62+ --weight-decay 1e-1 \
63+ --lr-warmup-fraction 0.01 \
64+ --clip-grad 1.0 \
65+ --adam-beta1 0.9 \
66+ --initial-loss-scale 65536 \
67+ --adam-beta2 0.999 \
68+ --no-gradient-accumulation-fusion \
69+ --no-load-optim \
70+ --no-load-rng \
71+ --use-distributed-optimizer \
72+ --use-fused-swiglu \
73+ --use-fused-rotary-pos-emb \
74+ --overlap-grad-reduce \
75+ --variable-seq-lengths \
76+ --scale-emb 12 \
77+ --dim-model-base 256 \
78+ --scale-depth 1.4 \
79+ --lora-r 8 \
80+ --lora-alpha 16 \
81+ --lora-fusion \
82+ --lora-target-modules linear_qkv linear_proj linear_fc1 linear_fc2 \
83+ --bf16 \
84+"
85+ 
86+DATA_ARGS="
87+ --data-path $DATA_PATH \
88+ --split 100,0,0 \
89+ --finetune \
90+ --is-instruction-dataset \
91+ --prompt-type cpm \
92+ --no-shuffle \
93+"
94+ 
95+OUTPUT_ARGS="
96+ --log-interval 1 \
97+ --save-interval 1000 \
98+ --eval-interval 1000 \
99+ --eval-iters 10 \
100+"
101+ 
102+torchrun $DISTRIBUTED_ARGS pretrain_gpt.py \
103+ $GPT_ARGS \
104+ $DATA_ARGS \
105+ $OUTPUT_ARGS \
106+ --distributed-backend nccl \
107+ --load $CKPT_LOAD_DIR \
108+ --save $CKPT_SAVE_DIR \
109+ | tee logs/tune_minicpm_2b_lora_npu.log
Mmodellink/tasks/preprocess/templates.py+7-0
@@ -549,6 +549,13 @@ _register_template(
549)549)
550 550 
551 551 
552+_register_template(
553+ name="cpm",
554+ format_user=StringFormatter(slots=["<用户>{{content}}<AI>"]),
555+ format_prefix=EmptyFormatter(slots=[{"bos_token"}]),
556+)
557+ 
558+ 
552_register_template(559_register_template(
553 name="default",560 name="default",
554 format_user=StringFormatter(slots=["Human: {{content}}\nAssistant:"]),561 format_user=StringFormatter(slots=["Human: {{content}}\nAssistant:"]),
Mmodellink/training/arguments.py+2-1
@@ -494,7 +494,8 @@ def _add_training_args(parser):
494 help='Setting jit compile mode to True')494 help='Setting jit compile mode to True')
495 group.add_argument('--prompt-type', type=str, default=None,495 group.add_argument('--prompt-type', type=str, default=None,
496 choices=['default', 'empty', 'chatglm2', 'chatglm3', 'chatglm3_system', 'glm4', 'chatml',496 choices=['default', 'empty', 'chatglm2', 'chatglm3', 'chatglm3_system', 'glm4', 'chatml',
497- 'chatml_de', 'qwen', 'llama3', 'llama2', 'mistral', 'mixtral', 'gemma', 'alpaca', 'deepseek2', 'deepseek2-lite'],497+ 'chatml_de', 'qwen', 'llama3', 'llama2', 'mistral', 'mixtral', 'gemma', 'alpaca',
498+ 'deepseek2', 'deepseek2-lite', 'cpm'],
498 help='Which template to use for constructing prompts in training/inference.' 'e.g., "qwen"')499 help='Which template to use for constructing prompts in training/inference.' 'e.g., "qwen"')
499 group.add_argument('--pad-to-multiple-of', type=int, default=8,500 group.add_argument('--pad-to-multiple-of', type=int, default=8,
500 help='Used for Padding multiple in finetune. The default is 8.')501 help='Used for Padding multiple in finetune. The default is 8.')
Mpreprocess_data.py+2-1
@@ -106,7 +106,8 @@ def add_data_args(parser):
106 # LlamaFactory106 # LlamaFactory
107 group.add_argument('--prompt-type', type=str, default=None,107 group.add_argument('--prompt-type', type=str, default=None,
108 choices=['default', 'empty', 'chatglm2', 'chatglm3', 'chatglm3_system', 'glm4', 'chatml',108 choices=['default', 'empty', 'chatglm2', 'chatglm3', 'chatglm3_system', 'glm4', 'chatml',
109- 'chatml_de', 'qwen', 'llama3', 'llama2', 'mistral', 'mixtral', 'gemma', 'alpaca', 'deepseek2', 'deepseek2-lite'],109+ 'chatml_de', 'qwen', 'llama3', 'llama2', 'mistral', 'mixtral', 'gemma', 'alpaca',
110+ 'deepseek2', 'deepseek2-lite', 'cpm'],
110 help='Which template to use for constructing prompts in training.'111 help='Which template to use for constructing prompts in training.'
111 'e.g., "qwen"')112 'e.g., "qwen"')
112 group.add_argument("--interleave-probs", default=None,113 group.add_argument("--interleave-probs", default=None,