新增gemma2-9b mcore全参微调Loss对齐脚本-MindSpeed-LLM-AtomGit

已合并

新增gemma2-9b mcore全参微调Loss对齐脚本 #1746

AtomGit-Bot创建于 2024年10月5日从refs/pull/1746/head 合入到 master

新增gemma2-9b mcore全参微调Loss对齐脚本 #1746

已合并

AtomGit-Bot创建于 2024年10月5日

从refs/pull/1746/head合入到master

共 3 个文件变更+194-0

Aexamples/mcore/gemma/chat_gemma2_9b_ptd.sh+68-0

@@ -0,0 +1,68 @@
	1	+#!/bin/bash
	2	+export CUDA_DEVICE_MAX_CONNECTIONS=1
	3	+
	4	+# Change for multinode config
	5	+NPUS_PER_NODE=8
	6	+MASTER_ADDR=localhost
	7	+MASTER_PORT=6001
	8	+NNODES=1
	9	+NODE_RANK=0
	10	+WORLD_SIZE=$(($NPUS_PER_NODE*$NNODES))
	11	+
	12	+# please fill these path configurations
	13	+TOKENIZER_PATH="your tokenizer directory path"
	14	+CHECKPOINT="your model directory path"
	15	+
	16	+DISTRIBUTED_ARGS="
	17	+ --nproc_per_node $NPUS_PER_NODE \
	18	+ --nnodes $NNODES \
	19	+ --node_rank $NODE_RANK \
	20	+ --master_addr $MASTER_ADDR \
	21	+ --master_port $MASTER_PORT"
	22	+
	23	+torchrun $DISTRIBUTED_ARGS inference.py \
	24	+ --tensor-model-parallel-size 8 \
	25	+ --pipeline-model-parallel-size 1 \
	26	+ --use-mcore-models \
	27	+ --gelu-tanh \
	28	+ --prompt-type gemma \
	29	+ --post-norm \
	30	+ --query-pre-attn-scalar 256 \
	31	+ --output-logit-softcapping 30.0 \
	32	+ --attn-logit-softcapping 50.0 \
	33	+ --interleave-sliding-window 4096 \
	34	+ --group-query-attention \
	35	+ --num-query-groups 8 \
	36	+ --load ${CHECKPOINT} \
	37	+ --num-layers 42 \
	38	+ --hidden-size 3584 \
	39	+ --kv-channels 256 \
	40	+ --ffn-hidden-size 14336 \
	41	+ --num-attention-heads 16 \
	42	+ --position-embedding-type rope \
	43	+ --seq-length 8192 \
	44	+ --max-position-embeddings 8192 \
	45	+ --max-new-tokens 256 \
	46	+ --micro-batch-size 1 \
	47	+ --tokenizer-type PretrainedFromHF \
	48	+ --tokenizer-name-or-path ${TOKENIZER_PATH} \
	49	+ --tokenizer-not-use-fast \
	50	+ --normalization RMSNorm \
	51	+ --add-rmsnorm-offset \
	52	+ --norm-epsilon 1e-06 \
	53	+ --input-embeds-norm \
	54	+ --disable-bias-linear \
	55	+ --hidden-dropout 0 \
	56	+ --attention-dropout 0 \
	57	+ --attention-softmax-in-fp32 \
	58	+ --no-load-optim \
	59	+ --no-load-rng \
	60	+ --no-masked-softmax-fusion \
	61	+ --no-gradient-accumulation-fusion \
	62	+ --exit-on-missing-checkpoint \
	63	+ --make-vocab-size-divisible-by 1 \
	64	+ --vocab-size 256000 \
	65	+ --bf16 \
	66	+ --seed 42 \
	67	+ \| tee logs/chat_gemma2_9b_ptd.log
	68	+

Aexamples/mcore/gemma2/data_convert_gemma2_instruction.sh+16-0

@@ -0,0 +1,16 @@
	1	+# 请根据 examples/README.md 下 “数据集准备及处理” 章节下载 Alpaca 数据集
	2	+# 请按照您的真实环境修改 set_env.sh 路径
	3	+source /usr/local/Ascend/ascend-toolkit/set_env.sh
	4	+mkdir ./finetune_dataset
	5	+
	6	+python ./preprocess_data.py \
	7	+ --input ./dataset/train-00000-of-00001.parquet \
	8	+ --tokenizer-name-or-path ./model_from_hf/gemma2_hf/ \
	9	+ --output-prefix ./finetune_dataset/alpaca \
	10	+ --workers 4 \
	11	+ --log-interval 1000 \
	12	+ --tokenizer-type PretrainedFromHF \
	13	+ --handler-name AlpacaStyleInstructionHandler \
	14	+ --prompt-type gemma \
	15	+ --map-keys '{"history":"histories"}'
	16	+ # --map-keys '{"prompt":"instruction","query":"input","response":"output"}' # 默认值，可不传

Aexamples/mcore/gemma2/tune_gemma2_9b_full_ptd.sh+110-0

@@ -0,0 +1,110 @@
	1	+#!/bin/bash
	2	+
	3	+export CUDA_DEVICE_MAX_CONNECTIONS=1
	4	+
	5	+NPUS_PER_NODE=8
	6	+MASTER_ADDR=localhost
	7	+MASTER_PORT=6014
	8	+NNODES=1
	9	+NODE_RANK=0
	10	+WORLD_SIZE=$(($NPUS_PER_NODE*$NNODES))
	11	+
	12	+CKPT_SAVE_DIR="your model save ckpt path"
	13	+DATA_PATH="your data path"
	14	+TOKENIZER_MODEL="your tokenizer path"
	15	+CKPT_LOAD_DIR="your model ckpt path"
	16	+
	17	+TP=1
	18	+PP=8
	19	+
	20	+DISTRIBUTED_ARGS="
	21	+ --nproc_per_node $NPUS_PER_NODE \
	22	+ --nnodes $NNODES \
	23	+ --node_rank $NODE_RANK \
	24	+ --master_addr $MASTER_ADDR \
	25	+ --master_port $MASTER_PORT
	26	+"
	27	+
	28	+GPT_ARGS="
	29	+ --tensor-model-parallel-size ${TP} \
	30	+ --pipeline-model-parallel-size ${PP} \
	31	+ --sequence-parallel \
	32	+ --use-mcore-models \
	33	+ --use-mc2 \
	34	+ --use-fused-rmsnorm \
	35	+ --use-fused-rotary-pos-emb \
	36	+ --gelu-tanh \
	37	+ --post-norm \
	38	+ --query-pre-attn-scalar 256 \
	39	+ --output-logit-softcapping 30.0 \
	40	+ --interleave-sliding-window 4096 \
	41	+ --num-layers 42 \
	42	+ --num-layer-list 5,5,5,5,5,5,5,7 \
	43	+ --hidden-size 3584 \
	44	+ --ffn-hidden-size 14336 \
	45	+ --num-attention-heads 16 \
	46	+ --tokenizer-type PretrainedFromHF \
	47	+ --tokenizer-name-or-path ${TOKENIZER_MODEL} \
	48	+ --seq-length 8192 \
	49	+ --max-position-embeddings 8192 \
	50	+ --micro-batch-size 1 \
	51	+ --global-batch-size 64 \
	52	+ --kv-channels 256 \
	53	+ --group-query-attention \
	54	+ --num-query-groups 8 \
	55	+ --make-vocab-size-divisible-by 1 \
	56	+ --lr 1e-6 \
	57	+ --train-iters 2000 \
	58	+ --lr-decay-style cosine \
	59	+ --disable-bias-linear \
	60	+ --attention-dropout 0.0 \
	61	+ --init-method-std 0.02 \
	62	+ --hidden-dropout 0.0 \
	63	+ --position-embedding-type rope \
	64	+ --normalization RMSNorm \
	65	+ --add-rmsnorm-offset \
	66	+ --norm-epsilon 1e-06 \
	67	+ --input-embeds-norm \
	68	+ --use-flash-attn \
	69	+ --use-distributed-optimizer \
	70	+ --no-masked-softmax-fusion \
	71	+ --attention-softmax-in-fp32 \
	72	+ --weight-decay 1e-1 \
	73	+ --clip-grad 1.0 \
	74	+ --adam-beta1 0.9 \
	75	+ --adam-beta2 0.999 \
	76	+ --initial-loss-scale 1 \
	77	+ --no-gradient-accumulation-fusion \
	78	+ --no-load-optim \
	79	+ --no-load-rng \
	80	+ --vocab-size 256000 \
	81	+ --bf16 \
	82	+ --finetune \
	83	+ --is-instruction-dataset \
	84	+ --tokenizer-padding-side right \
	85	+ --variable-seq-lengths \
	86	+ --tokenizer-not-use-fast \
	87	+ --prompt-type gemma \
	88	+ --rotary-base 10000.0
	89	+"
	90	+
	91	+DATA_ARGS="
	92	+ --data-path $DATA_PATH \
	93	+ --split 100,0,0
	94	+"
	95	+
	96	+OUTPUT_ARGS="
	97	+ --log-interval 1 \
	98	+ --save-interval 2000 \
	99	+ --eval-interval 1000 \
	100	+ --eval-iters 0 \
	101	+"
	102	+
	103	+torchrun $DISTRIBUTED_ARGS pretrain_gpt.py \
	104	+ $GPT_ARGS \
	105	+ $DATA_ARGS \
	106	+ $OUTPUT_ARGS \
	107	+ --distributed-backend nccl \
	108	+ --load ${CKPT_LOAD_DIR} \
	109	+ --save ${CKPT_SAVE_DIR} \
	110	+ \| tee logs/tune_gemma2_9b_full_ptd.log