[mindspore][master]add mixtral sh-MindSpeed-LLM-AtomGit

Mexamples/mindspore/mixtral/pretrain_mixtral_8x22b_32k_ms.sh+1-1

	@@ -127,5 +127,5 @@ msrun $DISTRIBUTED_ARGS pretrain_gpt.py \
127	$DATA_ARGS \	127	$DATA_ARGS \
128	$OUTPUT_ARGS \	128	$OUTPUT_ARGS \
129	--distributed-backend nccl \	129	--distributed-backend nccl \
130	- --ai-framework mindspore \	130	+ --ai-framework mindspore \
131	\| tee logs/train_mcore_mixtral_8x22b_32k_ptd.log	131	\| tee logs/train_mcore_mixtral_8x22b_32k_ptd.log

Aexamples/mindspore/mixtral/pretrain_mixtral_8x7b_ms.sh+127-0

@@ -0,0 +1,127 @@
	1	+#!/bin/bash
	2	+export CUDA_DEVICE_MAX_CONNECTIONS=1
	3	+export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
	4	+
	5	+NPUS_PER_NODE=8
	6	+MASTER_ADDR="your master node IP"
	7	+MASTER_PORT=6000
	8	+NNODES=8
	9	+NODE_RANK=0
	10	+WORLD_SIZE=$(($NPUS_PER_NODE * $NNODES))
	11	+
	12	+DISTRIBUTED_ARGS="
	13	+ --master_addr $MASTER_ADDR \
	14	+ --node_rank $NODE_RANK \
	15	+ --worker_num $WORLD_SIZE \
	16	+ --local_worker_num $NPUS_PER_NODE \
	17	+ --master_port $MASTER_PORT \
	18	+ --log_dir=msrun_log \
	19	+ --join=False \
	20	+ --cluster_time_out=300 \
	21	+ --bind_core=True \
	22	+"
	23	+
	24	+
	25	+echo "NODE_RANK ${NODE_RANK}"
	26	+
	27	+DATA_PATH="your data path"
	28	+TOKENIZER_MODEL="your tokenizer path"
	29	+CKPT_SAVE_DIR="your model save ckpt path"
	30	+CKPT_LOAD_DIR="your model ckpt path"
	31	+
	32	+TP=8
	33	+PP=1
	34	+EP=1
	35	+CP=4
	36	+CP_TYPE='megatron_cp_algo'
	37	+NUM_LAYERS=32
	38	+
	39	+MOE_ARGS="
	40	+ --num-experts 8 \
	41	+ --expert-model-parallel-size ${EP} \
	42	+ --moe-router-topk 2 \
	43	+ --moe-router-load-balancing-type aux_loss \
	44	+ --moe-aux-loss-coeff 0.02 \
	45	+ --moe-permutation-async-comm \
	46	+ --moe-token-dispatcher-type alltoall_seq \
	47	+ --moe-grouped-gemm \
	48	+ --moe-layer-freq -1 \
	49	+ --first-k-dense-replace -1 \
	50	+ --use-cp-send-recv-overlap \
	51	+"
	52	+
	53	+GPT_ARGS="
	54	+ --use-mcore-models \
	55	+ --disable-bias-linear \
	56	+ --seq-length 32768 \
	57	+ --max-position-embeddings 32768 \
	58	+ --num-layers ${NUM_LAYERS} \
	59	+ --hidden-size 4096 \
	60	+ --ffn-hidden-size 14336 \
	61	+ --num-attention-heads 32 \
	62	+ --init-method-std 0.01 \
	63	+ --attention-dropout 0.0 \
	64	+ --hidden-dropout 0.0 \
	65	+ --normalization RMSNorm \
	66	+ --position-embedding-type rope \
	67	+ --swiglu \
	68	+ --untie-embeddings-and-output-weights \
	69	+ --group-query-attention \
	70	+ --num-query-groups 8 \
	71	+ --vocab-size 32000 \
	72	+ --rotary-base 1000000 \
	73	+ --no-masked-softmax-fusion \
	74	+ --use-fused-rotary-pos-emb \
	75	+ --use-flash-attn \
	76	+ --use-fused-swiglu \
	77	+ --use-fused-rmsnorm \
	78	+ --no-check-for-nan-in-loss-and-grad \
	79	+ --overlap-grad-reduce \
	80	+ --overlap-param-gather \
	81	+ --make-vocab-size-divisible-by 1 \
	82	+ --tensor-model-parallel-size ${TP} \
	83	+ --pipeline-model-parallel-size ${PP} \
	84	+ --sequence-parallel \
	85	+ --use-distributed-optimizer \
	86	+ --context-parallel-size ${CP} \
	87	+ --context-parallel-algo ${CP_TYPE} \
	88	+ --tokenizer-type Llama2Tokenizer \
	89	+ --tokenizer-model ${TOKENIZER_MODEL} \
	90	+ --load ${CKPT_LOAD_DIR} \
	91	+ --save ${CKPT_SAVE_DIR} \
	92	+ --micro-batch-size 1 \
	93	+ --global-batch-size 32 \
	94	+ --lr 1e-5 \
	95	+ --train-iters 2000 \
	96	+ --lr-decay-iters 1280 \
	97	+ --lr-decay-style cosine \
	98	+ --min-lr 1.0e-6 \
	99	+ --weight-decay 0.1 \
	100	+ --lr-warmup-iters 2 \
	101	+ --clip-grad 1.0 \
	102	+ --bf16 \
	103	+ --no-load-optim \
	104	+ --no-load-rng \
	105	+ --no-shared-storage \
	106	+"
	107	+
	108	+DATA_ARGS="
	109	+ --data-path $DATA_PATH \
	110	+ --split 99990,8,2 \
	111	+"
	112	+
	113	+OUTPUT_ARGS="
	114	+ --log-interval 1 \
	115	+ --save-interval 2000 \
	116	+ --eval-interval 5001 \
	117	+ --eval-iters 100 \
	118	+"
	119	+
	120	+msrun $DISTRIBUTED_ARGS pretrain_gpt.py \
	121	+ $MOE_ARGS \
	122	+ $GPT_ARGS \
	123	+ $DATA_ARGS \
	124	+ $OUTPUT_ARGS \
	125	+ --distributed-backend nccl \
	126	+ --ai-framework mindspore \
	127	+ \| tee logs/train_mixtral_8x7b_ptd.log

Aexamples/mindspore/mixtral/tune_mixtral_8x22b_qlora_sh.sh+136-0

@@ -0,0 +1,136 @@
	1	+#!/bin/bash
	2	+
	3	+export CUDA_DEVICE_MAX_CONNECTIONS=1
	4	+export HCCL_DETERMINISTIC=True
	5	+
	6	+GPUS_PER_NODE=2
	7	+MASTER_ADDR=localhost
	8	+MASTER_PORT=6005
	9	+NNODES=1
	10	+NODE_RANK=0
	11	+WORLD_SIZE=$(($GPUS_PER_NODE * $NNODES))
	12	+
	13	+DISTRIBUTED_ARGS="
	14	+ --master_addr $MASTER_ADDR \
	15	+ --node_rank $NODE_RANK \
	16	+ --worker_num $WORLD_SIZE \
	17	+ --local_worker_num $GPUS_PER_NODE \
	18	+ --master_port $MASTER_PORT \
	19	+ --log_dir=msrun_log \
	20	+ --join=False \
	21	+ --cluster_time_out=300 \
	22	+ --bind_core=True \
	23	+"
	24	+
	25	+CKPT_SAVE_DIR="your model save ckpt path"
	26	+DATA_PATH="your data path"
	27	+TOKENIZER_MODEL="your tokenizer path"
	28	+CKPT_LOAD_DIR="your model ckpt path"
	29	+
	30	+TP=2
	31	+PP=1
	32	+EP=1
	33	+NUM_LAYERS=56
	34	+SEQ_LEN=32768
	35	+
	36	+MOE_ARGS="
	37	+ --num-experts 8 \
	38	+ --expert-model-parallel-size ${EP} \
	39	+ --moe-router-topk 2 \
	40	+ --moe-router-load-balancing-type aux_loss \
	41	+ --moe-token-dispatcher-type alltoall_seq \
	42	+ --moe-aux-loss-coeff 0.001 \
	43	+ --moe-permutation-async-comm \
	44	+ --moe-layer-freq -1 \
	45	+ --first-k-dense-replace -1 \
	46	+"
	47	+
	48	+GPT_ARGS="
	49	+ --use-mcore-models \
	50	+ --disable-bias-linear \
	51	+ --seq-length ${SEQ_LEN} \
	52	+ --max-position-embeddings ${SEQ_LEN} \
	53	+ --num-layers ${NUM_LAYERS} \
	54	+ --hidden-size 6144 \
	55	+ --ffn-hidden-size 16384 \
	56	+ --num-attention-heads 48 \
	57	+ --init-method-std 0.01 \
	58	+ --attention-dropout 0.0 \
	59	+ --hidden-dropout 0.0 \
	60	+ --normalization RMSNorm \
	61	+ --position-embedding-type rope \
	62	+ --swiglu \
	63	+ --untie-embeddings-and-output-weights \
	64	+ --group-query-attention \
	65	+ --num-query-groups 8 \
	66	+ --vocab-size 32000 \
	67	+ --rotary-base 1000000 \
	68	+ --no-masked-softmax-fusion \
	69	+ --use-fused-rotary-pos-emb \
	70	+ --use-flash-attn \
	71	+ --use-fused-swiglu \
	72	+ --use-fused-rmsnorm \
	73	+ --no-check-for-nan-in-loss-and-grad \
	74	+ --overlap-grad-reduce \
	75	+ --overlap-param-gather \
	76	+ --make-vocab-size-divisible-by 1 \
	77	+ --tensor-model-parallel-size ${TP} \
	78	+ --pipeline-model-parallel-size ${PP} \
	79	+ --sequence-parallel \
	80	+ --variable-seq-lengths \
	81	+ --use-distributed-optimizer \
	82	+ --load ${CKPT_LOAD_DIR} \
	83	+ --save ${CKPT_SAVE_DIR} \
	84	+ --tokenizer-type PretrainedFromHF \
	85	+ --tokenizer-name-or-path ${TOKENIZER_MODEL} \
	86	+ --lr-warmup-fraction 0.0 \
	87	+ --micro-batch-size 1 \
	88	+ --global-batch-size 1 \
	89	+ --lr 2e-7 \
	90	+ --train-iters 2000 \
	91	+ --lr-decay-style cosine \
	92	+ --weight-decay 0 \
	93	+ --no-gradient-accumulation-fusion \
	94	+ --no-load-optim \
	95	+ --no-load-rng \
	96	+ --no-shared-storage \
	97	+ --prompt-type mixtral \
	98	+ --adam-beta1 0.9 \
	99	+ --adam-beta2 0.999 \
	100	+ --seed 42 \
	101	+ --bf16 \
	102	+ --exit-on-missing-checkpoint
	103	+"
	104	+
	105	+FINETUNE_ARGS="
	106	+ --finetune \
	107	+ --stage sft \
	108	+ --is-instruction-dataset \
	109	+ --lora-r 8 \
	110	+ --lora-alpha 16 \
	111	+ --qlora \
	112	+ --lora-target-modules linear_qkv linear_proj linear_fc1 linear_fc2 \
	113	+"
	114	+
	115	+DATA_ARGS="
	116	+ --data-path ${DATA_PATH} \
	117	+ --split 100,0,0 \
	118	+"
	119	+
	120	+OUTPUT_ARGS="
	121	+ --log-interval 1 \
	122	+ --save-interval 2000 \
	123	+ --eval-interval 2000 \
	124	+ --eval-iters 1 \
	125	+"
	126	+
	127	+msrun $DISTRIBUTED_ARGS posttrain_gpt.py \
	128	+ $MOE_ARGS \
	129	+ $GPT_ARGS \
	130	+ $DATA_ARGS \
	131	+ $OUTPUT_ARGS \
	132	+ $FINETUNE_ARGS \
	133	+ --log-throughput \
	134	+ --distributed-backend nccl \
	135	+ --ai-framework mindspore \
	136	+ \| tee logs/tune_mixtral_8x22b_qlora_ptd.log