新增DeepSeek-V2.5 8k/32k预训练-MindSpeed-LLM-AtomGit

MREADME.md+0-1

	@@ -51,7 +51,6 @@ MindSpeed-LLM当前已支持和预置大模型列表：
51	- Mamba2: [<u>Transformers are SSMs</u>](https://arxiv.org/pdf/2405.21060)	51	- Mamba2: [<u>Transformers are SSMs</u>](https://arxiv.org/pdf/2405.21060)
52	- Mamba-Hybird: [<u>An Empirical Study of Mamba-based Language Models</u>](https://arxiv.org/pdf/2406.07887)	52	- Mamba-Hybird: [<u>An Empirical Study of Mamba-based Language Models</u>](https://arxiv.org/pdf/2406.07887)
53	- DeepSeek-V2: [236B](https://huggingface.co/deepseek-ai/DeepSeek-V2)	53	- DeepSeek-V2: [236B](https://huggingface.co/deepseek-ai/DeepSeek-V2)
54	-- DeepSeek-V2.5: [236B](https://huggingface.co/deepseek-ai/DeepSeek-V2.5)
55	- QWen2.5: [7B](https://huggingface.co/Qwen/Qwen2.5-7B), [14B](https://huggingface.co/Qwen/Qwen2.5-14B), [32B](https://huggingface.co/Qwen/Qwen2.5-32B), [72B](https://huggingface.co/Qwen/Qwen2.5-72B)	54	- QWen2.5: [7B](https://huggingface.co/Qwen/Qwen2.5-7B), [14B](https://huggingface.co/Qwen/Qwen2.5-14B), [32B](https://huggingface.co/Qwen/Qwen2.5-32B), [72B](https://huggingface.co/Qwen/Qwen2.5-72B)
56	- InternLM2.5: [1.8B](https://huggingface.co/internlm/internlm2_5-1_8b), [7B](https://huggingface.co/internlm/internlm2_5-7b), [20B](https://huggingface.co/internlm/internlm2_5-20b)	55	- InternLM2.5: [1.8B](https://huggingface.co/internlm/internlm2_5-1_8b), [7B](https://huggingface.co/internlm/internlm2_5-7b), [20B](https://huggingface.co/internlm/internlm2_5-20b)
57	- MiniCPM3: [4B](https://huggingface.co/openbmb/MiniCPM3-4B)	56	- MiniCPM3: [4B](https://huggingface.co/openbmb/MiniCPM3-4B)

Mdocs/models/evaluation.md+1-1

	@@ -52,7 +52,7 @@ MindSpeed-LLM 已支持的大模型评估数据统计如下：
52	\| CodeQWen2.5-7B \| Human. \| 66.5% \| [61.6%](https://modelscope.cn/models/Qwen/Qwen2.5-Coder-7B) \| Qwen2.5-Math-7B \|MMLU-STEM\| 65.1% \| [67.8%](https://github.com/QwenLM/Qwen2.5-Math/tree/main/) \|	52	\| CodeQWen2.5-7B \| Human. \| 66.5% \| [61.6%](https://modelscope.cn/models/Qwen/Qwen2.5-Coder-7B) \| Qwen2.5-Math-7B \|MMLU-STEM\| 65.1% \| [67.8%](https://github.com/QwenLM/Qwen2.5-Math/tree/main/) \|
53	\| Qwen2.5-Math-72B \|MMLU-STEM\| 83.7% \| [82.8%](https://github.com/QwenLM/Qwen2.5-Math/tree/main/) \| MiniCPM3-4B \| MMLU \| 63.7% \| 64.6% \|	53	\| Qwen2.5-Math-72B \|MMLU-STEM\| 83.7% \| [82.8%](https://github.com/QwenLM/Qwen2.5-Math/tree/main/) \| MiniCPM3-4B \| MMLU \| 63.7% \| 64.6% \|
54	\| Phi-3.5-mini-instruct \| MMLU \| 64.39% \| 64.34% \| Phi-3.5-MoE-instruct \| MMLU \| 78.5% \| [78.9%](https://huggingface.co/microsoft/Phi-3.5-MoE-instruct) \|	54	\| Phi-3.5-mini-instruct \| MMLU \| 64.39% \| 64.34% \| Phi-3.5-MoE-instruct \| MMLU \| 78.5% \| [78.9%](https://huggingface.co/microsoft/Phi-3.5-MoE-instruct) \|
55	-\| DeepSeek-Math-7B \|MMLU-STEM\| 56.5% \| [56.5%](https://github.com/deepseek-ai/DeepSeek-Math) \| \| \| \| \|	55	+\| DeepSeek-Math-7B \|MMLU-STEM\| 56.5% \| [56.5%](https://github.com/deepseek-ai/DeepSeek-Math) \| DeepSeek-V2.5 \| MMLU \| 79.3% \| [80.6%](https://github.com/deepseek-ai/DeepSeek-V3) \|
56		56
57	具体的权重转换功能命令介绍见 [MindSpeed-LLM 大模型使用指南分布式评估](../USER_GUIDE.md/#大模型分布式评估)章节.	57	具体的权重转换功能命令介绍见 [MindSpeed-LLM 大模型使用指南分布式评估](../USER_GUIDE.md/#大模型分布式评估)章节.
58		58

Mdocs/models/pretrain.md+9-0

	@@ -688,6 +688,15 @@
688	<td>【昇腾】</td>	688	<td>【昇腾】</td>
689	<td>【Test】</td>	689	<td>【Test】</td>
690	</tr>	690	</tr>
		691	+ <tr>
		692	+ <td rowspan="1"><a href="https://huggingface.co/deepseek-ai/DeepSeek-V2.5">DeepSeek-V2.5</a></td>
		693	+ <td><a href="https://huggingface.co/deepseek-ai/DeepSeek-V2.5/tree/main">236B</a></td>
		694	+ <td> 8K </td>
		695	+ <th>Mcore</th>
		696	+ <td> 20x8 </td>
		697	+ <td>【NAIE】</td>
		698	+ <td>【Test】</td>
		699	+ </tr>
691	<tr>	700	<tr>
692	<td rowspan="2"><a href="https://github.com/OpenBMB/MiniCPM">MiniCPM</a></td>	701	<td rowspan="2"><a href="https://github.com/OpenBMB/MiniCPM">MiniCPM</a></td>
693	<td> <a href="https://huggingface.co/openbmb/MiniCPM-2B-sft-bf16/tree/main">2B</a> </td>	702	<td> <a href="https://huggingface.co/openbmb/MiniCPM-2B-sft-bf16/tree/main">2B</a> </td>

Mexamples/mcore/deepseek2/ckpt_convert_deepseek2_hf2mcore.sh+1-1

	@@ -15,5 +15,5 @@ python convert_ckpt.py \
15	--target-expert-parallel-size 8 \	15	--target-expert-parallel-size 8 \
16	--load-dir ./model_from_hf/deepseek2-hf/ \	16	--load-dir ./model_from_hf/deepseek2-hf/ \
17	--save-dir ./model_weights/deepseek2-mcore/ \	17	--save-dir ./model_weights/deepseek2-mcore/ \
18	- --tokenizer-model ./model_from_hf/deepseek2-hf/	18	+ --tokenizer-model ./model_from_hf/deepseek2-hf/ \
19	--spec mindspeed_llm.tasks.models.spec.deepseek_spec layer_spec	19	--spec mindspeed_llm.tasks.models.spec.deepseek_spec layer_spec

Aexamples/mcore/deepseek25/ckpt_convert_deepseek25_hf2mcore.sh+22-0

@@ -0,0 +1,22 @@
	1	+# 修改 ascend-toolkit 路径
	2	+source /usr/local/Ascend/ascend-toolkit/set_env.sh
	3	+
	4	+export CUDA_DEVICE_MAX_CONNECTIONS=1
	5	+
	6	+# 权重格式转换，设置需要的并行策略
	7	+# 使用内存至少2T的主机来转换本权重
	8	+python convert_ckpt.py \
	9	+ --use-mcore-models \
	10	+ --moe-grouped-gemm \
	11	+ --model-type-hf deepseek2 \
	12	+ --model-type GPT \
	13	+ --load-model-type hf \
	14	+ --save-model-type mg \
	15	+ --params-dtype bf16 \
	16	+ --target-tensor-parallel-size 1 \
	17	+ --target-pipeline-parallel-size 2 \
	18	+ --target-expert-parallel-size 8 \
	19	+ --load-dir ./model_from_hf/deepseek25-hf/ \
	20	+ --save-dir ./model_weights/deepseek25-mcore/ \
	21	+ --tokenizer-model ./model_from_hf/deepseek25-hf/ \
	22	+ --spec mindspeed_llm.tasks.models.spec.deepseek_spec layer_spec

Aexamples/mcore/deepseek25/ckpt_convert_deepseek25_mcore2hf.sh+21-0

@@ -0,0 +1,21 @@
	1	+# 修改 ascend-toolkit 路径
	2	+source /usr/local/Ascend/ascend-toolkit/set_env.sh
	3	+
	4	+export CUDA_DEVICE_MAX_CONNECTIONS=1
	5	+
	6	+# 权重格式转换
	7	+# 使用内存至少2T的主机来转换本权重
	8	+python convert_ckpt.py \
	9	+ --use-mcore-models \
	10	+ --moe-grouped-gemm \
	11	+ --model-type-hf deepseek2 \
	12	+ --model-type GPT \
	13	+ --load-model-type mg \
	14	+ --save-model-type hf \
	15	+ --params-dtype bf16 \
	16	+ --target-tensor-parallel-size 1 \
	17	+ --target-pipeline-parallel-size 1 \
	18	+ --target-expert-parallel-size 1 \
	19	+ --load-dir ./model_weights/deepseek25-mcore/ \
	20	+ --save-dir ./model_from_hf/deepseek25-hf/ \
	21	+ --spec mindspeed_llm.tasks.models.spec.deepseek_spec layer_spec

Aexamples/mcore/deepseek25/data_convert_deepseek25_pretrain.sh+14-0

@@ -0,0 +1,14 @@
	1	+# 请根据 examples/README.md 下 “数据集准备及处理” 章节下载 Enwiki 数据集（一般取第一条即可）
	2	+# 请按照您的真实环境修改 set_env.sh 路径
	3	+source /usr/local/Ascend/ascend-toolkit/set_env.sh
	4	+mkdir ./dataset
	5	+
	6	+python ./preprocess_data.py \
	7	+ --input ./dataset/train-00000-of-00042-d964455e17e96d5a.parquet \
	8	+ --tokenizer-name-or-path ./model_from_hf/deepseek25-hf/ \
	9	+ --tokenizer-type PretrainedFromHF \
	10	+ --handler-name GeneralPretrainHandler \
	11	+ --output-prefix ./dataset/enwiki \
	12	+ --json-keys text \
	13	+ --workers 4 \
	14	+ --log-interval 1000

Aexamples/mcore/deepseek25/evaluation_deepseek25_236b_ptd.sh+129-0

@@ -0,0 +1,129 @@
	1	+#!/bin/bash
	2	+
	3	+export CUDA_DEVICE_MAX_CONNECTIONS=1
	4	+export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
	5	+export HCCL_CONNECT_TIMEOUT=3600
	6	+
	7	+GPUS_PER_NODE=8
	8	+MASTER_ADDR=localhost #主节点IP
	9	+MASTER_PORT=6000
	10	+NNODES=2
	11	+NODE_RANK=0
	12	+WORLD_SIZE=$(($GPUS_PER_NODE*$NNODES))
	13	+
	14	+TASK="mmlu"
	15	+CHECKPOINT="Your ckpt file path"
	16	+TOKENIZER_PATH="Your vocab file path"
	17	+DATA_PATH="Your data path (such as ./mmlu/test/)"
	18	+
	19	+TP=1
	20	+PP=2
	21	+EP=8
	22	+CP=1
	23	+CP_TYPE='ulysses_cp_algo'
	24	+NUM_LAYERS=60
	25	+SEQ_LEN=8192
	26	+
	27	+DISTRIBUTED_ARGS="
	28	+ --nproc_per_node $GPUS_PER_NODE \
	29	+ --nnodes $NNODES \
	30	+ --node_rank $NODE_RANK \
	31	+ --master_addr $MASTER_ADDR \
	32	+ --master_port $MASTER_PORT
	33	+"
	34	+
	35	+MLA_ARGS="
	36	+ --multi-head-latent-attention \
	37	+ --qk-rope-head-dim 64 \
	38	+ --qk-nope-head-dim 128 \
	39	+ --q-lora-rank 1536 \
	40	+ --kv-lora-rank 512 \
	41	+ --v-head-dim 128 \
	42	+ --qk-layernorm
	43	+"
	44	+
	45	+MOE_ARGS="
	46	+ --moe-permutation-async-comm \
	47	+ --moe-grouped-gemm \
	48	+ --moe-token-dispatcher-type alltoall \
	49	+ --use-fused-moe-token-permute-and-unpermute \
	50	+ --first-k-dense-replace 1 \
	51	+ --moe-layer-freq 1 \
	52	+ --n-shared-experts 2 \
	53	+ --num-experts 160 \
	54	+ --moe-router-topk 6 \
	55	+ --moe-intermediate-size 1536 \
	56	+ --moe-router-load-balancing-type group_limited_greedy \
	57	+ --topk-group 3 \
	58	+ --moe-aux-loss-coeff 0.003 \
	59	+ --moe-device-level-aux-loss-coeff 0.05 \
	60	+ --moe-comm-aux-loss-coeff 0.02 \
	61	+ --routed-scaling-factor 16.0 \
	62	+ --seq-aux
	63	+"
	64	+
	65	+ROPE_ARGS="
	66	+ --rope-scaling-beta-fast 32 \
	67	+ --rope-scaling-beta-slow 1 \
	68	+ --rope-scaling-factor 40 \
	69	+ --rope-scaling-mscale 1.0 \
	70	+ --rope-scaling-mscale-all-dim 1.0 \
	71	+ --rope-scaling-original-max-position-embeddings 4096 \
	72	+ --rope-scaling-type yarn
	73	+"
	74	+
	75	+GPT_ARGS="
	76	+ --spec mindspeed_llm.tasks.models.spec.deepseek_spec layer_spec \
	77	+ --use-distributed-optimizer \
	78	+ --reuse-fp32-param \
	79	+ --use-flash-attn \
	80	+ --shape-order BNSD \
	81	+ --use-mcore-models \
	82	+ --tensor-model-parallel-size ${TP} \
	83	+ --pipeline-model-parallel-size ${PP} \
	84	+ --expert-model-parallel-size ${EP} \
	85	+ --context-parallel-size ${CP} \
	86	+ --context-parallel-algo ${CP_TYPE} \
	87	+ --num-layers ${NUM_LAYERS} \
	88	+ --hidden-size 5120 \
	89	+ --ffn-hidden-size 12288 \
	90	+ --num-attention-heads 128 \
	91	+ --tokenizer-type PretrainedFromHF \
	92	+ --tokenizer-name-or-path ${TOKENIZER_PATH} \
	93	+ --seq-length ${SEQ_LEN} \
	94	+ --max-position-embeddings 163840 \
	95	+ --micro-batch-size 1 \
	96	+ --make-vocab-size-divisible-by 1 \
	97	+ --untie-embeddings-and-output-weights \
	98	+ --disable-bias-linear \
	99	+ --attention-dropout 0.0 \
	100	+ --init-method-std 0.02 \
	101	+ --hidden-dropout 0.0 \
	102	+ --position-embedding-type rope \
	103	+ --normalization RMSNorm \
	104	+ --use-fused-rotary-pos-emb \
	105	+ --use-rotary-position-embeddings \
	106	+ --use-fused-swiglu \
	107	+ --use-fused-rmsnorm \
	108	+ --swiglu \
	109	+ --no-masked-softmax-fusion \
	110	+ --attention-softmax-in-fp32 \
	111	+ --vocab-size 102400 \
	112	+ --padded-vocab-size 102400 \
	113	+ --rotary-base 10000 \
	114	+ --no-gradient-accumulation-fusion \
	115	+ --norm-epsilon 1e-6 \
	116	+ --max-new-tokens 1 \
	117	+ --task-data-path ${DATA_PATH} \
	118	+ --task ${TASK} \
	119	+ --bf16
	120	+"
	121	+
	122	+torchrun $DISTRIBUTED_ARGS evaluation.py \
	123	+ $GPT_ARGS \
	124	+ $MLA_ARGS \
	125	+ $ROPE_ARGS \
	126	+ $MOE_ARGS \
	127	+ --load ${CHECKPOINT} \
	128	+ --distributed-backend nccl \
	129	+ \| tee logs/evaluation_deepseek25_236b_8k_ptd.log

Aexamples/mcore/deepseek25/generate_deepseek25_236b_ptd.sh+126-0

@@ -0,0 +1,126 @@
	1	+#!/bin/bash
	2	+
	3	+export CUDA_DEVICE_MAX_CONNECTIONS=1
	4	+export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
	5	+export HCCL_CONNECT_TIMEOUT=3600
	6	+
	7	+GPUS_PER_NODE=8
	8	+MASTER_ADDR=localhost #主节点IP
	9	+MASTER_PORT=6000
	10	+NNODES=2
	11	+NODE_RANK=0
	12	+WORLD_SIZE=$(($GPUS_PER_NODE*$NNODES))
	13	+
	14	+# please fill these path configurations
	15	+CHECKPOINT="Your ckpt file path"
	16	+TOKENIZER_PATH="Your vocab file path"
	17	+
	18	+TP=1
	19	+PP=2
	20	+EP=8
	21	+CP=1
	22	+CP_TYPE='ulysses_cp_algo'
	23	+NUM_LAYERS=60
	24	+SEQ_LEN=8192
	25	+
	26	+DISTRIBUTED_ARGS="
	27	+ --nproc_per_node $GPUS_PER_NODE \
	28	+ --nnodes $NNODES \
	29	+ --node_rank $NODE_RANK \
	30	+ --master_addr $MASTER_ADDR \
	31	+ --master_port $MASTER_PORT
	32	+"
	33	+
	34	+MLA_ARGS="
	35	+ --multi-head-latent-attention \
	36	+ --qk-rope-head-dim 64 \
	37	+ --qk-nope-head-dim 128 \
	38	+ --q-lora-rank 1536 \
	39	+ --kv-lora-rank 512 \
	40	+ --v-head-dim 128 \
	41	+ --qk-layernorm
	42	+"
	43	+
	44	+MOE_ARGS="
	45	+ --moe-permutation-async-comm \
	46	+ --moe-grouped-gemm \
	47	+ --moe-token-dispatcher-type alltoall \
	48	+ --use-fused-moe-token-permute-and-unpermute \
	49	+ --first-k-dense-replace 1 \
	50	+ --moe-layer-freq 1 \
	51	+ --n-shared-experts 2 \
	52	+ --num-experts 160 \
	53	+ --moe-router-topk 6 \
	54	+ --moe-intermediate-size 1536 \
	55	+ --moe-router-load-balancing-type group_limited_greedy \
	56	+ --topk-group 3 \
	57	+ --moe-aux-loss-coeff 0.003 \
	58	+ --moe-device-level-aux-loss-coeff 0.05 \
	59	+ --moe-comm-aux-loss-coeff 0.02 \
	60	+ --routed-scaling-factor 16.0 \
	61	+ --seq-aux
	62	+"
	63	+
	64	+ROPE_ARGS="
	65	+ --rope-scaling-beta-fast 32 \
	66	+ --rope-scaling-beta-slow 1 \
	67	+ --rope-scaling-factor 40 \
	68	+ --rope-scaling-mscale 1.0 \
	69	+ --rope-scaling-mscale-all-dim 1.0 \
	70	+ --rope-scaling-original-max-position-embeddings 4096 \
	71	+ --rope-scaling-type yarn
	72	+"
	73	+
	74	+GPT_ARGS="
	75	+ --spec mindspeed_llm.tasks.models.spec.deepseek_spec layer_spec \
	76	+ --use-distributed-optimizer \
	77	+ --reuse-fp32-param \
	78	+ --use-flash-attn \
	79	+ --shape-order BNSD \
	80	+ --use-mcore-models \
	81	+ --tensor-model-parallel-size ${TP} \
	82	+ --pipeline-model-parallel-size ${PP} \
	83	+ --expert-model-parallel-size ${EP} \
	84	+ --context-parallel-size ${CP} \
	85	+ --context-parallel-algo ${CP_TYPE} \
	86	+ --num-layers ${NUM_LAYERS} \
	87	+ --hidden-size 5120 \
	88	+ --ffn-hidden-size 12288 \
	89	+ --num-attention-heads 128 \
	90	+ --tokenizer-type PretrainedFromHF \
	91	+ --tokenizer-name-or-path ${TOKENIZER_PATH} \
	92	+ --seq-length ${SEQ_LEN} \
	93	+ --max-position-embeddings 163840 \
	94	+ --micro-batch-size 1 \
	95	+ --make-vocab-size-divisible-by 1 \
	96	+ --untie-embeddings-and-output-weights \
	97	+ --disable-bias-linear \
	98	+ --attention-dropout 0.0 \
	99	+ --init-method-std 0.02 \
	100	+ --hidden-dropout 0.0 \
	101	+ --position-embedding-type rope \
	102	+ --normalization RMSNorm \
	103	+ --use-fused-rotary-pos-emb \
	104	+ --use-rotary-position-embeddings \
	105	+ --use-fused-swiglu \
	106	+ --use-fused-rmsnorm \
	107	+ --swiglu \
	108	+ --no-masked-softmax-fusion \
	109	+ --attention-softmax-in-fp32 \
	110	+ --vocab-size 102400 \
	111	+ --padded-vocab-size 102400 \
	112	+ --rotary-base 10000 \
	113	+ --no-gradient-accumulation-fusion \
	114	+ --norm-epsilon 1e-6 \
	115	+ --max-new-tokens 256 \
	116	+ --bf16
	117	+"
	118	+
	119	+torchrun $DISTRIBUTED_ARGS inference.py \
	120	+ $GPT_ARGS \
	121	+ $MLA_ARGS \
	122	+ $ROPE_ARGS \
	123	+ $MOE_ARGS \
	124	+ --load ${CHECKPOINT} \
	125	+ --distributed-backend nccl \
	126	+ \| tee logs/generate_deepseek25_236b_8k_ptd.log

Aexamples/mcore/deepseek25/pretrain_deepseek25_236b_8k_ptd.sh+164-0

@@ -0,0 +1,164 @@
	1	+#!/bin/bash
	2	+export CUDA_DEVICE_MAX_CONNECTIONS=1
	3	+export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
	4	+export HCCL_CONNECT_TIMEOUT=3600
	5	+
	6	+GPUS_PER_NODE=8
	7	+MASTER_ADDR=localhost #主节点IP
	8	+MASTER_PORT=6000
	9	+NNODES=20
	10	+NODE_RANK=0
	11	+WORLD_SIZE=$(($GPUS_PER_NODE*$NNODES))
	12	+
	13	+CKPT_SAVE_DIR="your model save ckpt path"
	14	+DATA_PATH="your data path"
	15	+TOKENIZER_PATH="your tokenizer path"
	16	+CKPT_LOAD_DIR="your model ckpt path"
	17	+
	18	+TP=1
	19	+PP=10
	20	+EP=8
	21	+CP=1
	22	+CP_TYPE='ulysses_cp_algo'
	23	+NUM_LAYERS=60
	24	+SEQ_LEN=8192
	25	+MBS=1
	26	+GBS=640
	27	+
	28	+DISTRIBUTED_ARGS="
	29	+ --nproc_per_node $GPUS_PER_NODE \
	30	+ --nnodes $NNODES \
	31	+ --node_rank $NODE_RANK \
	32	+ --master_addr $MASTER_ADDR \
	33	+ --master_port $MASTER_PORT
	34	+"
	35	+
	36	+MLA_ARGS="
	37	+ --multi-head-latent-attention \
	38	+ --qk-rope-head-dim 64 \
	39	+ --qk-nope-head-dim 128 \
	40	+ --q-lora-rank 1536 \
	41	+ --kv-lora-rank 512 \
	42	+ --v-head-dim 128 \
	43	+ --qk-layernorm \
	44	+"
	45	+
	46	+MOE_ARGS="
	47	+ --moe-grouped-gemm \
	48	+ --moe-permutation-async-comm \
	49	+ --moe-token-dispatcher-type alltoall \
	50	+ --use-fused-moe-token-permute-and-unpermute \
	51	+ --first-k-dense-replace 1 \
	52	+ --moe-layer-freq 1 \
	53	+ --n-shared-experts 2 \
	54	+ --num-experts 160 \
	55	+ --moe-router-topk 6 \
	56	+ --moe-intermediate-size 1536 \
	57	+ --moe-router-load-balancing-type group_limited_greedy \
	58	+ --topk-group 3 \
	59	+ --moe-aux-loss-coeff 0.003 \
	60	+ --moe-device-level-aux-loss-coeff 0.05 \
	61	+ --moe-comm-aux-loss-coeff 0.02 \
	62	+ --routed-scaling-factor 16.0 \
	63	+ --seq-aux
	64	+"
	65	+
	66	+ROPE_ARGS="
	67	+ --rope-scaling-beta-fast 32 \
	68	+ --rope-scaling-beta-slow 1 \
	69	+ --rope-scaling-factor 40 \
	70	+ --rope-scaling-mscale 1.0 \
	71	+ --rope-scaling-mscale-all-dim 1.0 \
	72	+ --rope-scaling-original-max-position-embeddings 4096 \
	73	+ --rope-scaling-type yarn
	74	+"
	75	+
	76	+GPT_ARGS="
	77	+ --spec mindspeed_llm.tasks.models.spec.deepseek_spec layer_spec \
	78	+ --num-layers-per-virtual-pipeline-stage 2 \
	79	+ --recompute-granularity full \
	80	+ --recompute-method uniform \
	81	+ --recompute-num-layers 1 \
	82	+ --no-shared-storage \
	83	+ --use-distributed-optimizer \
	84	+ --reuse-fp32-param \
	85	+ --use-flash-attn \
	86	+ --shape-order BNSD \
	87	+ --use-mcore-models \
	88	+ --tensor-model-parallel-size ${TP} \
	89	+ --pipeline-model-parallel-size ${PP} \
	90	+ --expert-model-parallel-size ${EP} \
	91	+ --sequence-parallel \
	92	+ --output-layer-slice-num 10 \
	93	+ --context-parallel-size ${CP} \
	94	+ --context-parallel-algo ${CP_TYPE} \
	95	+ --num-layers ${NUM_LAYERS} \
	96	+ --hidden-size 5120 \
	97	+ --ffn-hidden-size 12288 \
	98	+ --num-attention-heads 128 \
	99	+ --tokenizer-type PretrainedFromHF \
	100	+ --tokenizer-name-or-path ${TOKENIZER_PATH} \
	101	+ --seq-length ${SEQ_LEN} \
	102	+ --max-position-embeddings 163840 \
	103	+ --micro-batch-size ${MBS} \
	104	+ --global-batch-size ${GBS} \
	105	+ --make-vocab-size-divisible-by 1 \
	106	+ --lr 1.0e-5 \
	107	+ --train-iters 2000 \
	108	+ --lr-decay-style cosine \
	109	+ --untie-embeddings-and-output-weights \
	110	+ --disable-bias-linear \
	111	+ --attention-dropout 0.0 \
	112	+ --init-method-std 0.02 \
	113	+ --hidden-dropout 0.0 \
	114	+ --position-embedding-type rope \
	115	+ --normalization RMSNorm \
	116	+ --use-fused-rotary-pos-emb \
	117	+ --use-rotary-position-embeddings \
	118	+ --use-fused-swiglu \
	119	+ --use-fused-rmsnorm \
	120	+ --swiglu \
	121	+ --no-masked-softmax-fusion \
	122	+ --attention-softmax-in-fp32 \
	123	+ --min-lr 1.0e-7 \
	124	+ --weight-decay 1e-2 \
	125	+ --lr-warmup-iters 500 \
	126	+ --clip-grad 1.0 \
	127	+ --adam-beta1 0.9 \
	128	+ --adam-beta2 0.999 \
	129	+ --initial-loss-scale 65536 \
	130	+ --vocab-size 102400 \
	131	+ --padded-vocab-size 102400 \
	132	+ --rotary-base 10000 \
	133	+ --no-gradient-accumulation-fusion \
	134	+ --norm-epsilon 1e-6 \
	135	+ --no-load-optim \
	136	+ --no-load-rng \
	137	+ --bf16
	138	+"
	139	+
	140	+DATA_ARGS="
	141	+ --data-path $DATA_PATH \
	142	+ --split 100,0,0
	143	+"
	144	+
	145	+OUTPUT_ARGS="
	146	+ --log-interval 1 \
	147	+ --save-interval 2000 \
	148	+ --eval-interval 2000 \
	149	+ --eval-iters 0 \
	150	+ --no-save-optim \
	151	+ --no-save-rng
	152	+"
	153	+
	154	+python -m torch.distributed.launch $DISTRIBUTED_ARGS pretrain_gpt.py \
	155	+ $GPT_ARGS \
	156	+ $DATA_ARGS \
	157	+ $OUTPUT_ARGS \
	158	+ $MLA_ARGS \
	159	+ $ROPE_ARGS \
	160	+ $MOE_ARGS \
	161	+ --distributed-backend nccl \
	162	+ --save $CKPT_SAVE_DIR \
	163	+ --load $CKPT_LOAD_DIR \
	164	+ \| tee logs/pretrain_deepseek25_236b_8k_ptd.log

Aexamples/mcore/deepseek25/pretrain_deepseek25_60b_32k_ptd.sh+164-0

@@ -0,0 +1,164 @@
	1	+#!/bin/bash
	2	+export CUDA_DEVICE_MAX_CONNECTIONS=1
	3	+export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
	4	+export HCCL_CONNECT_TIMEOUT=3600
	5	+
	6	+GPUS_PER_NODE=8
	7	+MASTER_ADDR=localhost #主节点IP
	8	+MASTER_PORT=6000
	9	+NNODES=4
	10	+NODE_RANK=0
	11	+WORLD_SIZE=$(($GPUS_PER_NODE*$NNODES))
	12	+
	13	+CKPT_SAVE_DIR="your model save ckpt path"
	14	+DATA_PATH="your data path"
	15	+TOKENIZER_PATH="your tokenizer path"
	16	+CKPT_LOAD_DIR="your model ckpt path"
	17	+
	18	+TP=1
	19	+PP=4
	20	+EP=8
	21	+CP=8
	22	+CP_TYPE='ulysses_cp_algo'
	23	+NUM_LAYERS=16
	24	+SEQ_LEN=32768
	25	+MBS=1
	26	+GBS=64
	27	+
	28	+DISTRIBUTED_ARGS="
	29	+ --nproc_per_node $GPUS_PER_NODE \
	30	+ --nnodes $NNODES \
	31	+ --node_rank $NODE_RANK \
	32	+ --master_addr $MASTER_ADDR \
	33	+ --master_port $MASTER_PORT
	34	+"
	35	+
	36	+MLA_ARGS="
	37	+ --multi-head-latent-attention \
	38	+ --qk-rope-head-dim 64 \
	39	+ --qk-nope-head-dim 128 \
	40	+ --q-lora-rank 1536 \
	41	+ --kv-lora-rank 512 \
	42	+ --v-head-dim 128 \
	43	+ --qk-layernorm \
	44	+"
	45	+
	46	+MOE_ARGS="
	47	+ --moe-grouped-gemm \
	48	+ --moe-permutation-async-comm \
	49	+ --moe-token-dispatcher-type alltoall \
	50	+ --use-fused-moe-token-permute-and-unpermute \
	51	+ --first-k-dense-replace 1 \
	52	+ --moe-layer-freq 1 \
	53	+ --n-shared-experts 2 \
	54	+ --num-experts 160 \
	55	+ --moe-router-topk 6 \
	56	+ --moe-intermediate-size 1536 \
	57	+ --moe-router-load-balancing-type group_limited_greedy \
	58	+ --topk-group 3 \
	59	+ --moe-aux-loss-coeff 0.003 \
	60	+ --moe-device-level-aux-loss-coeff 0.05 \
	61	+ --moe-comm-aux-loss-coeff 0.02 \
	62	+ --routed-scaling-factor 16.0 \
	63	+ --seq-aux
	64	+"
	65	+
	66	+ROPE_ARGS="
	67	+ --rope-scaling-beta-fast 32 \
	68	+ --rope-scaling-beta-slow 1 \
	69	+ --rope-scaling-factor 40 \
	70	+ --rope-scaling-mscale 1.0 \
	71	+ --rope-scaling-mscale-all-dim 1.0 \
	72	+ --rope-scaling-original-max-position-embeddings 4096 \
	73	+ --rope-scaling-type yarn
	74	+"
	75	+
	76	+GPT_ARGS="
	77	+ --spec mindspeed_llm.tasks.models.spec.deepseek_spec layer_spec \
	78	+ --num-layers-per-virtual-pipeline-stage 2 \
	79	+ --recompute-granularity full \
	80	+ --recompute-method uniform \
	81	+ --recompute-num-layers 1 \
	82	+ --no-shared-storage \
	83	+ --use-distributed-optimizer \
	84	+ --reuse-fp32-param \
	85	+ --use-flash-attn \
	86	+ --shape-order BNSD \
	87	+ --use-mcore-models \
	88	+ --tensor-model-parallel-size ${TP} \
	89	+ --pipeline-model-parallel-size ${PP} \
	90	+ --expert-model-parallel-size ${EP} \
	91	+ --sequence-parallel \
	92	+ --output-layer-slice-num 10 \
	93	+ --context-parallel-size ${CP} \
	94	+ --context-parallel-algo ${CP_TYPE} \
	95	+ --num-layers ${NUM_LAYERS} \
	96	+ --hidden-size 5120 \
	97	+ --ffn-hidden-size 12288 \
	98	+ --num-attention-heads 128 \
	99	+ --tokenizer-type PretrainedFromHF \
	100	+ --tokenizer-name-or-path ${TOKENIZER_PATH} \
	101	+ --seq-length ${SEQ_LEN} \
	102	+ --max-position-embeddings 163840 \
	103	+ --micro-batch-size ${MBS} \
	104	+ --global-batch-size ${GBS} \
	105	+ --make-vocab-size-divisible-by 1 \
	106	+ --lr 1.0e-5 \
	107	+ --train-iters 2000 \
	108	+ --lr-decay-style cosine \
	109	+ --untie-embeddings-and-output-weights \
	110	+ --disable-bias-linear \
	111	+ --attention-dropout 0.0 \
	112	+ --init-method-std 0.02 \
	113	+ --hidden-dropout 0.0 \
	114	+ --position-embedding-type rope \
	115	+ --normalization RMSNorm \
	116	+ --use-fused-rotary-pos-emb \
	117	+ --use-rotary-position-embeddings \
	118	+ --use-fused-swiglu \
	119	+ --use-fused-rmsnorm \
	120	+ --swiglu \
	121	+ --no-masked-softmax-fusion \
	122	+ --attention-softmax-in-fp32 \
	123	+ --min-lr 1.0e-7 \
	124	+ --weight-decay 1e-2 \
	125	+ --lr-warmup-iters 500 \
	126	+ --clip-grad 1.0 \
	127	+ --adam-beta1 0.9 \
	128	+ --adam-beta2 0.999 \
	129	+ --initial-loss-scale 65536 \
	130	+ --vocab-size 102400 \
	131	+ --padded-vocab-size 102400 \
	132	+ --rotary-base 10000 \
	133	+ --no-gradient-accumulation-fusion \
	134	+ --norm-epsilon 1e-6 \
	135	+ --no-load-optim \
	136	+ --no-load-rng \
	137	+ --bf16
	138	+"
	139	+
	140	+DATA_ARGS="
	141	+ --data-path $DATA_PATH \
	142	+ --split 100,0,0
	143	+"
	144	+
	145	+OUTPUT_ARGS="
	146	+ --log-interval 1 \
	147	+ --save-interval 2000 \
	148	+ --eval-interval 2000 \
	149	+ --eval-iters 0 \
	150	+ --no-save-optim \
	151	+ --no-save-rng
	152	+"
	153	+
	154	+python -m torch.distributed.launch $DISTRIBUTED_ARGS pretrain_gpt.py \
	155	+ $GPT_ARGS \
	156	+ $DATA_ARGS \
	157	+ $OUTPUT_ARGS \
	158	+ $MLA_ARGS \
	159	+ $ROPE_ARGS \
	160	+ $MOE_ARGS \
	161	+ --distributed-backend nccl \
	162	+ --save $CKPT_SAVE_DIR \
	163	+ --load $CKPT_LOAD_DIR \
	164	+ \| tee logs/pretrain_deepseek25_60b_32k_ptd.log

Aexamples/mcore/deepseek25/pretrain_deepseek25_60b_8k_ptd.sh+157-0

@@ -0,0 +1,157 @@
	1	+#!/bin/bash
	2	+export CUDA_DEVICE_MAX_CONNECTIONS=1
	3	+export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
	4	+export HCCL_CONNECT_TIMEOUT=1200
	5	+
	6	+GPUS_PER_NODE=8
	7	+MASTER_ADDR=localhost #主节点IP
	8	+MASTER_PORT=6000
	9	+NNODES=4
	10	+NODE_RANK=0
	11	+WORLD_SIZE=$(($GPUS_PER_NODE*$NNODES))
	12	+
	13	+CKPT_SAVE_DIR="your model save ckpt path"
	14	+DATA_PATH="your data path"
	15	+TOKENIZER_PATH="your tokenizer path"
	16	+CKPT_LOAD_DIR="your model ckpt path"
	17	+
	18	+TP=1
	19	+PP=4
	20	+EP=8
	21	+NUM_LAYERS=16
	22	+
	23	+DISTRIBUTED_ARGS="
	24	+ --nproc_per_node $GPUS_PER_NODE \
	25	+ --nnodes $NNODES \
	26	+ --node_rank $NODE_RANK \
	27	+ --master_addr $MASTER_ADDR \
	28	+ --master_port $MASTER_PORT
	29	+"
	30	+
	31	+MLA_ARGS="
	32	+ --multi-head-latent-attention \
	33	+ --qk-rope-head-dim 64 \
	34	+ --qk-nope-head-dim 128 \
	35	+ --q-lora-rank 1536 \
	36	+ --kv-lora-rank 512 \
	37	+ --v-head-dim 128 \
	38	+ --qk-layernorm \
	39	+"
	40	+
	41	+MOE_ARGS="
	42	+ --moe-grouped-gemm \
	43	+ --moe-permutation-async-comm \
	44	+ --moe-token-dispatcher-type alltoall \
	45	+ --use-fused-moe-token-permute-and-unpermute \
	46	+ --first-k-dense-replace 1 \
	47	+ --moe-layer-freq 1 \
	48	+ --n-shared-experts 2 \
	49	+ --num-experts 160 \
	50	+ --moe-router-topk 6 \
	51	+ --moe-intermediate-size 1536 \
	52	+ --moe-router-load-balancing-type group_limited_greedy \
	53	+ --topk-group 3 \
	54	+ --moe-aux-loss-coeff 0.003 \
	55	+ --moe-device-level-aux-loss-coeff 0.05 \
	56	+ --moe-comm-aux-loss-coeff 0.02 \
	57	+ --routed-scaling-factor 16.0 \
	58	+ --seq-aux
	59	+"
	60	+
	61	+ROPE_ARGS="
	62	+ --rope-scaling-beta-fast 32 \
	63	+ --rope-scaling-beta-slow 1 \
	64	+ --rope-scaling-factor 40 \
	65	+ --rope-scaling-mscale 1.0 \
	66	+ --rope-scaling-mscale-all-dim 1.0 \
	67	+ --rope-scaling-original-max-position-embeddings 4096 \
	68	+ --rope-scaling-type yarn
	69	+"
	70	+
	71	+GPT_ARGS="
	72	+ --spec mindspeed_llm.tasks.models.spec.deepseek_spec layer_spec \
	73	+ --recompute-granularity full \
	74	+ --recompute-method uniform \
	75	+ --recompute-num-layers 1 \
	76	+ --reuse-fp32-param \
	77	+ --no-shared-storage \
	78	+ --use-distributed-optimizer \
	79	+ --use-flash-attn \
	80	+ --shape-order BNSD \
	81	+ --use-mcore-models \
	82	+ --tensor-model-parallel-size ${TP} \
	83	+ --pipeline-model-parallel-size ${PP} \
	84	+ --num-layers-per-virtual-pipeline-stage 2 \
	85	+ --expert-model-parallel-size ${EP} \
	86	+ --sequence-parallel \
	87	+ --output-layer-slice-num 10 \
	88	+ --num-layers ${NUM_LAYERS} \
	89	+ --hidden-size 5120 \
	90	+ --ffn-hidden-size 12288 \
	91	+ --num-attention-heads 128 \
	92	+ --tokenizer-type PretrainedFromHF \
	93	+ --tokenizer-name-or-path ${TOKENIZER_PATH} \
	94	+ --seq-length 8192 \
	95	+ --max-position-embeddings 163840 \
	96	+ --micro-batch-size 1 \
	97	+ --global-batch-size 256 \
	98	+ --make-vocab-size-divisible-by 1 \
	99	+ --lr 1.0e-5 \
	100	+ --train-iters 2000 \
	101	+ --lr-decay-style cosine \
	102	+ --untie-embeddings-and-output-weights \
	103	+ --disable-bias-linear \
	104	+ --attention-dropout 0.0 \
	105	+ --init-method-std 0.02 \
	106	+ --hidden-dropout 0.0 \
	107	+ --position-embedding-type rope \
	108	+ --normalization RMSNorm \
	109	+ --use-fused-rotary-pos-emb \
	110	+ --use-rotary-position-embeddings \
	111	+ --use-fused-swiglu \
	112	+ --use-fused-rmsnorm \
	113	+ --swiglu \
	114	+ --no-masked-softmax-fusion \
	115	+ --attention-softmax-in-fp32 \
	116	+ --min-lr 1.0e-7 \
	117	+ --weight-decay 1e-2 \
	118	+ --lr-warmup-iters 500 \
	119	+ --clip-grad 1.0 \
	120	+ --adam-beta1 0.9 \
	121	+ --adam-beta2 0.999 \
	122	+ --initial-loss-scale 65536 \
	123	+ --vocab-size 102400 \
	124	+ --padded-vocab-size 102400 \
	125	+ --rotary-base 10000 \
	126	+ --no-gradient-accumulation-fusion \
	127	+ --norm-epsilon 1e-6 \
	128	+ --no-load-optim \
	129	+ --no-load-rng \
	130	+ --bf16
	131	+"
	132	+
	133	+DATA_ARGS="
	134	+ --data-path $DATA_PATH \
	135	+ --split 100,0,0
	136	+"
	137	+
	138	+OUTPUT_ARGS="
	139	+ --log-interval 1 \
	140	+ --save-interval 2000 \
	141	+ --eval-interval 2000 \
	142	+ --eval-iters 0 \
	143	+ --no-save-optim \
	144	+ --no-save-rng
	145	+"
	146	+
	147	+python -m torch.distributed.launch $DISTRIBUTED_ARGS pretrain_gpt.py \
	148	+ $GPT_ARGS \
	149	+ $DATA_ARGS \
	150	+ $OUTPUT_ARGS \
	151	+ $MLA_ARGS \
	152	+ $ROPE_ARGS \
	153	+ $MOE_ARGS \
	154	+ --distributed-backend nccl \
	155	+ --save $CKPT_SAVE_DIR \
	156	+ --load $CKPT_LOAD_DIR \
	157	+ \| tee logs/pretrain_deepseek25_60b_8k_ptd.log

Mmindspeed_llm/core/transformer/moe/moe_layer.py+20-18

	@@ -77,32 +77,34 @@ def moe_layer_forward(self, hidden_states: torch.Tensor):
77		77
78	args = get_args()	78	args = get_args()
79	if args.moe_router_load_balancing_type == "group_limited_greedy":	79	if args.moe_router_load_balancing_type == "group_limited_greedy":
80	- save_to_aux_losses_tracker(	80	+ # forward only need no loss track
81	- "load_balancing_loss",	81	+ if hasattr(args, "do_train") and args.do_train:
82	- self.router.l_aux,
83	- self.layer_number,
84	- self.config.num_layers,
85	- )
86	- save_to_aux_losses_tracker(
87	- "load_balancing_expert_level_loss",
88	- self.router.l_expert_aux / args.moe_aux_loss_coeff,
89	- self.layer_number,
90	- self.config.num_layers,
91	- )
92	- if hasattr(self.router, 'l_device_aux'):
93	save_to_aux_losses_tracker(	82	save_to_aux_losses_tracker(
94	- "load_balancing_device_level_loss",	83	+ "load_balancing_loss",
95	- self.router.l_device_aux / args.moe_device_level_aux_loss_coeff,	84	+ self.router.l_aux,
96	self.layer_number,	85	self.layer_number,
97	self.config.num_layers,	86	self.config.num_layers,
98	)	87	)
99	- if hasattr(self.router, 'l_comm_aux'):
100	save_to_aux_losses_tracker(	88	save_to_aux_losses_tracker(
101	- "load_balancing_comm_level_loss",	89	+ "load_balancing_expert_level_loss",
102	- self.router.l_comm_aux / args.moe_comm_aux_loss_coeff,	90	+ self.router.l_expert_aux / args.moe_aux_loss_coeff,
103	self.layer_number,	91	self.layer_number,
104	self.config.num_layers,	92	self.config.num_layers,
105	)	93	)
		94	+ if hasattr(self.router, 'l_device_aux'):
		95	+ save_to_aux_losses_tracker(
		96	+ "load_balancing_device_level_loss",
		97	+ self.router.l_device_aux / args.moe_device_level_aux_loss_coeff,
		98	+ self.layer_number,
		99	+ self.config.num_layers,
		100	+ )
		101	+ if hasattr(self.router, 'l_comm_aux'):
		102	+ save_to_aux_losses_tracker(
		103	+ "load_balancing_comm_level_loss",
		104	+ self.router.l_comm_aux / args.moe_comm_aux_loss_coeff,
		105	+ self.layer_number,
		106	+ self.config.num_layers,
		107	+ )
106	output = MoEAuxLossAutoScaler.apply(output, self.router.l_aux)	108	output = MoEAuxLossAutoScaler.apply(output, self.router.l_aux)
107		109
108	if args.n_shared_experts:	110	if args.n_shared_experts:

Mtests/ut/inference/test_inference.json+78-0

	@@ -295,5 +295,83 @@
295	"make-vocab-size-divisible-by": 32	295	"make-vocab-size-divisible-by": 32
296	}	296	}
297	}	297	}
		298	+ ],
		299	+ "test_deepseek2_mcore_greedy_search": [
		300	+ {
		301	+ "param": {
		302	+ "use-mcore-models": null,
		303	+ "spec": ["mindspeed_llm.tasks.models.spec.deepseek_spec", "layer_spec"],
		304	+ "tensor-model-parallel-size": 1,
		305	+ "pipeline-model-parallel-size": 1,
		306	+ "expert-model-parallel-size": 8,
		307	+ "use-flash-attn": null,
		308	+ "num-layers": 4,
		309	+ "hidden-size": 5120,
		310	+ "ffn-hidden-size": 12288,
		311	+ "seq-length": 8192,
		312	+ "max-new-tokens": 30,
		313	+ "micro-batch-size": 1,
		314	+ "global-batch-size": 16,
		315	+ "num-attention-heads": 128,
		316	+ "max-position-embeddings": 163840,
		317	+ "position-embedding-type": "rope",
		318	+ "swiglu": null,
		319	+ "tokenizer-type": "PretrainedFromHF",
		320	+ "tokenizer-name-or-path": "/data/gemm/",
		321	+ "bf16": null,
		322	+ "normalization": "RMSNorm",
		323	+ "untie-embeddings-and-output-weights": null,
		324	+ "disable-bias-linear": null,
		325	+ "attention-softmax-in-fp32": null,
		326	+ "no-load-optim": null,
		327	+ "no-load-rng": null,
		328	+ "no-masked-softmax-fusion": null,
		329	+ "no-gradient-accumulation-fusion": null,
		330	+ "task": "greedy",
		331	+ "use-deter-comp": null,
		332	+ "make-vocab-size-divisible-by": 1,
		333	+ "shape-order": "BNSD",
		334	+ "output-layer-slice-num": 10,
		335	+ "use-fused-swiglu": null,
		336	+ "use-fused-rmsnorm": null,
		337	+ "use-fused-rotary-pos-emb": null,
		338	+ "use-rotary-position-embeddings": null,
		339	+ "vocab-size": 102400,
		340	+ "padded-vocab-size": 102400,
		341	+ "rotary-base": 10000,
		342	+ "norm-epsilon": 1e-6,
		343	+ "multi-head-latent-attention": null,
		344	+ "qk-rope-head-dim": 64,
		345	+ "qk-nope-head-dim": 128,
		346	+ "q-lora-rank": 1536,
		347	+ "kv-lora-rank": 512,
		348	+ "v-head-dim": 128,
		349	+ "qk-layernorm": null,
		350	+ "moe-grouped-gemm": null,
		351	+ "moe-permutation-async-comm": null,
		352	+ "moe-token-dispatcher-type": "alltoall",
		353	+ "use-fused-moe-token-permute-and-unpermute": null,
		354	+ "first-k-dense-replace": 1,
		355	+ "moe-layer-freq": 1,
		356	+ "n-shared-experts": 2,
		357	+ "num-experts": 160,
		358	+ "moe-router-topk": 6,
		359	+ "moe-intermediate-size": 1536,
		360	+ "moe-router-load-balancing-type": "group_limited_greedy",
		361	+ "topk-group": 3,
		362	+ "moe-aux-loss-coeff": 0.003,
		363	+ "moe-device-level-aux-loss-coeff": 0.05,
		364	+ "moe-comm-aux-loss-coeff": 0.02,
		365	+ "routed-scaling-factor": 16.0,
		366	+ "seq-aux": null,
		367	+ "rope-scaling-beta-fast": 32,
		368	+ "rope-scaling-beta-slow": 1,
		369	+ "rope-scaling-factor": 40,
		370	+ "rope-scaling-mscale": 1.0,
		371	+ "rope-scaling-mscale-all-dim": 1.0,
		372	+ "rope-scaling-original-max-position-embeddings": 4096,
		373	+ "rope-scaling-type": "yarn"
		374	+ }
		375	+ }
298	]	376	]
299	}	377	}

Mtests/ut/inference/test_inference.py+19-1

	@@ -161,4 +161,22 @@ class TestInference(DistributedTest):
161	context = acquire_context(log_capture)	161	context = acquire_context(log_capture)
162	assert [context] == [	162	assert [context] == [
163	"” “I’m fine.” “I’m glad to hear it.” “I’m glad to hear it too.” “"	163	"” “I’m fine.” “I’m glad to hear it.” “I’m glad to hear it too.” “"
164	- ], "forward pass has been changed, check it!"	164	+ ], "forward pass has been changed, check it!"
		165	+
		166	+ @pytest.mark.parametrize("params", test_config["test_deepseek2_mcore_greedy_search"])
		167	+ def test_deepseek2_mcore_greedy_search(self, build_args, params):
		168	+ os.environ["CUDA_DEVICE_MAX_CONNECTIONS"] = "1"
		169	+ os.environ["CLOSE_MATMUL_K_SHIFT"] = "1"
		170	+ if dist.get_rank() == 0:
		171	+ handler, log_capture = setup_logger(PATTERN)
		172	+
		173	+ main()
		174	+ if dist.get_rank() == 0:
		175	+ print("=============== deepseek2 mcore greedy search =============")
		176	+ print(log_capture)
		177	+ context = acquire_context(log_capture)
		178	+ # 减层
		179	+ assert [context] == [
		180	+ "て argumento detectar revers^{-}\|\| GR rust liaisonidi изследва pron查处Navrack在本 Howmodern组成的vark Lou枸 "
		181	+ "Lizzie нощта ultimate和管理 Confedermarried"
		182	+ ], "forward pass has been changed, check it!"