[pytorch][feature]integrate moe merge process into FSDP2 training-MindSpeed-LLM-AtomGit

已合并

[pytorch][feature]integrate moe merge process into FSDP2 training #4250

[pytorch][feature]integrate moe merge process into FSDP2 training #4250

已合并

JZY_SC30创建于 2月27日

共 7 个文件变更+20-8

	@@ -22,4 +22,5 @@ DISTRIBUTED_ARGS="
22	--master_port $MASTER_PORT	22	--master_port $MASTER_PORT
23	"	23	"
24		24
		25	+bash tests/tools/fsdp2/moe_hf_param_merge_experts.sh
25	torchrun $DISTRIBUTED_ARGS train_fsdp2.py examples/fsdp2/qwen3_moe/pretrain_qwen3_235b_4k_fsdp2_A3.yaml \| tee logs/train_fsdp2_qwen3_235b_A3.log	26	torchrun $DISTRIBUTED_ARGS train_fsdp2.py examples/fsdp2/qwen3_moe/pretrain_qwen3_235b_4k_fsdp2_A3.yaml \| tee logs/train_fsdp2_qwen3_235b_A3.log

	@@ -1,6 +1,6 @@
1	model:	1	model:
2	model_id: qwen3_moe	2	model_id: qwen3_moe
3	- model_name_or_path: ./qwen3-235b/	3	+ model_name_or_path: ./model_weights/qwen3-235b-mergeExperts/ #此处需填转换后保存的权重路径
4	trust_remote_code: True	4	trust_remote_code: True
5	train_from_scratch: False	5	train_from_scratch: False
6	init_model_with_meta_device: False	6	init_model_with_meta_device: False

	@@ -18,6 +18,7 @@ DISTRIBUTED_ARGS="
18	"	18	"
19		19
20	mkdir -p ./logs	20	mkdir -p ./logs
		21	+bash tests/tools/fsdp2/moe_hf_param_merge_experts.sh
21	torchrun $DISTRIBUTED_ARGS train_fsdp2.py \	22	torchrun $DISTRIBUTED_ARGS train_fsdp2.py \
22	examples/fsdp2/qwen3_moe/pretrain_qwen3_30b_4k_fsdp2_A3.yaml \	23	examples/fsdp2/qwen3_moe/pretrain_qwen3_30b_4k_fsdp2_A3.yaml \
23	\| tee logs/pretrain_qwen3_moe_30b_a3b_4K_fsdp2_${TIMESTAMP}.log	24	\| tee logs/pretrain_qwen3_moe_30b_a3b_4K_fsdp2_${TIMESTAMP}.log

	@@ -1,9 +1,9 @@
1	model:	1	model:
2	model_id: qwen3_moe	2	model_id: qwen3_moe
3	- model_name_or_path: ./Qwen3-30B-A3B/	3	+ model_name_or_path: ./model_weights/Qwen3-30B-A3B-mergeExperts/ #此处需填转换后保存的权重路径
4	trust_remote_code: True	4	trust_remote_code: True
5	train_from_scratch: False	5	train_from_scratch: False
6	- tokenizer_name_or_path: ./Qwen3-30B-A3B/	6	+ tokenizer_name_or_path: ./model_weights/Qwen3-30B-A3B-mergeExperts/ #此处需填转换后保存的权重路径
7	init_model_with_meta_device: True	7	init_model_with_meta_device: True
8		8
9	data:	9	data:

	@@ -1,5 +0,0 @@
1	-#!/bin/bash
2	-
3	-python examples/fsdp2/qwen3-next/qwen3_next_hf_param_merge_experts.py \
4	- --load-dir ./model_weights/Qwen3-Next-A3B \
5	- --save-dir ./model_weights/Qwen3-Next-A3B-mergeExperts

Rtests/fsdp2/qwen3_next_hf_param_merge_experts.py→tests/tools/fsdp2/moe_hf_param_merge_experts.py+0-0

文件重命名但无更改。

@@ -0,0 +1,15 @@
	1	+#!/bin/bash
	2	+
	3	+LOAD_DIR=./hf_weights/Qwen3-MoE
	4	+SAVE_DIR=./model_weights/Qwen3-MoE-mergeExperts
	5	+CONVERTER=./tests/tools/fsdp2/moe_hf_param_merge_experts.py
	6	+
	7	+# Skip conversion when merged weights already exist.
	8	+if [[ -f "${SAVE_DIR}/model.safetensors.index.json" ]] && ls "${SAVE_DIR}"/model-*.safetensors >/dev/null 2>&1; then
	9	+ echo "[skip] merged weights already exist at ${SAVE_DIR}, skip conversion."
	10	+ exit 0
	11	+fi
	12	+
	13	+python "${CONVERTER}" \
	14	+ --load-dir "${LOAD_DIR}" \
	15	+ --save-dir "${SAVE_DIR}"