已合并
[pytorch][bugfix]fix num-layer-list of ckpt mg2hf #3777
温一盏创建于 2025年11月26日
[pytorch][bugfix]fix num-layer-list of ckpt mg2hf #3777
已合并
共 4 个文件变更+9-4
| @@ -23,8 +23,9 @@ class NumLayerListFeature(MindSpeedFeature): | |||
| 23 | 23 | ||
| 24 | def validate_args(self, args: Namespace): | 24 | def validate_args(self, args: Namespace): |
| 25 | if args.num_layer_list: | 25 | if args.num_layer_list: |
| 26 | - if len(args.num_layer_list.split(',')) != args.pipeline_model_parallel_size: | 26 | + if getattr(args, 'save_model_type', None) != 'hf': |
| 27 | - raise ValueError("len(args.num_layer_list) != args.pipeline_model_parallel_size") | 27 | + if len(args.num_layer_list.split(',')) != args.pipeline_model_parallel_size: |
| 28 | + raise ValueError("len(args.num_layer_list) != args.pipeline_model_parallel_size") | ||
| 28 | if not args.pipeline_model_parallel_size > 1: | 29 | if not args.pipeline_model_parallel_size > 1: |
| 29 | raise ValueError("Dynamic pipeline model should work with pipeline parallel.") | 30 | raise ValueError("Dynamic pipeline model should work with pipeline parallel.") |
| 30 | if args.num_layers_per_virtual_pipeline_stage: | 31 | if args.num_layers_per_virtual_pipeline_stage: |
| @@ -97,6 +97,7 @@ def build_metadata(args, margs): | |||
| 97 | md.moe_tp_extend_ep = margs.moe_tp_extend_ep | 97 | md.moe_tp_extend_ep = margs.moe_tp_extend_ep |
| 98 | md.spec = margs.spec | 98 | md.spec = margs.spec |
| 99 | md.num_experts = getattr(margs, "num_experts", None) | 99 | md.num_experts = getattr(margs, "num_experts", None) |
| 100 | + md.num_layer_list = getattr(margs, "num_layer_list", None) | ||
| 100 | md.n_shared_experts = getattr(margs, "n_shared_experts", None) | 101 | md.n_shared_experts = getattr(margs, "n_shared_experts", None) |
| 101 | md.qk_layernorm = getattr(margs, "qk_layernorm", False) | 102 | md.qk_layernorm = getattr(margs, "qk_layernorm", False) |
| 102 | md.moe_intermediate_size = getattr(margs, "moe_intermediate_size", None) | 103 | md.moe_intermediate_size = getattr(margs, "moe_intermediate_size", None) |
| @@ -1104,6 +1104,7 @@ class MegatronModel(ModelBase): | |||
| 1104 | self.args.q_lora_rank = getattr(self.args_megatron_checkpoint, "q_lora_rank", None) | 1104 | self.args.q_lora_rank = getattr(self.args_megatron_checkpoint, "q_lora_rank", None) |
| 1105 | self.args.sequence_parallel = getattr(self.args_megatron_checkpoint, "sequence_parallel", False) | 1105 | self.args.sequence_parallel = getattr(self.args_megatron_checkpoint, "sequence_parallel", False) |
| 1106 | self.args.shared_expert_gate = getattr(self.args_megatron_checkpoint, "shared_expert_gate", False) | 1106 | self.args.shared_expert_gate = getattr(self.args_megatron_checkpoint, "shared_expert_gate", False) |
| 1107 | + self.args.num_layer_list = getattr(self.args_megatron_checkpoint, "num_layer_list", None) | ||
| 1107 | 1108 | ||
| 1108 | def update_megatron_args_from_cmd_config(self, loader_megatron): | 1109 | def update_megatron_args_from_cmd_config(self, loader_megatron): |
| 1109 | self.args.ckpt_format = self.args_cmd.ckpt_format | 1110 | self.args.ckpt_format = self.args_cmd.ckpt_format |
| @@ -1137,8 +1138,9 @@ class MegatronModel(ModelBase): | |||
| 1137 | if not self.args_cmd.model_type == 'GPT': | 1138 | if not self.args_cmd.model_type == 'GPT': |
| 1138 | raise ValueError("Llama-2 is a GPT model.") | 1139 | raise ValueError("Llama-2 is a GPT model.") |
| 1139 | 1140 | ||
| 1140 | - if self.md and self.args_cmd.num_layer_list: | 1141 | + if self.args_cmd.save_model_type != "mg": |
| 1141 | - self.args.num_layer_list = self.args_cmd.num_layer_list | 1142 | + if self.md and self.args_cmd.num_layer_list: |
| 1143 | + self.args.num_layer_list = self.args_cmd.num_layer_list | ||
| 1142 | 1144 | ||
| 1143 | if self.args_cmd.noop_layers: | 1145 | if self.args_cmd.noop_layers: |
| 1144 | self.args.noop_layers = self.args_cmd.noop_layers.split(',') | 1146 | self.args.noop_layers = self.args_cmd.noop_layers.split(',') |
| @@ -127,6 +127,7 @@ | |||
| 127 | "save-dir":"/data/ci/models/qwen25/hf/Qwen2.5-7B", | 127 | "save-dir":"/data/ci/models/qwen25/hf/Qwen2.5-7B", |
| 128 | "use-mcore-models": null, | 128 | "use-mcore-models": null, |
| 129 | "add-qkv-bias": null, | 129 | "add-qkv-bias": null, |
| 130 | + "num-layer-list": "1,3", | ||
| 130 | "model-type-hf": "llama2", | 131 | "model-type-hf": "llama2", |
| 131 | "params-dtype": "bf16" | 132 | "params-dtype": "bf16" |
| 132 | } | 133 | } |