已合并
【HunyuanLargeMoE】part of checkpoint #2246
zhoubeirong创建于 2025年2月19日
【HunyuanLargeMoE】part of checkpoint #2246
已合并
从refs/pull/2246/head合入到master
共 5 个文件变更+79-15
| @@ -440,6 +440,53 @@ | |||
| 440 | "layers_mlp_experts_up_proj": "model.layers[layer_idx].block_sparse_moe.experts[expert_idx].w3", | 440 | "layers_mlp_experts_up_proj": "model.layers[layer_idx].block_sparse_moe.experts[expert_idx].w3", |
| 441 | "layers_mlp_experts_linear_fc2": "model.layers[layer_idx].block_sparse_moe.experts[expert_idx].w2" | 441 | "layers_mlp_experts_linear_fc2": "model.layers[layer_idx].block_sparse_moe.experts[expert_idx].w2" |
| 442 | } | 442 | } |
| 443 | - } | 443 | + }, |
| 444 | + "hunyuan": { | ||
| 445 | + "__base__": "base", | ||
| 446 | + "config_set_value": { | ||
| 447 | + "seq_length": 4096, | ||
| 448 | + "global_batch_size": 64, | ||
| 449 | + "qkv_type": "unpack", | ||
| 450 | + "mlp_experts_flag": true, | ||
| 451 | + "first_k_dense_replace": 0, | ||
| 452 | + "moe_layer_freq": 1, | ||
| 453 | + "qk_layernorm": true, | ||
| 454 | + "q_lora_rank": true | ||
| 455 | + }, | ||
| 456 | + "config_hf_key_mapping": { | ||
| 457 | + "num_layers": "num_hidden_layers", | ||
| 458 | + "moe_intermediate_size": "intermediate_size", | ||
| 459 | + "moe_layer_freq": "moe_layer_freq", | ||
| 460 | + "num_experts": "num_experts", | ||
| 461 | + "n_shared_experts": "num_shared_expert", | ||
| 462 | + "norm_epsilon": "rms_norm_eps", | ||
| 463 | + "rotary_base": "rope_theta", | ||
| 464 | + "shared_expert_intermediate_size": "shared_expert_intermediate_size" | ||
| 465 | + }, | ||
| 466 | + "model_hf_key_mapping": { | ||
| 467 | + "model": "module[0]", | ||
| 468 | + "embedding_word_embeddings": "model.embed_tokens", | ||
| 469 | + "embedding_word_embeddings_norm": "model.embedding.word_embeddings.norm", | ||
| 470 | + "layers": "model.layers", | ||
| 471 | + "layers_input_layernorm": "model.layers[layer_idx].input_layernorm", | ||
| 472 | + "layers_self_attention_linear_proj": "model.layers[layer_idx].self_attn.o_proj", | ||
| 473 | + "layers_self_attention_linear_q_proj": "model.layers[layer_idx].self_attn.q_proj", | ||
| 474 | + "layers_self_attention_linear_k_proj": "model.layers[layer_idx].self_attn.k_proj", | ||
| 475 | + "layers_self_attention_linear_v_proj": "model.layers[layer_idx].self_attn.v_proj", | ||
| 476 | + "layers_self_attention_q_layernorm": "model.layers[layer_idx].self_attn.query_layernorm", | ||
| 477 | + "layers_self_attention_k_layernorm": "model.layers[layer_idx].self_attn.key_layernorm", | ||
| 478 | + "layers_self_attention_pre_mlp_layernorm": "model.layers[layer_idx].post_attention_layernorm", | ||
| 479 | + "layers_mlp_router": "model.layers[layer_idx].mlp.gate.wg", | ||
| 480 | + "layers_mlp_experts_gate_proj": "model.layers[layer_idx].mlp.experts[expert_idx].gate_proj", | ||
| 481 | + "layers_mlp_experts_up_proj": "model.layers[layer_idx].mlp.experts[expert_idx].up_proj", | ||
| 482 | + "layers_mlp_experts_linear_fc2": "model.layers[layer_idx].mlp.experts[expert_idx].down_proj", | ||
| 483 | + | ||
| 484 | + "layers_mlp_shared_experts_gate_proj": "model.layers[layer_idx].mlp.shared_mlp.gate_proj", | ||
| 485 | + "layers_mlp_shared_experts_up_proj": "model.layers[layer_idx].mlp.shared_mlp.up_proj", | ||
| 486 | + "layers_mlp_shared_experts_linear_fc2": "model.layers[layer_idx].mlp.shared_mlp.down_proj", | ||
| 487 | + "final_layernorm": "model.norm", | ||
| 488 | + "output_layer": "lm_head" | ||
| 489 | + } | ||
| 490 | + } | ||
| 444 | } | 491 | } |
| 445 | } | 492 | } |
| @@ -63,7 +63,7 @@ def main(): | |||
| 63 | parser.add_argument('--model-type-hf', type=str, default="llama2", | 63 | parser.add_argument('--model-type-hf', type=str, default="llama2", |
| 64 | choices=['baichuan', 'baichuan2', 'llama2', 'mixtral', 'chatglm3', 'gemma', 'gemma2', | 64 | choices=['baichuan', 'baichuan2', 'llama2', 'mixtral', 'chatglm3', 'gemma', 'gemma2', |
| 65 | 'bloom', 'bloom_3b', 'qwen', 'internlm2', 'deepseek2', 'minicpm', 'minicpm3', 'minicpm-moe', | 65 | 'bloom', 'bloom_3b', 'qwen', 'internlm2', 'deepseek2', 'minicpm', 'minicpm3', 'minicpm-moe', |
| 66 | - 'deepseek2-lite', 'qwen2-moe', 'phi3.5', 'phi3.5-moe'], | 66 | + 'deepseek2-lite', 'qwen2-moe', 'phi3.5', 'phi3.5-moe', 'hunyuan'], |
| 67 | help='model type of huggingface') | 67 | help='model type of huggingface') |
| 68 | parser.add_argument('--ckpt-cfg-path', type=str, default="configs/checkpoint/model_cfg.json", | 68 | parser.add_argument('--ckpt-cfg-path', type=str, default="configs/checkpoint/model_cfg.json", |
| 69 | help="Path to the config directory. If not specified, the default path in the repository will be used.") | 69 | help="Path to the config directory. If not specified, the default path in the repository will be used.") |
| @@ -109,6 +109,8 @@ def build_metadata(args, margs): | |||
| 109 | md.first_k_dense_replace = getattr(margs, "first_k_dense_replace", None) | 109 | md.first_k_dense_replace = getattr(margs, "first_k_dense_replace", None) |
| 110 | md.moe_layer_freq = getattr(margs, "moe_layer_freq", None) | 110 | md.moe_layer_freq = getattr(margs, "moe_layer_freq", None) |
| 111 | md.multi_head_latent_attention = getattr(margs, "multi_head_latent_attention", False) | 111 | md.multi_head_latent_attention = getattr(margs, "multi_head_latent_attention", False) |
| 112 | + md.cla_share_factor = getattr(margs, "cla_share_factor", 1) | ||
| 113 | + md.q_lora_rank = getattr(margs, "q_lora_rank", None) | ||
| 112 | 114 | ||
| 113 | if md.multi_head_latent_attention: | 115 | if md.multi_head_latent_attention: |
| 114 | md.qk_rope_head_dim = getattr(margs, "qk_rope_head_dim", None) | 116 | md.qk_rope_head_dim = getattr(margs, "qk_rope_head_dim", None) |
| @@ -99,6 +99,7 @@ def build_metadata(args, margs): | |||
| 99 | md.moe_intermediate_size = getattr(margs, "moe_intermediate_size", None) | 99 | md.moe_intermediate_size = getattr(margs, "moe_intermediate_size", None) |
| 100 | md.first_k_dense_replace = getattr(margs, "first_k_dense_replace", None) | 100 | md.first_k_dense_replace = getattr(margs, "first_k_dense_replace", None) |
| 101 | md.moe_layer_freq = getattr(margs, "moe_layer_freq", None) | 101 | md.moe_layer_freq = getattr(margs, "moe_layer_freq", None) |
| 102 | + md.q_lora_rank = getattr(margs, "q_lora_rank", None) | ||
| 102 | md.multi_head_latent_attention = getattr(margs, "multi_head_latent_attention", False) | 103 | md.multi_head_latent_attention = getattr(margs, "multi_head_latent_attention", False) |
| 103 | if md.multi_head_latent_attention: | 104 | if md.multi_head_latent_attention: |
| 104 | md.qk_rope_head_dim = getattr(margs, "qk_rope_head_dim", None) | 105 | md.qk_rope_head_dim = getattr(margs, "qk_rope_head_dim", None) |
| @@ -507,14 +507,20 @@ class HuggingfaceModel(ModelBase): | |||
| 507 | 507 | ||
| 508 | qkv_type = self.args.qkv_type | 508 | qkv_type = self.args.qkv_type |
| 509 | if qkv_type == "unpack": | 509 | if qkv_type == "unpack": |
| 510 | - q_proj = self.get_layers_self_attention_linear_q_proj_module(layer_idx=layer_idx) | 510 | + if hasattr(self.args, 'cla_share_factor') and layer_idx % self.args.cla_share_factor == 1: |
| 511 | - k_proj = self.get_layers_self_attention_linear_k_proj_module(layer_idx=layer_idx) | 511 | + q_proj = self.get_layers_self_attention_linear_q_proj_module(layer_idx=layer_idx) |
| 512 | - v_proj = self.get_layers_self_attention_linear_v_proj_module(layer_idx=layer_idx) | 512 | + query_key_value_weight = q_proj.weight |
| 513 | - query_key_value_weight = [q_proj.weight, k_proj.weight, v_proj.weight] | 513 | + self.layers_self_attention_linear_qkv_caches["weight"] = query_key_value_weight |
| 514 | - query_key_value_bias = [q_proj.bias, k_proj.bias, v_proj.bias] | 514 | + else: |
| 515 | - self.layers_self_attention_linear_qkv_caches["weight"] = (qkv_concatenate_weight(query_key_value_weight)) | 515 | + q_proj = self.get_layers_self_attention_linear_q_proj_module(layer_idx=layer_idx) |
| 516 | - if self.args_cmd.add_qkv_bias: | 516 | + k_proj = self.get_layers_self_attention_linear_k_proj_module(layer_idx=layer_idx) |
| 517 | - self.layers_self_attention_linear_qkv_caches["bias"] = (qkv_concatenate_bias(query_key_value_bias)) | 517 | + v_proj = self.get_layers_self_attention_linear_v_proj_module(layer_idx=layer_idx) |
| 518 | + query_key_value_weight = [q_proj.weight, k_proj.weight, v_proj.weight] | ||
| 519 | + query_key_value_bias = [q_proj.bias, k_proj.bias, v_proj.bias] | ||
| 520 | + self.layers_self_attention_linear_qkv_caches["weight"] = ( | ||
| 521 | + qkv_concatenate_weight(query_key_value_weight)) | ||
| 522 | + if self.args_cmd.add_qkv_bias: | ||
| 523 | + self.layers_self_attention_linear_qkv_caches["bias"] = (qkv_concatenate_bias(query_key_value_bias)) | ||
| 518 | elif qkv_type == "pack_mla": | 524 | elif qkv_type == "pack_mla": |
| 519 | q_proj = self.get_layers_self_attention_linear_q_proj_module(layer_idx=layer_idx) | 525 | q_proj = self.get_layers_self_attention_linear_q_proj_module(layer_idx=layer_idx) |
| 520 | kv_proj = self.get_layers_self_attention_linear_kv_proj_module(layer_idx=layer_idx) | 526 | kv_proj = self.get_layers_self_attention_linear_kv_proj_module(layer_idx=layer_idx) |
| @@ -681,10 +687,14 @@ class HuggingfaceModel(ModelBase): | |||
| 681 | 687 | ||
| 682 | qkv_type = self.args.qkv_type | 688 | qkv_type = self.args.qkv_type |
| 683 | if qkv_type == "unpack": | 689 | if qkv_type == "unpack": |
| 684 | - q_weight, k_weight, v_weight = qkv_split_weight(data) | 690 | + if hasattr(self.args, 'cla_share_factor') and layer_idx % self.args.cla_share_factor == 1: |
| 685 | - self.set_layers_self_attention_linear_q_proj_weight(layer_idx=layer_idx, data=q_weight) | 691 | + q_weight = data |
| 686 | - self.set_layers_self_attention_linear_k_proj_weight(layer_idx=layer_idx, data=k_weight) | 692 | + self.set_layers_self_attention_linear_q_proj_weight(layer_idx=layer_idx, data=q_weight) |
| 687 | - self.set_layers_self_attention_linear_v_proj_weight(layer_idx=layer_idx, data=v_weight) | 693 | + else: |
| 694 | + q_weight, k_weight, v_weight = qkv_split_weight(data) | ||
| 695 | + self.set_layers_self_attention_linear_q_proj_weight(layer_idx=layer_idx, data=q_weight) | ||
| 696 | + self.set_layers_self_attention_linear_k_proj_weight(layer_idx=layer_idx, data=k_weight) | ||
| 697 | + self.set_layers_self_attention_linear_v_proj_weight(layer_idx=layer_idx, data=v_weight) | ||
| 688 | elif qkv_type == "pack_gqa": | 698 | elif qkv_type == "pack_gqa": |
| 689 | qw, k_weight, v_weight = qkv_split_weight(data) | 699 | qw, k_weight, v_weight = qkv_split_weight(data) |
| 690 | qkv = torch.cat((qw, k_weight, v_weight), dim=0) | 700 | qkv = torch.cat((qw, k_weight, v_weight), dim=0) |
| @@ -796,6 +806,9 @@ class MegatronModel(ModelBase): | |||
| 796 | ) | 806 | ) |
| 797 | setattr(self.args, arg, value) | 807 | setattr(self.args, arg, value) |
| 798 | 808 | ||
| 809 | + if hasattr(self.md, 'cla_share_factor'): | ||
| 810 | + self.args.cla_share_factor = self.md.cla_share_factor | ||
| 811 | + | ||
| 799 | if hasattr(self.md, 'consumed_train_samples'): | 812 | if hasattr(self.md, 'consumed_train_samples'): |
| 800 | self.args.consumed_train_samples = self.md.consumed_train_samples | 813 | self.args.consumed_train_samples = self.md.consumed_train_samples |
| 801 | self.args.consumed_valid_samples = self.md.consumed_valid_samples | 814 | self.args.consumed_valid_samples = self.md.consumed_valid_samples |
| @@ -846,6 +859,7 @@ class MegatronModel(ModelBase): | |||
| 846 | self.args.first_k_dense_replace = getattr(hf_args, "first_k_dense_replace", None) | 859 | self.args.first_k_dense_replace = getattr(hf_args, "first_k_dense_replace", None) |
| 847 | self.args.moe_layer_freq = getattr(hf_args, "moe_layer_freq", None) | 860 | self.args.moe_layer_freq = getattr(hf_args, "moe_layer_freq", None) |
| 848 | self.args.multi_head_latent_attention = getattr(hf_args, "multi_head_latent_attention", False) | 861 | self.args.multi_head_latent_attention = getattr(hf_args, "multi_head_latent_attention", False) |
| 862 | + self.args.cla_share_factor = getattr(hf_args, "cla_share_factor", 1) | ||
| 849 | self.args.shared_expert_intermediate_size = getattr(hf_args, "shared_expert_intermediate_size", None) | 863 | self.args.shared_expert_intermediate_size = getattr(hf_args, "shared_expert_intermediate_size", None) |
| 850 | if self.args.shared_expert_intermediate_size is not None and self.args.n_shared_experts is None: | 864 | if self.args.shared_expert_intermediate_size is not None and self.args.n_shared_experts is None: |
| 851 | self.args.n_shared_experts = self.args.shared_expert_intermediate_size // self.args.moe_intermediate_size | 865 | self.args.n_shared_experts = self.args.shared_expert_intermediate_size // self.args.moe_intermediate_size |
| @@ -855,7 +869,7 @@ class MegatronModel(ModelBase): | |||
| 855 | self.args.q_lora_rank = getattr(hf_args, "q_lora_rank", None) | 869 | self.args.q_lora_rank = getattr(hf_args, "q_lora_rank", None) |
| 856 | self.args.kv_lora_rank = getattr(hf_args, "kv_lora_rank", None) | 870 | self.args.kv_lora_rank = getattr(hf_args, "kv_lora_rank", None) |
| 857 | self.args.v_head_dim = getattr(hf_args, "v_head_dim", None) | 871 | self.args.v_head_dim = getattr(hf_args, "v_head_dim", None) |
| 858 | - | 872 | + self.args.q_lora_rank = getattr(hf_args, "q_lora_rank", None) |
| 859 | if self.args.add_dense_bias: | 873 | if self.args.add_dense_bias: |
| 860 | self.args.skip_bias_add = False | 874 | self.args.skip_bias_add = False |
| 861 | 875 | ||