Star208
317
代码
代码
Issues63
Pull Requests93
项目讨论
Wiki
分析
Star208
317
已合并
适配Phi3.5-mini模型 #1945
AtomGit-Bot创建于 2024年11月27日
适配Phi3.5-mini模型 #1945
已合并
AtomGit-Bot创建于 2024年11月27日
从refs/pull/1945/head合入到master
共 12 个文件变更+330-8
Mconfigs/checkpoint/model_cfg.json+10-0
@@ -374,6 +374,16 @@
374 "model_hf_key_mapping": {374 "model_hf_key_mapping": {
375 "layers_self_attention_linear_qkv_pack": "model.layers[layer_idx].self_attn.W_pack"375 "layers_self_attention_linear_qkv_pack": "model.layers[layer_idx].self_attn.W_pack"
376 }376 }
377+ },
378+ "phi3.5": {
379+ "__base__": "base",
380+ "config_set_value": {
381+ "qkv_type": "pack_gqa"
382+ },
383+ "model_hf_key_mapping": {
384+ "layers_self_attention_linear_qkv_pack": "model.layers[layer_idx].self_attn.qkv_proj",
385+ "layers_mlp_linear_fc1": "model.layers[layer_idx].mlp.gate_up_proj"
386+ }
377 }387 }
378 }388 }
379}389}
Mconvert_ckpt.py+3-2
@@ -75,8 +75,9 @@ def main():
75 help='Specify the <module_location function_name> pair '75 help='Specify the <module_location function_name> pair '
76 'that returns a spec to customize transformer layer, depending on the use case.')76 'that returns a spec to customize transformer layer, depending on the use case.')
77 parser.add_argument('--model-type-hf', type=str, default="llama2",77 parser.add_argument('--model-type-hf', type=str, default="llama2",
78- choices=['baichuan', 'baichuan2', 'llama2', 'mixtral', 'chatglm3', 'gemma', 'gemma2', 'bloom',78+ choices=['baichuan', 'baichuan2', 'llama2', 'mixtral', 'chatglm3', 'gemma', 'gemma2',
79- 'qwen', 'internlm2', 'deepseek2', 'minicpm', 'minicpm3', 'minicpm-moe', 'deepseek2-lite', 'qwen2-moe'],79+ 'bloom', 'qwen', 'internlm2', 'deepseek2', 'minicpm', 'minicpm3', 'minicpm-moe',
80+ 'deepseek2-lite', 'qwen2-moe', 'phi3.5'],
80 help='model type of huggingface')81 help='model type of huggingface')
81 parser.add_argument('--ckpt-cfg-path', type=str, default="configs/checkpoint/model_cfg.json",82 parser.add_argument('--ckpt-cfg-path', type=str, default="configs/checkpoint/model_cfg.json",
82 help="Path to the config directory. If not specified, the default path in the repository will be used.")83 help="Path to the config directory. If not specified, the default path in the repository will be used.")
Mdocs/models/evaluation.md+1-0
@@ -51,6 +51,7 @@ MindSpeed-LLM 已支持的大模型评估数据统计如下：
51| Yi1.5-9B | MMLU | 69.2% | [69.5%](https://huggingface.co/01-ai/Yi-1.5-9B/tree/main) | Yi1.5-34B | MMLU | 76.9% | [77.1%](https://huggingface.co/01-ai/Yi-1.5-34B/tree/main) | 51| Yi1.5-9B | MMLU | 69.2% | [69.5%](https://huggingface.co/01-ai/Yi-1.5-9B/tree/main) | Yi1.5-34B | MMLU | 76.9% | [77.1%](https://huggingface.co/01-ai/Yi-1.5-34B/tree/main) |
52| CodeQWen2.5-7B | Human. | 66.5% | [61.6%](https://modelscope.cn/models/Qwen/Qwen2.5-Coder-7B) | Qwen2.5-Math-7B |MMLU-STEM| 65.1% | [67.8%](https://github.com/QwenLM/Qwen2.5-Math/tree/main/) |52| CodeQWen2.5-7B | Human. | 66.5% | [61.6%](https://modelscope.cn/models/Qwen/Qwen2.5-Coder-7B) | Qwen2.5-Math-7B |MMLU-STEM| 65.1% | [67.8%](https://github.com/QwenLM/Qwen2.5-Math/tree/main/) |
53| Qwen2.5-Math-72B |MMLU-STEM| 83.7% | [82.8%](https://github.com/QwenLM/Qwen2.5-Math/tree/main/) | MiniCPM3-4B | MMLU | 63.7% | 64.6% |53| Qwen2.5-Math-72B |MMLU-STEM| 83.7% | [82.8%](https://github.com/QwenLM/Qwen2.5-Math/tree/main/) | MiniCPM3-4B | MMLU | 63.7% | 64.6% |
54+| Phi3.5-Mini | MMLU | 64.39% | 64.34% |
54 55 
55具体的权重转换功能命令介绍见 [MindSpeed-LLM 大模型使用指南分布式评估](../USER_GUIDE.md/#大模型分布式评估)章节.56具体的权重转换功能命令介绍见 [MindSpeed-LLM 大模型使用指南分布式评估](../USER_GUIDE.md/#大模型分布式评估)章节.
56 57 
Mdocs/models/pretrain.md+9-0
@@ -706,6 +706,15 @@
706 <td>【GTS】</td>706 <td>【GTS】</td>
707 <td>【Test】</td>707 <td>【Test】</td>
708 </tr>708 </tr>
709+ <tr>
710+ <td rowspan="2"><a href="https://huggingface.co/microsoft">Phi3.5</a></td>
711+ <td> <a href="https://huggingface.co/microsoft/Phi-3.5-mini-instruct/tree/main">mini</a> </td>
712+ <td> 4K </td>
713+ <th>Mcore</th>
714+ <td> 1x8 </td>
715+ <td>【GTS】</td>
716+ <td>【Test】</td>
717+ </tr>
709 </tbody>718 </tbody>
710</table>719</table>
711 720 
Aexamples/mcore/phi35/ckpt_convert_phi35_mini_hf2mcore.sh+15-0
@@ -0,0 +1,15 @@
1+# 修改 ascend-toolkit 路径
2+source /usr/local/Ascend/ascend-toolkit/set_env.sh
3+ 
4+# 权重格式转换
5+python convert_ckpt.py \
6+ --use-mcore-models \
7+ --model-type-hf phi3.5 \
8+ --model-type GPT \
9+ --load-model-type hf \
10+ --save-model-type mg \
11+ --target-tensor-parallel-size 1 \
12+ --target-pipeline-parallel-size 8 \
13+ --load-dir ./model_from_hf/Phi-3.5-mini-instruct/ \
14+ --save-dir ./model_weights/phi35_mini_mcore/ \
15+ --tokenizer-model ./model_from_hf/Phi-3.5-mini-instruct/tokenizer.model # --num-layers-per-virtual-pipeline-stage 1 等参数根据模型需要添加
Aexamples/mcore/phi35/ckpt_convert_phi35_mini_mcore2hf.sh+13-0
@@ -0,0 +1,13 @@
1+# 请按照您的真实环境修改 set_env.sh 路径
2+source /usr/local/Ascend/ascend-toolkit/set_env.sh
3+ 
4+python convert_ckpt.py \
5+ --use-mcore-models \
6+ --model-type-hf phi3.5 \
7+ --model-type GPT \
8+ --load-model-type mg \
9+ --save-model-type hf \
10+ --target-tensor-parallel-size 1 \
11+ --target-pipeline-parallel-size 1 \
12+ --load-dir ./model_weights/phi35_mini_mcore/ \
13+ --save-dir ./model_from_hf/Phi-3.5-mini-instruct/ \
Aexamples/mcore/phi35/data_convert_phi35_mini_pretrain.sh+11-0
@@ -0,0 +1,11 @@
1+# 请按照您的真实环境修改 set_env.sh 路径
2+source /usr/local/Ascend/ascend-toolkit/set_env.sh
3+mkdir ./dataset
4+ 
5+python ./preprocess_data.py \
6+ --input ./dataset/train-00000-of-00001-a09b74b3ef9c3b56.parquet \
7+ --tokenizer-name-or-path ./model_from_hf/Phi-3.5-mini-instruct \
8+ --output-prefix ./dataset/alpaca_Phi35/alpaca \
9+ --workers 4 \
10+ --log-interval 1000 \
11+ --tokenizer-type PretrainedFromHF
Aexamples/mcore/phi35/evaluate_phi35_mini_ptd.sh+79-0
@@ -0,0 +1,79 @@
1+#!/bin/bash
2+export CUDA_DEVICE_MAX_CONNECTIONS=1
3+ 
4+# distributed config
5+MASTER_ADDR=localhost
6+MASTER_PORT=6001
7+NNODES=1
8+NODE_RANK=0
9+NPUS_PER_NODE=8
10+ 
11+# modify script model path and tokenizer path
12+TOKENIZER_PATH="your tokenizer directory path"
13+CHECKPOINT="your model directory path"
14+DATA_PATH="../mmlu/test/"
15+TASK="mmlu"
16+ 
17+DISTRIBUTED_ARGS="
18+ --nproc_per_node $NPUS_PER_NODE \
19+ --nnodes $NNODES \
20+ --node_rank $NODE_RANK \
21+ --master_addr $MASTER_ADDR \
22+ --master_port $MASTER_PORT
23+"
24+ 
25+LONG_FACTOR="1.0800000429153442,1.1100000143051147,1.1399999856948853,1.340000033378601,1.5899999141693115,1.600000023841858,1.6200000047683716,2.620000123977661,3.2300000190734863,3.2300000190734863,4.789999961853027,7.400000095367432,7.700000286102295,9.09000015258789,12.199999809265137,17.670000076293945,24.46000099182129,28.57000160217285,30.420001983642578,30.840002059936523,32.590003967285156,32.93000411987305,42.320003509521484,44.96000289916992,50.340003967285156,50.45000457763672,57.55000305175781,57.93000411987305,58.21000289916992,60.1400032043457,62.61000442504883,62.62000274658203,62.71000289916992,63.1400032043457,63.1400032043457,63.77000427246094,63.93000411987305,63.96000289916992,63.970001220703125,64.02999877929688,64.06999969482422,64.08000183105469,64.12000274658203,64.41000366210938,64.4800033569336,64.51000213623047,64.52999877929688,64.83999633789062"
26+SHORT_FACTOR="1.0,1.0199999809265137,1.0299999713897705,1.0299999713897705,1.0499999523162842,1.0499999523162842,1.0499999523162842,1.0499999523162842,1.0499999523162842,1.0699999332427979,1.0999999046325684,1.1099998950958252,1.1599998474121094,1.1599998474121094,1.1699998378753662,1.2899998426437378,1.339999794960022,1.679999828338623,1.7899998426437378,1.8199998140335083,1.8499997854232788,1.8799997568130493,1.9099997282028198,1.9399996995925903,1.9899996519088745,2.0199997425079346,2.0199997425079346,2.0199997425079346,2.0199997425079346,2.0199997425079346,2.0199997425079346,2.0299997329711914,2.0299997329711914,2.0299997329711914,2.0299997329711914,2.0299997329711914,2.0299997329711914,2.0299997329711914,2.0299997329711914,2.0299997329711914,2.0799996852874756,2.0899996757507324,2.189999580383301,2.2199995517730713,2.5899994373321533,2.729999542236328,2.749999523162842,2.8399994373321533"
27+ 
28+# configure generation parameters
29+torchrun $DISTRIBUTED_ARGS evaluation.py \
30+ --task-data-path ${DATA_PATH} \
31+ --task ${TASK}\
32+ --load ${CHECKPOINT} \
33+ --use-mcore-models \
34+ --use-flash-attn \
35+ --hidden-dropout 0.0 \
36+ --attention-dropout 0.0 \
37+ --no-chat-template \
38+ --tensor-model-parallel-size 1 \
39+ --pipeline-model-parallel-size 8 \
40+ --evaluation-batch-size 1 \
41+ --exit-on-missing-checkpoint \
42+ --rope-scaling-type longrope \
43+ --longrope-freqs-type outer \
44+ --rope-scaling-original-max-position-embeddings 4096 \
45+ --num-layers 32 \
46+ --hidden-size 3072 \
47+ --ffn-hidden-size 8192 \
48+ --num-attention-heads 32 \
49+ --position-embedding-type rope \
50+ --no-rope-fusion \
51+ --swiglu \
52+ --seq-length 4096 \
53+ --max-position-embeddings 131072 \
54+ --max-new-tokens 1 \
55+ --micro-batch-size 1 \
56+ --tokenizer-type PretrainedFromHF \
57+ --tokenizer-name-or-path ${TOKENIZER_PATH} \
58+ --tokenizer-not-use-fast \
59+ --normalization RMSNorm \
60+ --norm-epsilon 1e-5 \
61+ --disable-bias-linear \
62+ --rotary-base 10000 \
63+ --untie-embeddings-and-output-weights \
64+ --attention-softmax-in-fp32 \
65+ --no-load-optim \
66+ --no-load-rng \
67+ --no-masked-softmax-fusion \
68+ --make-vocab-size-divisible-by 1 \
69+ --padded-vocab-size 32064 \
70+ --seed 42 \
71+ --bf16 \
72+ --long-factor ${LONG_FACTOR} \
73+ --short-factor ${SHORT_FACTOR} \
74+ | tee logs/evaluate_mcore_phi35_mini.log
75+ 
76+ 
77+ 
78+ 
79+ 
Aexamples/mcore/phi35/generate_phi35_mini_ptd.sh+67-0
@@ -0,0 +1,67 @@
1+#!/bin/bash
2+export CUDA_DEVICE_MAX_CONNECTIONS=1
3+ 
4+# Change for multinode config
5+NPUS_PER_NODE=8
6+MASTER_ADDR=localhost
7+MASTER_PORT=6001
8+NNODES=1
9+NODE_RANK=0
10+WORLD_SIZE=$(($NPUS_PER_NODE*$NNODES))
11+ 
12+# please fill these path configurations
13+TOKENIZER_PATH="your tokenizer directory path"
14+CHECKPOINT="your model directory path"
15+ 
16+DISTRIBUTED_ARGS="
17+ --nproc_per_node $NPUS_PER_NODE \
18+ --nnodes $NNODES \
19+ --node_rank $NODE_RANK \
20+ --master_addr $MASTER_ADDR \
21+ --master_port $MASTER_PORT
22+"
23+LONG_FACTOR="1.0800000429153442,1.1100000143051147,1.1399999856948853,1.340000033378601,1.5899999141693115,1.600000023841858,1.6200000047683716,2.620000123977661,3.2300000190734863,3.2300000190734863,4.789999961853027,7.400000095367432,7.700000286102295,9.09000015258789,12.199999809265137,17.670000076293945,24.46000099182129,28.57000160217285,30.420001983642578,30.840002059936523,32.590003967285156,32.93000411987305,42.320003509521484,44.96000289916992,50.340003967285156,50.45000457763672,57.55000305175781,57.93000411987305,58.21000289916992,60.1400032043457,62.61000442504883,62.62000274658203,62.71000289916992,63.1400032043457,63.1400032043457,63.77000427246094,63.93000411987305,63.96000289916992,63.970001220703125,64.02999877929688,64.06999969482422,64.08000183105469,64.12000274658203,64.41000366210938,64.4800033569336,64.51000213623047,64.52999877929688,64.83999633789062"
24+SHORT_FACTOR="1.0,1.0199999809265137,1.0299999713897705,1.0299999713897705,1.0499999523162842,1.0499999523162842,1.0499999523162842,1.0499999523162842,1.0499999523162842,1.0699999332427979,1.0999999046325684,1.1099998950958252,1.1599998474121094,1.1599998474121094,1.1699998378753662,1.2899998426437378,1.339999794960022,1.679999828338623,1.7899998426437378,1.8199998140335083,1.8499997854232788,1.8799997568130493,1.9099997282028198,1.9399996995925903,1.9899996519088745,2.0199997425079346,2.0199997425079346,2.0199997425079346,2.0199997425079346,2.0199997425079346,2.0199997425079346,2.0299997329711914,2.0299997329711914,2.0299997329711914,2.0299997329711914,2.0299997329711914,2.0299997329711914,2.0299997329711914,2.0299997329711914,2.0299997329711914,2.0799996852874756,2.0899996757507324,2.189999580383301,2.2199995517730713,2.5899994373321533,2.729999542236328,2.749999523162842,2.8399994373321533"
25+ 
26+torchrun $DISTRIBUTED_ARGS inference.py \
27+ --tensor-model-parallel-size 1 \
28+ --pipeline-model-parallel-size 8 \
29+ --use-mcore-models \
30+ --use-flash-attn \
31+ --rope-scaling-type longrope \
32+ --longrope-freqs-type outer \
33+ --rope-scaling-original-max-position-embeddings 4096 \
34+ --load ${CHECKPOINT} \
35+ --num-layers 32 \
36+ --hidden-size 3072 \
37+ --ffn-hidden-size 8192 \
38+ --num-attention-heads 32 \
39+ --position-embedding-type rope \
40+ --no-rope-fusion \
41+ --swiglu \
42+ --seq-length 4096 \
43+ --max-position-embeddings 131072 \
44+ --max-new-tokens 256 \
45+ --micro-batch-size 1 \
46+ --tokenizer-type PretrainedFromHF \
47+ --tokenizer-name-or-path ${TOKENIZER_PATH} \
48+ --tokenizer-not-use-fast \
49+ --normalization RMSNorm \
50+ --norm-epsilon 1e-05 \
51+ --disable-bias-linear \
52+ --hidden-dropout 0.0 \
53+ --attention-dropout 0.0 \
54+ --rotary-base 10000 \
55+ --untie-embeddings-and-output-weights \
56+ --attention-softmax-in-fp32 \
57+ --no-load-optim \
58+ --no-load-rng \
59+ --no-masked-softmax-fusion \
60+ --no-gradient-accumulation-fusion \
61+ --exit-on-missing-checkpoint \
62+ --make-vocab-size-divisible-by 1 \
63+ --padded-vocab-size 32064 \
64+ --seed 42 \
65+ --long-factor ${LONG_FACTOR} \
66+ --short-factor ${SHORT_FACTOR} \
67+ | tee logs/generate_mcore_phi35_mini.log
Aexamples/mcore/phi35/pretrain_phi35_mini_ptd.sh+107-0
@@ -0,0 +1,107 @@
1+#!/bin/bash
2+export CUDA_DEVICE_MAX_CONNECTIONS=1
3+ 
4+# distributed config
5+NPUS_PER_NODE=8
6+MASTER_ADDR=localhost
7+MASTER_PORT=6001
8+NNODES=1
9+NODE_RANK=0
10+WORLD_SIZE=$(($NPUS_PER_NODE*$NNODES))
11+ 
12+CKPT_SAVE_DIR="your model save ckpt path"
13+DATA_PATH="your data path"
14+TOKENIZER_MODEL="your tokenizer path"
15+CKPT_LOAD_DIR="your model ckpt path"
16+ 
17+TP=1
18+PP=8
19+ 
20+DISTRIBUTED_ARGS="
21+ --nproc_per_node $NPUS_PER_NODE \
22+ --nnodes $NNODES \
23+ --node_rank $NODE_RANK \
24+ --master_addr $MASTER_ADDR \
25+ --master_port $MASTER_PORT
26+"
27+LONG_FACTOR="1.0800000429153442,1.1100000143051147,1.1399999856948853,1.340000033378601,1.5899999141693115,1.600000023841858,1.6200000047683716,2.620000123977661,3.2300000190734863,3.2300000190734863,4.789999961853027,7.400000095367432,7.700000286102295,9.09000015258789,12.199999809265137,17.670000076293945,24.46000099182129,28.57000160217285,30.420001983642578,30.840002059936523,32.590003967285156,32.93000411987305,42.320003509521484,44.96000289916992,50.340003967285156,50.45000457763672,57.55000305175781,57.93000411987305,58.21000289916992,60.1400032043457,62.61000442504883,62.62000274658203,62.71000289916992,63.1400032043457,63.1400032043457,63.77000427246094,63.93000411987305,63.96000289916992,63.970001220703125,64.02999877929688,64.06999969482422,64.08000183105469,64.12000274658203,64.41000366210938,64.4800033569336,64.51000213623047,64.52999877929688,64.83999633789062"
28+SHORT_FACTOR="1.0,1.0199999809265137,1.0299999713897705,1.0299999713897705,1.0499999523162842,1.0499999523162842,1.0499999523162842,1.0499999523162842,1.0499999523162842,1.0699999332427979,1.0999999046325684,1.1099998950958252,1.1599998474121094,1.1599998474121094,1.1699998378753662,1.2899998426437378,1.339999794960022,1.679999828338623,1.7899998426437378,1.8199998140335083,1.8499997854232788,1.8799997568130493,1.9099997282028198,1.9399996995925903,1.9899996519088745,2.0199997425079346,2.0199997425079346,2.0199997425079346,2.0199997425079346,2.0199997425079346,2.0199997425079346,2.0299997329711914,2.0299997329711914,2.0299997329711914,2.0299997329711914,2.0299997329711914,2.0299997329711914,2.0299997329711914,2.0299997329711914,2.0299997329711914,2.0799996852874756,2.0899996757507324,2.189999580383301,2.2199995517730713,2.5899994373321533,2.729999542236328,2.749999523162842,2.8399994373321533"
29+ 
30+GPT_ARGS="
31+ --tensor-model-parallel-size ${TP} \
32+ --pipeline-model-parallel-size ${PP} \
33+ --num-layers-per-virtual-pipeline-stage 1 \
34+ --sequence-parallel \
35+ --use-mcore-models \
36+ --rope-scaling-type longrope \
37+ --longrope-freqs-type outer \
38+ --rope-scaling-original-max-position-embeddings 4096 \
39+ --use-mc2 \
40+ --use-fused-rmsnorm \
41+ --use-fused-rotary-pos-emb \
42+ --use-fused-swiglu \
43+ --swiglu \
44+ --num-layers 32 \
45+ --hidden-size 3072 \
46+ --ffn-hidden-size 8192 \
47+ --num-attention-heads 32 \
48+ --tokenizer-type PretrainedFromHF \
49+ --tokenizer-name-or-path ${TOKENIZER_MODEL} \
50+ --seq-length 4096 \
51+ --max-position-embeddings 131072 \
52+ --micro-batch-size 2 \
53+ --global-batch-size 128 \
54+ --make-vocab-size-divisible-by 1 \
55+ --lr 1.25e-6 \
56+ --train-iters 2000 \
57+ --lr-decay-style cosine \
58+ --min-lr 1.25e-7 \
59+ --disable-bias-linear \
60+ --attention-dropout 0.0 \
61+ --init-method-std 0.01 \
62+ --hidden-dropout 0.0 \
63+ --position-embedding-type rope \
64+ --normalization RMSNorm \
65+ --norm-epsilon 1e-5 \
66+ --use-flash-attn \
67+ --rotary-base 10000 \
68+ --use-distributed-optimizer \
69+ --no-masked-softmax-fusion \
70+ --attention-softmax-in-fp32 \
71+ --weight-decay 1e-1 \
72+ --lr-warmup-fraction 0.01 \
73+ --clip-grad 1.0 \
74+ --adam-beta1 0.9 \
75+ --adam-beta2 0.95 \
76+ --initial-loss-scale 4096 \
77+ --no-gradient-accumulation-fusion \
78+ --no-load-optim \
79+ --no-load-rng \
80+ --bf16 \
81+ --padded-vocab-size 32064 \
82+ --untie-embeddings-and-output-weights \
83+ --seed 42 \
84+"
85+ 
86+DATA_ARGS="
87+ --data-path $DATA_PATH \
88+ --split 100,0,0
89+"
90+ 
91+OUTPUT_ARGS="
92+ --log-interval 1 \
93+ --save-interval 2000 \
94+ --eval-interval 1000 \
95+ --eval-iters 0 \
96+"
97+ 
98+torchrun $DISTRIBUTED_ARGS pretrain_gpt.py \
99+ $GPT_ARGS \
100+ $DATA_ARGS \
101+ $OUTPUT_ARGS \
102+ --distributed-backend nccl \
103+ --load ${CKPT_LOAD_DIR} \
104+ --save ${CKPT_SAVE_DIR} \
105+ --long-factor ${LONG_FACTOR} \
106+ --short-factor ${SHORT_FACTOR} \
107+ | tee logs/train_phi35_mini_mcore.log
Mmindspeed_llm/core/models/common/embeddings/rotary_pos_embedding.py+13-6
@@ -68,6 +68,7 @@ def rotary_embedding_init_wrapper(fn):
68 68 
69 fn(self, *args, **kwargs)69 fn(self, *args, **kwargs)
70 70 
71+ self.dim = kwargs['kv_channels']
71 if hasattr(_args, "rope_scaling_type") and _args.rope_scaling_type == "llama3":72 if hasattr(_args, "rope_scaling_type") and _args.rope_scaling_type == "llama3":
72 self.inv_freq = apply_llama3_scaling(self.inv_freq)73 self.inv_freq = apply_llama3_scaling(self.inv_freq)
73 elif hasattr(_args, "rope_scaling_type") and _args.rope_scaling_type == "yarn":74 elif hasattr(_args, "rope_scaling_type") and _args.rope_scaling_type == "yarn":
@@ -103,10 +104,16 @@ def rotary_embedding_forward(self, max_seq_len: int, offset: int = 0):
103 else:104 else:
104 ext_factors = torch.tensor(args.short_factor, dtype=torch.float32,105 ext_factors = torch.tensor(args.short_factor, dtype=torch.float32,
105 device=self.inv_freq.device)106 device=self.inv_freq.device)
106- freqs = torch.mul(107+ if args.longrope_freqs_type == "outer":
107- torch.outer(seq, 1.0 / ext_factors).to(device=self.inv_freq.device),108+ self.inv_freq_shape = torch.arange(0, self.dim, 2, dtype=torch.int64,
108- self.inv_freq.to(device=self.inv_freq.device).to(self.inv_freq.dtype)109+ device=torch.cuda.current_device()).float() / self.dim
109- )110+ self.inv_freq = 1.0 / (ext_factors * args.rotary_base ** self.inv_freq_shape)
111+ freqs = torch.outer(seq, self.inv_freq)
112+ else:
113+ freqs = torch.mul(
114+ torch.outer(seq, 1.0 / ext_factors).to(device=self.inv_freq.device),
115+ self.inv_freq.to(device=self.inv_freq.device).to(self.inv_freq.dtype)
116+ )
110 else:117 else:
111 freqs = torch.outer(seq, self.inv_freq)118 freqs = torch.outer(seq, self.inv_freq)
112 # first part even vector components, second part odd vector components,119 # first part even vector components, second part odd vector components,
@@ -187,8 +194,8 @@ def apply_rotary_pos_emb_bshd(t: Tensor, freqs: Tensor, rotary_interleaved: bool
187 )194 )
188 elif args.rope_scaling_type == "longrope":195 elif args.rope_scaling_type == "longrope":
189 scale = args.max_position_embeddings / args.rope_scaling_original_max_position_embeddings196 scale = args.max_position_embeddings / args.rope_scaling_original_max_position_embeddings
190- _mscale = math.sqrt(1 + math.log(scale) /197+ _mscale = 1.0 if scale <= 1.0 else math.sqrt(
191- math.log(args.rope_scaling_original_max_position_embeddings))198+ 1 + math.log(scale) / math.log(args.rope_scaling_original_max_position_embeddings))
192 199 
193 rot_dim = freqs.shape[-1]200 rot_dim = freqs.shape[-1]
194 t, t_pass = t[..., :rot_dim], t[..., rot_dim:]201 t, t_pass = t[..., :rot_dim], t[..., rot_dim:]
Mmindspeed_llm/training/arguments.py+2-0
@@ -425,6 +425,8 @@ def _add_algorithm_args(parser):
425 group.add_argument('--rotary-base', type=float, help='rotary-base.')425 group.add_argument('--rotary-base', type=float, help='rotary-base.')
426 group.add_argument('--rope-scaling-type', type=str, default=None, choices=["llama3", "yarn", "longrope"],426 group.add_argument('--rope-scaling-type', type=str, default=None, choices=["llama3", "yarn", "longrope"],
427 help='The sub-variant of RoPE to use, support type llama3 and yarn and longrope.')427 help='The sub-variant of RoPE to use, support type llama3 and yarn and longrope.')
428+ group.add_argument('--longrope-freqs-type', type=str, default="mul", choices=["mul", "outer"],
429+ help='Specify the frequency type for long rope. Options are "mul" for multiplication or "outer" for outer product.')
428 group.add_argument('--long-factor', type=str, default=None,430 group.add_argument('--long-factor', type=str, default=None,
429 help='rope factor list for long rope scaling type.')431 help='rope factor list for long rope scaling type.')
430 group.add_argument('--short-factor', type=str, default=None,432 group.add_argument('--short-factor', type=str, default=None,