已合并
fix neat-pack bug #2666
jzh6229创建于 2025年5月15日
fix neat-pack bug #2666
已合并
从refs/pull/2666/head合入到2.0.0
共 2 个文件变更+9-1
| @@ -72,7 +72,7 @@ MindSpeed-LLM的依赖配套如下表，安装步骤参考[安装指导](./docs/ | |||
| 72 | </tr> | 72 | </tr> |
| 73 | <tr> | 73 | <tr> |
| 74 | <td>Toolkit（开发套件）</td> | 74 | <td>Toolkit（开发套件）</td> |
| 75 | - <td rowspan="2">CANN 8.0.RC1</td> | 75 | + <td rowspan="2">CANN 8.1.RC1</td> |
| 76 | <td rowspan="2">《CANN 软件安装指南》</td> | 76 | <td rowspan="2">《CANN 软件安装指南》</td> |
| 77 | </tr> | 77 | </tr> |
| 78 | <tr> | 78 | <tr> |
| @@ -31,6 +31,7 @@ from megatron.core.datasets import indexed_dataset | |||
| 31 | 31 | ||
| 32 | from mindspeed_llm.tasks.preprocess.templates import Prompter, AlpacaTemplate, get_model_template | 32 | from mindspeed_llm.tasks.preprocess.templates import Prompter, AlpacaTemplate, get_model_template |
| 33 | from mindspeed_llm.tasks.posttrain.utils import convert_token_to_id | 33 | from mindspeed_llm.tasks.posttrain.utils import convert_token_to_id |
| 34 | +from .decoder_packed_mtf_dataset import _infer_seqlen | ||
| 34 | 35 | ||
| 35 | from .utils import ( | 36 | from .utils import ( |
| 36 | get_dataset_list, | 37 | get_dataset_list, |
| @@ -361,6 +362,13 @@ class LlamaFactoryInstructionHandler(BaseDatasetHandler): | |||
| 361 | else: | 362 | else: |
| 362 | source_mask = [self.ignored_label] * len(source_ids) | 363 | source_mask = [self.ignored_label] * len(source_ids) |
| 363 | 364 | ||
| 365 | + if self.args.neat_pack: | ||
| 366 | + # 只针对单轮超长数据，多轮对话功能待完善 | ||
| 367 | + source_len, target_len = _infer_seqlen(len(source_ids), len(target_ids), self.args.seq_length - 1) | ||
| 368 | + source_ids = source_ids[:source_len] | ||
| 369 | + source_mask = source_mask[:source_len] | ||
| 370 | + target_ids = target_ids[:target_len] | ||
| 371 | + | ||
| 364 | input_ids += source_ids + target_ids | 372 | input_ids += source_ids + target_ids |
| 365 | labels += source_mask + target_ids | 373 | labels += source_mask + target_ids |
| 366 | 374 | ||