已合并
门禁、流水整改三：工具代码框架整合，预留流水打样例子 #1640
AtomGit-Bot创建于 2024年9月9日
门禁、流水整改三：工具代码框架整合，预留流水打样例子 #1640
已合并
从refs/pull/1640/head合入到master
共 123 个文件变更+117-7468
| @@ -1,8 +1,46 @@ | |||
| 1 | import os | 1 | import os |
| 2 | -import stat | ||
| 3 | -import sys | ||
| 4 | from pathlib import Path | 2 | from pathlib import Path |
| 5 | -import subprocess | 3 | + |
| 4 | + | ||
| 5 | +def read_files_from_txt(txt_file): | ||
| 6 | + with open(txt_file, "r") as f: | ||
| 7 | + return [line.strip() for line in f.readlines()] | ||
| 8 | + | ||
| 9 | + | ||
| 10 | +def is_examples(file): | ||
| 11 | + return file.startswith("example/") | ||
| 12 | + | ||
| 13 | + | ||
| 14 | +def is_pipecase(file): | ||
| 15 | + return file.startswith("tests/pipeline") | ||
| 16 | + | ||
| 17 | + | ||
| 18 | +def is_markdown(file): | ||
| 19 | + return file.endswith(".md") | ||
| 20 | + | ||
| 21 | + | ||
| 22 | +def skip_ci_file(files, skip_cond): | ||
| 23 | + for file in files: | ||
| 24 | + if not any(condition(file) for condition in skip_cond): | ||
| 25 | + return False | ||
| 26 | + return True | ||
| 27 | + | ||
| 28 | + | ||
| 29 | +def alter_skip_ci(): | ||
| 30 | + parent_dir = Path(__file__).absolute().parents[2] | ||
| 31 | + raw_txt_file = os.path.join(parent_dir, "modify.txt") | ||
| 32 | + | ||
| 33 | + if not os.path.exists(raw_txt_file): | ||
| 34 | + return False | ||
| 35 | + | ||
| 36 | + file_list = read_files_from_txt(raw_txt_file) | ||
| 37 | + skip_conds = [ | ||
| 38 | + is_examples, | ||
| 39 | + is_pipecase, | ||
| 40 | + is_markdown | ||
| 41 | + ] | ||
| 42 | + | ||
| 43 | + return skip_ci_file(file_list, skip_conds) | ||
| 6 | 44 | ||
| 7 | 45 | ||
| 8 | def acquire_exitcode(command): | 46 | def acquire_exitcode(command): |
| @@ -46,18 +84,8 @@ class ST_Test: | |||
| 46 | 84 | ||
| 47 | st_dir = "st" | 85 | st_dir = "st" |
| 48 | self.st_shell = os.path.join( | 86 | self.st_shell = os.path.join( |
| 49 | - test_dir, st_dir, "run.sh" | 87 | + test_dir, st_dir, "st_run.sh" |
| 50 | ) | 88 | ) |
| 51 | - llama_instruction_shell_file = os.path.join( | ||
| 52 | - test_dir, st_dir, "test_llama_instruction_ptd.sh") | ||
| 53 | - llama_pretrain_ha_save_shell_file = os.path.join( | ||
| 54 | - test_dir, st_dir, "test_llama_pretrain_ha_save_ptd.sh") | ||
| 55 | - llama_pretrain_ha_load_shell_file = os.path.join( | ||
| 56 | - test_dir, st_dir, "test_llama_pretrain_ha_load_ptd.sh") | ||
| 57 | - | ||
| 58 | - self.st_file_list = [ | ||
| 59 | - llama_instruction_shell_file | ||
| 60 | - ] | ||
| 61 | 89 | ||
| 62 | def run_st(self): | 90 | def run_st(self): |
| 63 | rectify_case = f"bash {self.st_shell}" | 91 | rectify_case = f"bash {self.st_shell}" |
| @@ -65,24 +93,22 @@ class ST_Test: | |||
| 65 | if rectify_code != 0: | 93 | if rectify_code != 0: |
| 66 | print("rectify case failed, check it.") | 94 | print("rectify case failed, check it.") |
| 67 | exit(1) | 95 | exit(1) |
| 68 | - all_success = True | ||
| 69 | - for shell_file in self.st_file_list: | ||
| 70 | - command = f"sh {shell_file}" | ||
| 71 | - st_exitcode = acquire_exitcode(command) | ||
| 72 | - if st_exitcode != 0: | ||
| 73 | - all_success = False | ||
| 74 | - print(f"ST run {shell_file} failed") | ||
| 75 | - exit(1) | ||
| 76 | 96 | ||
| 77 | - if all_success: | ||
| 78 | - print("ST test success") | ||
| 79 | - else: | ||
| 80 | - print("ST failed") | ||
| 81 | - exit(1) | ||
| 82 | 97 | ||
| 98 | +def run_tests(): | ||
| 99 | + ut = UT_Test() | ||
| 100 | + st = ST_Test() | ||
| 101 | + | ||
| 102 | + ut.run_ut() | ||
| 103 | + st.run_st() | ||
| 104 | + | ||
| 105 | + | ||
| 106 | +def main(): | ||
| 107 | + if alter_skip_ci(): | ||
| 108 | + print("Skipping CI") | ||
| 109 | + else: | ||
| 110 | + run_tests() | ||
| 83 | 111 | ||
| 84 | if __name__ == "__main__": | 112 | if __name__ == "__main__": |
| 85 | - ut = UT_Test() | 113 | + main() |
| 86 | - ut.run_ut() | 114 | + |
| 87 | - st = ST_Test() | ||
| 88 | - st.run_st() | ||
| @@ -1,124 +0,0 @@ | |||
| 1 | -# GPT3 $\color{black}{\bf\tiny{【社区贡献模型】}}$ | ||
| 2 | - | ||
| 3 | -<p align="left"> | ||
| 4 | - <b>简体中文</b> | | ||
| 5 | - <b><a href="README.md">English</a> </b> | ||
| 6 | -</p> | ||
| 7 | - | ||
| 8 | -# 目录 | ||
| 9 | - | ||
| 10 | -- [GPT3](#GPT3) | ||
| 11 | -- [目录](#目录) | ||
| 12 | -- [GPT3-175B](#GPT3-175B) | ||
| 13 | - - [训练-175B](#训练) | ||
| 14 | - - [脚本](#脚本) | ||
| 15 | - | ||
| 16 | -# GPT3-175B | ||
| 17 | - | ||
| 18 | -## 训练 | ||
| 19 | - | ||
| 20 | -GPT3-175B 训练的硬件配置: | ||
| 21 | - | ||
| 22 | -| 硬件 | 配置 | | ||
| 23 | -| :--: | :-------------: | | ||
| 24 | -| NPU | 128 x Ascend NPUs | | ||
| 25 | - | ||
| 26 | -### 脚本 | ||
| 27 | - | ||
| 28 | -1. 克隆仓库到本地服务器: | ||
| 29 | - | ||
| 30 | - ```shell | ||
| 31 | - git clone https://gitee.com/ascend/ModelLink.git | ||
| 32 | - git clone https://github.com/NVIDIA/Megatron-LM.git | ||
| 33 | - cd Megatron-LM | ||
| 34 | - git checkout core_r0.6.0 | ||
| 35 | - cp -r megatron ../ModelLink/ | ||
| 36 | - cd .. | ||
| 37 | - cd ModelLink | ||
| 38 | - mkdir logs | ||
| 39 | - mkdir vocab_file | ||
| 40 | - mkdir dataset | ||
| 41 | - ``` | ||
| 42 | - | ||
| 43 | -2. 搭建环境 | ||
| 44 | - | ||
| 45 | - ```bash | ||
| 46 | - # python3.8 | ||
| 47 | - conda create -n test python=3.8 | ||
| 48 | - conda activate test | ||
| 49 | - | ||
| 50 | - # 安装 torch 和 torch_npu | ||
| 51 | - pip install torch-2.1.0-cp38-cp38m-manylinux2014_aarch64.whl | ||
| 52 | - pip install torch_npu-2.1.0*-cp38-cp38m-linux_aarch64.whl | ||
| 53 | - pip install apex-0.1_ascend*-cp38-cp38m-linux_aarch64.whl | ||
| 54 | - | ||
| 55 | - # 修改 ascend-toolkit 路径 | ||
| 56 | - source /usr/local/Ascend/ascend-toolkit/set_env.sh | ||
| 57 | - | ||
| 58 | - # 安装 MindSpeed | ||
| 59 | - git clone https://gitee.com/ascend/MindSpeed.git | ||
| 60 | - cd MindSpeed | ||
| 61 | - git checkout 2b0edd2 | ||
| 62 | - pip install -r requirements.txt | ||
| 63 | - pip3 install -e . | ||
| 64 | - cd .. | ||
| 65 | - | ||
| 66 | - # 安装其他依赖 | ||
| 67 | - pip install -r requirements.txt | ||
| 68 | - ``` | ||
| 69 | - | ||
| 70 | -3. 准备数据、词表来拉起模型 | ||
| 71 | - 3.1 准备数据 | ||
| 72 | - | ||
| 73 | - 可以从 [这里](https://huggingface.co/datasets/wikipedia/tree/main/data/20220301.en) 下载原始数据 | ||
| 74 | - ```shell | ||
| 75 | - # 下载 enwiki 数据 | ||
| 76 | - # 总共有 41 个文件，我们可以选择部分来制作数据 | ||
| 77 | - cd ./dataset | ||
| 78 | - wget https://huggingface.co/datasets/wikipedia/blob/main/data/20220301.en/train-00000-of-00041.parquet | ||
| 79 | - wget https://huggingface.co/datasets/wikipedia/blob/main/data/20220301.en/train-00001-of-00041.parquet | ||
| 80 | - wget https://huggingface.co/datasets/wikipedia/blob/main/data/20220301.en/train-00002-of-00041.parquet | ||
| 81 | - wget https://huggingface.co/datasets/wikipedia/blob/main/data/20220301.en/train-00003-of-00041.parquet | ||
| 82 | - wget https://huggingface.co/datasets/wikipedia/blob/main/data/20220301.en/train-00004-of-00041.parquet | ||
| 83 | - wget https://huggingface.co/datasets/wikipedia/blob/main/data/20220301.en/train-00005-of-00041.parquet | ||
| 84 | - wget https://huggingface.co/datasets/wikipedia/blob/main/data/20220301.en/train-00006-of-00041.parquet | ||
| 85 | - wget https://huggingface.co/datasets/wikipedia/blob/main/data/20220301.en/train-00007-of-00041.parquet | ||
| 86 | - cd .. | ||
| 87 | - | ||
| 88 | - # 下载 vocab file 和 merge table | ||
| 89 | - cd vocab_file | ||
| 90 | - wget https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-vocab.json | ||
| 91 | - wget https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-merges.txt | ||
| 92 | - cd .. | ||
| 93 | - | ||
| 94 | - # 处理成训练数据 | ||
| 95 | - python ./preprocess_data.py \ | ||
| 96 | - --input ./dataset/ \ | ||
| 97 | - --output-prefix ./dataset/gpt_text_sentence \ | ||
| 98 | - --tokenizer-type GPT2BPETokenizer \ | ||
| 99 | - --vocab-file ./vocab_file/gpt2-vocab.json \ | ||
| 100 | - --merge-file ./vocab_file/gpt2-merges.txt \ | ||
| 101 | - --append-eod \ | ||
| 102 | - --workers 4 \ | ||
| 103 | - --log-interval 1000 | ||
| 104 | - ``` | ||
| 105 | - | ||
| 106 | - 3.2 用 ptd 模式进行预训练 | ||
| 107 | - 配置 GPT3-175B PTD 预训练脚本: examples/gpt3/pretrain_gpt3_175B_ptd.sh | ||
| 108 | - | ||
| 109 | - ```shell | ||
| 110 | - # 请根据真实情况配置 ascend-toolkit 路径 | ||
| 111 | - source /usr/local/Ascend/ascend-toolkit/set_env.sh | ||
| 112 | - | ||
| 113 | - # 请根据真实存放路径配置以下参数 | ||
| 114 | - VOCAB_FILE="./vocab_file/gpt2-vocab.json" # 词表 | ||
| 115 | - MERGE_FILE="./vocab_file/gpt2-merges.txt" # BPE 合并表 | ||
| 116 | - DATA_PATH="./dataset/gpt_text_sentence" # 数据路径 | ||
| 117 | - ``` | ||
| 118 | - | ||
| 119 | - 拉起 GPT3-175B PTD 预训练脚本: examples/gpt3/pretrain_gpt3_175B_ptd.sh | ||
| 120 | - | ||
| 121 | - ```shell | ||
| 122 | - bash examples/gpt3/pretrain_gpt3_175B_ptd.sh | ||
| 123 | - ``` | ||
| 124 | - | ||
| @@ -1,124 +0,0 @@ | |||
| 1 | -# GPT3 $\color{black}{\rm\tiny{【model}}$ $\color{black}{\rm\tiny{contributed}}$ $\color{black}{\rm\tiny{by}}$ $\color{black}{\rm\tiny{Community】}}$ | ||
| 2 | - | ||
| 3 | -<p align="left"> | ||
| 4 | - <b>English</b> | | ||
| 5 | - <b><a href="README_en.md">English</a> </b> | ||
| 6 | -</p> | ||
| 7 | - | ||
| 8 | -# Contents | ||
| 9 | - | ||
| 10 | -- [GPT3](#GPT3) | ||
| 11 | -- [Contents](#contents) | ||
| 12 | -- [GPT3-175B](#GPT3-175B) | ||
| 13 | - - [Training-175B](#training) | ||
| 14 | - - [Script](#script) | ||
| 15 | - | ||
| 16 | -# GPT3-175B | ||
| 17 | - | ||
| 18 | -## Training | ||
| 19 | - | ||
| 20 | -Here is a hardware summary of pre-trianing GPT3-175B: | ||
| 21 | - | ||
| 22 | -| Hardware | Value | | ||
| 23 | -| :--: | :-------------: | | ||
| 24 | -| NPU | 128 x Ascend NPUs | | ||
| 25 | - | ||
| 26 | -### Script | ||
| 27 | - | ||
| 28 | -1. Clone repository to your local server: | ||
| 29 | - | ||
| 30 | - ```shell | ||
| 31 | - git clone https://gitee.com/ascend/ModelLink.git | ||
| 32 | - git clone https://github.com/NVIDIA/Megatron-LM.git | ||
| 33 | - cd Megatron-LM | ||
| 34 | - git checkout core_r0.6.0 | ||
| 35 | - cp -r megatron ../ModelLink/ | ||
| 36 | - cd .. | ||
| 37 | - cd ModelLink | ||
| 38 | - mkdir logs | ||
| 39 | - mkdir vocab_file | ||
| 40 | - mkdir dataset | ||
| 41 | - ``` | ||
| 42 | - | ||
| 43 | -2. Build environment | ||
| 44 | - | ||
| 45 | - ```bash | ||
| 46 | - # python3.8 | ||
| 47 | - conda create -n test python=3.8 | ||
| 48 | - conda activate test | ||
| 49 | - | ||
| 50 | - # install torch and torch_npu | ||
| 51 | - pip install torch-2.1.0-cp38-cp38m-manylinux2014_aarch64.whl | ||
| 52 | - pip install torch_npu-2.1.0*-cp38-cp38m-linux_aarch64.whl | ||
| 53 | - pip install apex-0.1_ascend*-cp38-cp38m-linux_aarch64.whl | ||
| 54 | - | ||
| 55 | - # modify ascend-toolkit path | ||
| 56 | - source /usr/local/Ascend/ascend-toolkit/set_env.sh | ||
| 57 | - | ||
| 58 | - # install MindSpeed | ||
| 59 | - git clone https://gitee.com/ascend/MindSpeed.git | ||
| 60 | - cd MindSpeed | ||
| 61 | - git checkout 2b0edd2 | ||
| 62 | - pip install -r requirements.txt | ||
| 63 | - pip3 install -e . | ||
| 64 | - cd .. | ||
| 65 | - | ||
| 66 | - # install other packages | ||
| 67 | - pip install -r requirements.txt | ||
| 68 | - ``` | ||
| 69 | - | ||
| 70 | -3. Prepare dataset and vocab file for pretrain | ||
| 71 | - 3.1 Prepare dataset | ||
| 72 | - | ||
| 73 | - Download the GPT raw dataset from [here](https://huggingface.co/datasets/wikipedia/tree/main/data/20220301.en) | ||
| 74 | - ```shell | ||
| 75 | - # download enwiki raw data | ||
| 76 | - # There are 41 files in total, we can just select part to make our datasets. | ||
| 77 | - cd ./dataset | ||
| 78 | - wget https://huggingface.co/datasets/wikipedia/blob/main/data/20220301.en/train-00000-of-00041.parquet | ||
| 79 | - wget https://huggingface.co/datasets/wikipedia/blob/main/data/20220301.en/train-00001-of-00041.parquet | ||
| 80 | - wget https://huggingface.co/datasets/wikipedia/blob/main/data/20220301.en/train-00002-of-00041.parquet | ||
| 81 | - wget https://huggingface.co/datasets/wikipedia/blob/main/data/20220301.en/train-00003-of-00041.parquet | ||
| 82 | - wget https://huggingface.co/datasets/wikipedia/blob/main/data/20220301.en/train-00004-of-00041.parquet | ||
| 83 | - wget https://huggingface.co/datasets/wikipedia/blob/main/data/20220301.en/train-00005-of-00041.parquet | ||
| 84 | - wget https://huggingface.co/datasets/wikipedia/blob/main/data/20220301.en/train-00006-of-00041.parquet | ||
| 85 | - wget https://huggingface.co/datasets/wikipedia/blob/main/data/20220301.en/train-00007-of-00041.parquet | ||
| 86 | - cd .. | ||
| 87 | - | ||
| 88 | - # download vocab file and merge table | ||
| 89 | - cd vocab_file | ||
| 90 | - wget https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-vocab.json | ||
| 91 | - wget https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-merges.txt | ||
| 92 | - cd .. | ||
| 93 | - | ||
| 94 | - # process formal dataset | ||
| 95 | - python ./preprocess_data.py \ | ||
| 96 | - --input ./dataset/ \ | ||
| 97 | - --output-prefix ./dataset/gpt_text_sentence \ | ||
| 98 | - --tokenizer-type GPT2BPETokenizer \ | ||
| 99 | - --vocab-file ./vocab_file/gpt2-vocab.json \ | ||
| 100 | - --merge-file ./vocab_file/gpt2-merges.txt \ | ||
| 101 | - --append-eod \ | ||
| 102 | - --workers 4 \ | ||
| 103 | - --log-interval 1000 | ||
| 104 | - ``` | ||
| 105 | - | ||
| 106 | - 3.2 pre-training in ptd mode | ||
| 107 | - Config GPT3-175B PTD pre-training script: examples/gpt3/pretrain_gpt3_175B_ptd.sh | ||
| 108 | - | ||
| 109 | - ```shell | ||
| 110 | - # modify ascend-toolkit path according to your own config | ||
| 111 | - source /usr/local/Ascend/ascend-toolkit/set_env.sh | ||
| 112 | - | ||
| 113 | - # modify config according to your own actual situation | ||
| 114 | - VOCAB_FILE="./vocab_file/gpt2-vocab.json" # vocab file for training | ||
| 115 | - MERGE_FILE="./vocab_file/gpt2-merges.txt" # BPE merge file for training | ||
| 116 | - DATA_PATH="./dataset/gpt_text_sentence" # dataset path | ||
| 117 | - ``` | ||
| 118 | - | ||
| 119 | - Launch GPT3-175B PTD pre-training script: examples/gpt3/pretrain_gpt3_175B_ptd.sh | ||
| 120 | - | ||
| 121 | - ```shell | ||
| 122 | - bash examples/gpt3/pretrain_gpt3_175B_ptd.sh | ||
| 123 | - ``` | ||
| 124 | - | ||
| @@ -84,6 +84,5 @@ torchrun $DISTRIBUTED_ARGS pretrain_gpt.py \ | |||
| 84 | $GPT_ARGS \ | 84 | $GPT_ARGS \ |
| 85 | $DATA_ARGS \ | 85 | $DATA_ARGS \ |
| 86 | $OUTPUT_ARGS \ | 86 | $OUTPUT_ARGS \ |
| 87 | - --jit-compile \ | 87 | + --distributed-backend nccl \ |
| 88 | - --distributed-backend nccl 2>&1 | ||
| 89 | | tee ./logs/pretrain_gpt3_175B_8layers.log | 88 | | tee ./logs/pretrain_gpt3_175B_8layers.log |
| @@ -85,6 +85,5 @@ torchrun $DISTRIBUTED_ARGS pretrain_gpt.py \ | |||
| 85 | $GPT_ARGS \ | 85 | $GPT_ARGS \ |
| 86 | $DATA_ARGS \ | 86 | $DATA_ARGS \ |
| 87 | $OUTPUT_ARGS \ | 87 | $OUTPUT_ARGS \ |
| 88 | - --jit-compile \ | 88 | + --distributed-backend nccl \ |
| 89 | - --distributed-backend nccl 2>&1 | ||
| 90 | | tee ./logs/pretrain_gpt3_175B.log | 89 | | tee ./logs/pretrain_gpt3_175B.log |
| @@ -41,3 +41,30 @@ def pytest_fixture_setup(fixturedef, request): | |||
| 41 | if getattr(fixturedef.func, "is_dist_fixture", False): | 41 | if getattr(fixturedef.func, "is_dist_fixture", False): |
| 42 | dist_fixture_class = fixturedef.func() | 42 | dist_fixture_class = fixturedef.func() |
| 43 | dist_fixture_class(request) | 43 | dist_fixture_class(request) |
| 44 | + | ||
| 45 | + | ||
| 46 | +# we still want to configure path argument by ourselves | ||
| 47 | +# for different prefix_name of different scripts so we use this method. | ||
| 48 | +# One more thing, as you can see, it has more scalibility. | ||
| 49 | +def pytest_addoption(parser: pytest.Parser): | ||
| 50 | + parser.addoption("--baseline-json", action="store", default=None, | ||
| 51 | + help="Path to the baseline JSON file") | ||
| 52 | + parser.addoption("--generate-log", action="store", default=None, | ||
| 53 | + help="Path to the generate log file") | ||
| 54 | + parser.addoption("--generate-json", action="store", default=None, | ||
| 55 | + help="Path to the generate JSON file") | ||
| 56 | + | ||
| 57 | + | ||
| 58 | +@pytest.fixture(autouse=True) | ||
| 59 | +def baseline_json(request: pytest.FixtureRequest): | ||
| 60 | + return request.config.getoption("--baseline-json") | ||
| 61 | + | ||
| 62 | + | ||
| 63 | +@pytest.fixture(autouse=True) | ||
| 64 | +def generate_log(request: pytest.FixtureRequest): | ||
| 65 | + return request.config.getoption("--generate-log") | ||
| 66 | + | ||
| 67 | + | ||
| 68 | +@pytest.fixture(autouse=True) | ||
| 69 | +def generate_json(request: pytest.FixtureRequest): | ||
| 70 | + return request.config.getoption("--generate-json") | ||
| @@ -1,6 +0,0 @@ | |||
| 1 | -#!/bin/bash | ||
| 2 | -# Provide uniform access for piepline. | ||
| 3 | - | ||
| 4 | -python tests/pipeline/Aquila2-7B/test_convert_weight_from_huggingface.py | ||
| 5 | -pytest -s tests/pipeline/Aquila2-7B/test_generation.py | ||
| 6 | -pytest -s tests/pipeline/Aquila2-7B/test_evaluation.py | ||
| @@ -1,60 +0,0 @@ | |||
| 1 | -{ | ||
| 2 | - "NETWORK_SIZE": [ | ||
| 3 | - "--num-layers", "32", | ||
| 4 | - "--hidden-size", "4096", | ||
| 5 | - "--ffn-hidden-size", "11008", | ||
| 6 | - "--num-attention-heads", "32", | ||
| 7 | - "--position-embedding-type", "rope", | ||
| 8 | - "--make-vocab-size-divisible-by", "1", | ||
| 9 | - "--max-position-embeddings", "2048", | ||
| 10 | - "--normalization", "RMSNorm", | ||
| 11 | - "--swiglu", | ||
| 12 | - "--untie-embeddings-and-output-weights", | ||
| 13 | - "--load", "/home/dataset/Aquila2-7B-tp8-pp1/" | ||
| 14 | - ], | ||
| 15 | - | ||
| 16 | - "TOKENIZER_PARAM": [ | ||
| 17 | - "--tokenizer-type", "PretrainedFromHF", | ||
| 18 | - "--tokenizer-name-or-path", "/home/dataset/Aquila2-7B-hf" | ||
| 19 | - ], | ||
| 20 | - | ||
| 21 | - "INFERENCE_PARAM": [ | ||
| 22 | - "--max-new-tokens", "512", | ||
| 23 | - "--tokenizer-not-use-fast", | ||
| 24 | - "--exit-on-missing-checkpoint" | ||
| 25 | - ], | ||
| 26 | - | ||
| 27 | - "EVALUATION_PARAM": [ | ||
| 28 | - "--tokenizer-not-use-fast", | ||
| 29 | - "--task-data-path", "/home/dataset/eval_dataset/boolq/test", | ||
| 30 | - "--task", "boolq", | ||
| 31 | - "--max-new-tokens", "1" | ||
| 32 | - ], | ||
| 33 | - | ||
| 34 | - "DISTRIBUTED_PARAM": [ | ||
| 35 | - "--tensor-model-parallel-size", "8", | ||
| 36 | - "--pipeline-model-parallel-size", "1" | ||
| 37 | - ], | ||
| 38 | - | ||
| 39 | - "AUXILIARY_PARAM": [ | ||
| 40 | - "--micro-batch-size", "8", | ||
| 41 | - "--global-batch-size", "64", | ||
| 42 | - "--no-masked-softmax-fusion", | ||
| 43 | - "--disable-bias-linear", | ||
| 44 | - "--no-gradient-accumulation-fusion", | ||
| 45 | - "--attention-softmax-in-fp32", | ||
| 46 | - "--no-load-optim", | ||
| 47 | - "--no-load-rng", | ||
| 48 | - "--seq-length", "2048" | ||
| 49 | - ], | ||
| 50 | - | ||
| 51 | - "CONVERT_WEIGHT_PARAM": [ | ||
| 52 | - "--model-type", "GPT", | ||
| 53 | - "--load-dir", "/home/dataset/Aquila2-7B-hf", | ||
| 54 | - "--save-dir", "/home/dataset/Aquila2-7B-tp8-pp1", | ||
| 55 | - "--loader", "llama2_hf", | ||
| 56 | - "--saver", "megatron", | ||
| 57 | - "--target-tensor-parallel-size", "8", | ||
| 58 | - "--tokenizer-model", "/home/dataset/Aquila2-7B-hf/tokenizer.json" | ||
| 59 | - ] | ||
| 60 | -} | ||
| @@ -1,64 +0,0 @@ | |||
| 1 | -import unittest | ||
| 2 | -import sys | ||
| 3 | -import os | ||
| 4 | -import subprocess | ||
| 5 | -import glob | ||
| 6 | -from pathlib import Path | ||
| 7 | -import torch | ||
| 8 | -from utils import ParamConfig | ||
| 9 | - | ||
| 10 | -import modellink | ||
| 11 | - | ||
| 12 | - | ||
| 13 | -class TestConvertCkptFromHuggingface(unittest.TestCase): | ||
| 14 | - def setUp(self): | ||
| 15 | - # configure params, the index starts from 1 | ||
| 16 | - sys.argv = [sys.argv[0]] + ParamConfig.convert_weight_param | ||
| 17 | - | ||
| 18 | - def test_file_exsit(self): | ||
| 19 | - """ | ||
| 20 | - Test if the file in the `--load-dir` exsit, including `.bin`, `.json`... | ||
| 21 | - """ | ||
| 22 | - bin_file = glob.glob(os.path.join(sys.argv[4], "*.bin")) | ||
| 23 | - self.assertEqual(len(bin_file), 3) | ||
| 24 | - self.assertTrue(os.path.exists(os.path.join(sys.argv[4], "pytorch_model.bin.index.json"))) | ||
| 25 | - | ||
| 26 | - def test_convert_weights_form_huggingface(self): | ||
| 27 | - """ | ||
| 28 | - Test whether the weight to be converted as we want in `--save-dir`. We will check the model layer name, | ||
| 29 | - including embedding, final_norm, output and encoder. In the encoder, there will be some different layers | ||
| 30 | - to compose the unique transformer layer and all these layer stack to compose the entity of the model. | ||
| 31 | - """ | ||
| 32 | - base_dir = Path(__file__).absolute().parent.parent.parent.parent | ||
| 33 | - file_path = os.path.join(base_dir, "convert_ckpt.py") | ||
| 34 | - arguments = sys.argv[1:] | ||
| 35 | - subprocess.run(["python", file_path] + arguments) | ||
| 36 | - output_dir = os.path.join(sys.argv[6], "iter_0000001") | ||
| 37 | - weight_content = torch.load(os.path.join(output_dir, "mp_rank_00/model_optim_rng.pt")) | ||
| 38 | - weight_common_content = weight_content['model']['language_model'] # extract commmon content | ||
| 39 | - | ||
| 40 | - # embedding, encoder, output_layer is three out layers. | ||
| 41 | - self.assertEqual(len(os.listdir(output_dir)), int(sys.argv[12])) | ||
| 42 | - self.assertEqual(weight_common_content['embedding']['word_embeddings']['weight'].size(), | ||
| 43 | - torch.Size([12501, 4096])) | ||
| 44 | - self.assertEqual(weight_common_content['encoder']['final_norm.weight'].size(), torch.Size([4096])) | ||
| 45 | - | ||
| 46 | - # encoder has a common final_norm and each one has folliowing six layers | ||
| 47 | - weight_common_content['encoder'].pop('final_norm.weight') | ||
| 48 | - self.assertEqual(weight_common_content['encoder']['layers.0.self_attention.query_key_value.weight'].size(), | ||
| 49 | - torch.Size([1536, 4096])) | ||
| 50 | - self.assertEqual(weight_common_content['encoder']['layers.0.self_attention.dense.weight'].size(), | ||
| 51 | - torch.Size([4096, 512])) | ||
| 52 | - self.assertEqual(weight_common_content['encoder']['layers.0.mlp.dense_h_to_4h.weight'].size(), | ||
| 53 | - torch.Size([2752, 4096])) | ||
| 54 | - self.assertEqual(weight_common_content['encoder']['layers.0.mlp.dense_4h_to_h.weight'].size(), | ||
| 55 | - torch.Size([4096, 1376])) | ||
| 56 | - self.assertEqual(weight_common_content['encoder']['layers.0.input_norm.weight'].size(), torch.Size([4096])) | ||
| 57 | - self.assertEqual(weight_common_content['encoder']['layers.0.post_attention_norm.weight'].size(), | ||
| 58 | - torch.Size([4096])) | ||
| 59 | - | ||
| 60 | - self.assertEqual(weight_common_content['output_layer']['weight'].size(), torch.Size([12501, 4096])) | ||
| 61 | - | ||
| 62 | - | ||
| 63 | -if __name__ == "__main__": | ||
| 64 | - unittest.main() | ||
| @@ -1,100 +0,0 @@ | |||
| 1 | -import sys | ||
| 2 | -import os | ||
| 3 | -import json | ||
| 4 | -import torch | ||
| 5 | -import tqdm | ||
| 6 | -import torch_npu | ||
| 7 | -from utils import ParamConfig, assert_judge | ||
| 8 | -from transformers import AutoTokenizer | ||
| 9 | -import modellink | ||
| 10 | -from megatron.legacy.model import GPTModel | ||
| 11 | -from tests.common import DistributedTest | ||
| 12 | -from modellink.tasks.evaluation.utils import add_text_generate_args | ||
| 13 | - | ||
| 14 | - | ||
| 15 | -class TestEvaluation(DistributedTest): | ||
| 16 | - world_size = 8 | ||
| 17 | - | ||
| 18 | - def init(self, config=ParamConfig): | ||
| 19 | - sys.argv = [sys.argv[0]] + config.distributed_param + config.network_size + \ | ||
| 20 | - config.evaluation_param + config.auxiliary_param + config.tokenizer_param | ||
| 21 | - from megatron.training.initialize import initialize_megatron | ||
| 22 | - os.environ.update({"CUDA_DEVICE_MAX_CONNECTIONS": "1"}) | ||
| 23 | - initialize_megatron(extra_args_provider=add_text_generate_args, | ||
| 24 | - args_defaults={'no_load_rng': True, | ||
| 25 | - 'no_load_optim': True}) | ||
| 26 | - from megatron.training import get_args | ||
| 27 | - self.args = get_args() | ||
| 28 | - | ||
| 29 | - def get_result(self, tokenizer, result): | ||
| 30 | - if result: | ||
| 31 | - final_result = [result[0]] | ||
| 32 | - if result[1][0][tokenizer.encode("Yes")[-1]] >= result[1][0][tokenizer.encode("No")[-1]]: | ||
| 33 | - final_result.append("T") | ||
| 34 | - else: | ||
| 35 | - final_result.append("F") | ||
| 36 | - else: | ||
| 37 | - final_result = None | ||
| 38 | - return final_result | ||
| 39 | - | ||
| 40 | - def test_boolq_evaluation(self): | ||
| 41 | - self.init(config=ParamConfig) | ||
| 42 | - from evaluation import model_provider | ||
| 43 | - model = GPTModel.from_pretrained( | ||
| 44 | - model_provider=model_provider, | ||
| 45 | - pretrained_model_name_or_path=self.args.load | ||
| 46 | - ) | ||
| 47 | - tokenizer = AutoTokenizer.from_pretrained(self.args.tokenizer_name_or_path, trust_remote_code=True) | ||
| 48 | - max_new_tokens = self.args.max_new_tokens | ||
| 49 | - | ||
| 50 | - instruction_template = "{passage}\nQuestion: {question}?\nAnswer:" | ||
| 51 | - | ||
| 52 | - answer_result = {} | ||
| 53 | - total_acc_n = 0 | ||
| 54 | - total_n = 0 | ||
| 55 | - test_dir = None | ||
| 56 | - for path in self.args.task_data_path: | ||
| 57 | - if "boolq" in path: | ||
| 58 | - test_dir = path | ||
| 59 | - for file in tqdm.tqdm(os.listdir(test_dir)): | ||
| 60 | - file_path = os.path.join(test_dir, file) | ||
| 61 | - with open(file_path, encoding='utf-8') as f: | ||
| 62 | - boolq_question_list = [] | ||
| 63 | - for line in f.readlines(): | ||
| 64 | - boolq_question_list.append(json.loads(line)) | ||
| 65 | - boolq_question_list = boolq_question_list[:60] | ||
| 66 | - subject_result = {} | ||
| 67 | - acc_n = 0 | ||
| 68 | - for index, item in enumerate(boolq_question_list): | ||
| 69 | - instruction = instruction_template.format(passage=item['passage'], question=item['question']) | ||
| 70 | - result = model.generate( | ||
| 71 | - instruction, | ||
| 72 | - do_sample=False, | ||
| 73 | - max_new_tokens=max_new_tokens, | ||
| 74 | - tokenizer=tokenizer, | ||
| 75 | - stream=False, | ||
| 76 | - return_output_log_probs=True | ||
| 77 | - ) | ||
| 78 | - result = self.get_result(tokenizer, result) | ||
| 79 | - if result: | ||
| 80 | - answer = result[1] | ||
| 81 | - else: | ||
| 82 | - answer = None | ||
| 83 | - try: | ||
| 84 | - if torch.distributed.get_rank() == 0: | ||
| 85 | - subject_result[str(index)] = answer | ||
| 86 | - if subject_result[str(index)] == str(item['answer'])[0]: | ||
| 87 | - acc_n += 1 | ||
| 88 | - except Exception as e: | ||
| 89 | - raise e | ||
| 90 | - if torch.distributed.get_rank() == 0: | ||
| 91 | - total_n += len(boolq_question_list) | ||
| 92 | - total_acc_n += acc_n | ||
| 93 | - answer_result['Boolq_dataset'] = subject_result | ||
| 94 | - if torch.distributed.get_rank() == 0: | ||
| 95 | - try: | ||
| 96 | - final_acc = total_acc_n / total_n | ||
| 97 | - except ZeroDivisionError as e: | ||
| 98 | - raise e | ||
| 99 | - print(final_acc) | ||
| 100 | - assert_judge(abs(final_acc - 0.83) < 0.03) | ||
| @@ -1,93 +0,0 @@ | |||
| 1 | -import sys | ||
| 2 | -import os | ||
| 3 | -import torch | ||
| 4 | -import torch_npu | ||
| 5 | -from utils import ParamConfig, assert_judge | ||
| 6 | -import modellink | ||
| 7 | -from megatron.legacy.model import GPTModel | ||
| 8 | -from modellink.tasks.inference.text_generation.infer_base import add_text_generate_args | ||
| 9 | -from tests.common import DistributedTest | ||
| 10 | - | ||
| 11 | - | ||
| 12 | -class TestGeneration(DistributedTest): | ||
| 13 | - world_size = 8 | ||
| 14 | - | ||
| 15 | - def init(self, config=ParamConfig): | ||
| 16 | - """ | ||
| 17 | - initialize the environment and arguments | ||
| 18 | - """ | ||
| 19 | - sys.argv = [sys.argv[0]] + config.distributed_param + config.network_size + \ | ||
| 20 | - config.inference_param + config.auxiliary_param + config.tokenizer_param | ||
| 21 | - from megatron.training.initialize import initialize_megatron | ||
| 22 | - os.environ.update({"CUDA_DEVICE_MAX_CONNECTIONS": "1"}) | ||
| 23 | - initialize_megatron(extra_args_provider=add_text_generate_args, | ||
| 24 | - args_defaults={'no_load_rng': True, | ||
| 25 | - 'no_load_optim': True}) | ||
| 26 | - from megatron.training import get_args | ||
| 27 | - self.args = get_args() | ||
| 28 | - | ||
| 29 | - def test_greedy_search(self): | ||
| 30 | - """ | ||
| 31 | - load weight to get model and construct the prompts to generate output, | ||
| 32 | - and compare with expected for `greedy search`. | ||
| 33 | - """ | ||
| 34 | - self.init(config=ParamConfig) | ||
| 35 | - from inference import model_provider | ||
| 36 | - model = GPTModel.from_pretrained( | ||
| 37 | - model_provider=model_provider, | ||
| 38 | - pretrained_model_name_or_path=self.args.load | ||
| 39 | - ) | ||
| 40 | - instruction = ["解释一下“温故而知新”"] | ||
| 41 | - output = model.generate(instruction, detokenize=False) | ||
| 42 | - expected_output = [14727, 29728, 261, 22051, 1156, 3101, 422, 278, 0, 87, | ||
| 43 | - 18392, 221, 1906, 358, 132, 237, 79, 221, 261, 81, | ||
| 44 | - 14572, 2449, 2369, 72, 8022, 2449, 221, 261, 88, 14572] | ||
| 45 | - | ||
| 46 | - if torch.distributed.get_rank() == 0: | ||
| 47 | - print(output) | ||
| 48 | - similarity = torch.nn.CosineSimilarity(dim=1) | ||
| 49 | - cos_sim = similarity(torch.tensor(expected_output)[:20].unsqueeze(0).float().npu(), | ||
| 50 | - output[:20].unsqueeze(0).float()) | ||
| 51 | - print(cos_sim) | ||
| 52 | - assert_judge(cos_sim > 0.80) | ||
| 53 | - | ||
| 54 | - def test_beam_search(self): | ||
| 55 | - """ | ||
| 56 | - load weight to get model and construct the prompts to generate output, | ||
| 57 | - and compare with expected for `beam search`. | ||
| 58 | - """ | ||
| 59 | - self.init(config=ParamConfig) | ||
| 60 | - from inference import model_provider | ||
| 61 | - model = GPTModel.from_pretrained( | ||
| 62 | - model_provider=model_provider, | ||
| 63 | - pretrained_model_name_or_path=self.args.load | ||
| 64 | - ) | ||
| 65 | - | ||
| 66 | - max_new_tokens = self.args.max_new_tokens | ||
| 67 | - prompt = "解释一下“温故而知新”" | ||
| 68 | - system_template = "" | ||
| 69 | - dialog_template = "{instruction}" | ||
| 70 | - template = system_template + dialog_template | ||
| 71 | - instruction = template.format(instruction=prompt) | ||
| 72 | - | ||
| 73 | - output = model.generate( | ||
| 74 | - instruction, | ||
| 75 | - num_beams=2, | ||
| 76 | - top_k=self.args.top_k, | ||
| 77 | - top_p=self.args.top_p, | ||
| 78 | - max_new_tokens=max_new_tokens, | ||
| 79 | - tokenizer=None, | ||
| 80 | - stream=False, | ||
| 81 | - detokenize=False | ||
| 82 | - ) | ||
| 83 | - expected_output = [391, 426, 1774, 1906, 8627, 10281, 261, 36018, 837, 42310, | ||
| 84 | - 434, 10045, 6468, 29728, 278, 0, 43652, 43652, 1774, 1906, | ||
| 85 | - 28043, 609, 43652, 43652, 1774, 1906, 8627, 10281, 261, 36018] | ||
| 86 | - | ||
| 87 | - if torch.distributed.get_rank() == 0: | ||
| 88 | - print(output) | ||
| 89 | - similarity = torch.nn.CosineSimilarity(dim=1) | ||
| 90 | - cos_sim = similarity(torch.tensor(expected_output)[:20].unsqueeze(0).float().npu(), | ||
| 91 | - output[:20].unsqueeze(0).float()) | ||
| 92 | - print(cos_sim) | ||
| 93 | - assert_judge(cos_sim > 0.80) | ||
| @@ -1,25 +0,0 @@ | |||
| 1 | -import json | ||
| 2 | -import os | ||
| 3 | -from pathlib import Path | ||
| 4 | -from dataclasses import dataclass | ||
| 5 | - | ||
| 6 | - | ||
| 7 | -@dataclass | ||
| 8 | -class ParamConfig: | ||
| 9 | - base_dir = Path(__file__).absolute().parent | ||
| 10 | - param_config = os.path.join(base_dir, "param_config.json") | ||
| 11 | - with open(param_config) as f: | ||
| 12 | - config_file = json.load(f) | ||
| 13 | - | ||
| 14 | - distributed_param = config_file["DISTRIBUTED_PARAM"] | ||
| 15 | - network_size = config_file["NETWORK_SIZE"] | ||
| 16 | - inference_param = config_file["INFERENCE_PARAM"] | ||
| 17 | - evaluation_param = config_file["EVALUATION_PARAM"] | ||
| 18 | - auxiliary_param = config_file["AUXILIARY_PARAM"] | ||
| 19 | - tokenizer_param = config_file["TOKENIZER_PARAM"] | ||
| 20 | - convert_weight_param = config_file["CONVERT_WEIGHT_PARAM"] | ||
| 21 | - | ||
| 22 | - | ||
| 23 | -def assert_judge(expression): | ||
| 24 | - if not expression: | ||
| 25 | - raise AssertionError | ||
| @@ -1,10 +0,0 @@ | |||
| 1 | -#!/bin/bash | ||
| 2 | -# Provide uniform access for piepline. | ||
| 3 | - | ||
| 4 | -python tests/pipeline/baichuan-13B/test_process_pretrain_data.py | ||
| 5 | -python tests/pipeline/baichuan-13B/test_process_instruction_data.py | ||
| 6 | -python tests/pipeline/baichuan-13B/test_convert_weight_from_huggingface.py | ||
| 7 | -pytest -s tests/pipeline/baichuan-13B/test_generation.py | ||
| 8 | -pytest -s tests/pipeline/baichuan-13B/test_evaluation.py | ||
| 9 | -pytest -s tests/pipeline/baichuan-13B/test_lora.py | ||
| 10 | -pytest -s tests/pipeline/baichuan-13B/test_trainer.py | ||
| @@ -1,136 +0,0 @@ | |||
| 1 | -{ | ||
| 2 | - "NETWORK_SIZE": [ | ||
| 3 | - "--num-layers", "40", | ||
| 4 | - "--hidden-size", "5120", | ||
| 5 | - "--ffn-hidden-size", "13696", | ||
| 6 | - "--num-attention-heads", "40", | ||
| 7 | - "--position-embedding-type", "alibi", | ||
| 8 | - "--make-vocab-size-divisible-by", "64", | ||
| 9 | - "--max-position-embeddings", "4096", | ||
| 10 | - "--normalization", "RMSNorm", | ||
| 11 | - "--swiglu", | ||
| 12 | - "--untie-embeddings-and-output-weights", | ||
| 13 | - "--load", "/home/dataset/baichuan-13B-tp8-pp1/" | ||
| 14 | - ], | ||
| 15 | - | ||
| 16 | - "TOKENIZER_PARAM": [ | ||
| 17 | - "--tokenizer-type", "PretrainedFromHF", | ||
| 18 | - "--tokenizer-name-or-path", "/home/dataset/baichuan-13B-hf" | ||
| 19 | - ], | ||
| 20 | - | ||
| 21 | - "PROCESS_DATA_INPUT_PATH": [ | ||
| 22 | - "--input", "/home/dataset/train-00000-of-00001-a09b74b3ef9c3b56.parquet" | ||
| 23 | - ], | ||
| 24 | - | ||
| 25 | - "PROCESS_PRETRAIN_DATA_PARAM": [ | ||
| 26 | - "--output-prefix", "/home/dataset/pretrain-dataset-baichuan-13B/alpaca", | ||
| 27 | - "--workers", "4", | ||
| 28 | - "--log-interval", "1000" | ||
| 29 | - ], | ||
| 30 | - | ||
| 31 | - "PROCESS_INSTRUCTION_DATA_PARAM": [ | ||
| 32 | - "--output-prefix", "/home/dataset/tune-dataset-baichuan-13B/alpaca", | ||
| 33 | - "--tokenizer-not-use-fast", | ||
| 34 | - "--handler-name", "GeneralInstructionHandler", | ||
| 35 | - "--append-eod" | ||
| 36 | - ], | ||
| 37 | - | ||
| 38 | - "INFERENCE_PARAM": [ | ||
| 39 | - "--max-new-tokens", "256", | ||
| 40 | - "--tokenizer-not-use-fast", | ||
| 41 | - "--exit-on-missing-checkpoint" | ||
| 42 | - ], | ||
| 43 | - | ||
| 44 | - "EVALUATION_PARAM": [ | ||
| 45 | - "--tokenizer-not-use-fast", | ||
| 46 | - "--task-data-path", "/home/dataset/eval_dataset/boolq/test", | ||
| 47 | - "--task", "boolq", | ||
| 48 | - "--max-new-tokens", "1" | ||
| 49 | - ], | ||
| 50 | - | ||
| 51 | - "LORA_PARAM": [ | ||
| 52 | - "--finetune", | ||
| 53 | - "--lora-r", "16", | ||
| 54 | - "--lora-alpha", "32", | ||
| 55 | - "--lora-target-modules", "query_key_value", "dense", "gate_proj","dense_h_to_4h", "dense_4h_to_h", | ||
| 56 | - "--is-instruction-dataset", | ||
| 57 | - "--tokenizer-not-use-fast", | ||
| 58 | - "--save", "/home/dataset/lora-save-weight-baichuan-13B", | ||
| 59 | - "--data-path", "/home/dataset/tune-dataset-baichuan-13B/alpaca", | ||
| 60 | - "--train-iters", "10" | ||
| 61 | - ], | ||
| 62 | - | ||
| 63 | - "LORA_INFERENCE_PARAM": [ | ||
| 64 | - "--lora-r", "16", | ||
| 65 | - "--lora-alpha", "32", | ||
| 66 | - "--lora-target-modules", "query_key_value", "dense", "gate_proj","dense_h_to_4h", "dense_4h_to_h", | ||
| 67 | - "--tokenizer-not-use-fast", | ||
| 68 | - "--lora-load", "/home/dataset/lora-save-weight-baichuan-13B" | ||
| 69 | - ], | ||
| 70 | - "TRAINING_PARAM": [ | ||
| 71 | - "--tokenizer-type", "Llama2Tokenizer", | ||
| 72 | - "--tokenizer-model", "/home/dataset/baichuan-13B-hf/tokenizer.model", | ||
| 73 | - "--save", "/autotest/dataset/save-weight-baichuan-13B", | ||
| 74 | - "--data-path", "/home/dataset/pretrain-dataset-baichuan-13B/alpaca_text_document", | ||
| 75 | - "--train-iters", "10" | ||
| 76 | - ], | ||
| 77 | - | ||
| 78 | - "REGULARIZATION": [ | ||
| 79 | - "--attention-dropout", "0.0", | ||
| 80 | - "--hidden-dropout", "0.0", | ||
| 81 | - "--weight-decay", "1e-1", | ||
| 82 | - "--clip-grad", "1.0", | ||
| 83 | - "--adam-beta1", "0.9", | ||
| 84 | - "--adam-beta2", "0.95", | ||
| 85 | - "--adam-eps","1.0e-5" | ||
| 86 | - ], | ||
| 87 | - | ||
| 88 | - "LEARNING_RATE": [ | ||
| 89 | - "--lr", "1e-5", | ||
| 90 | - "--lr-decay-style", "cosine", | ||
| 91 | - "--min-lr", "1e-7" | ||
| 92 | - ], | ||
| 93 | - | ||
| 94 | - "DISTRIBUTED_PARAM": [ | ||
| 95 | - "--tensor-model-parallel-size", "8", | ||
| 96 | - "--pipeline-model-parallel-size", "1" | ||
| 97 | - ], | ||
| 98 | - | ||
| 99 | - "AUXILIARY_PARAM": [ | ||
| 100 | - "--micro-batch-size", "1", | ||
| 101 | - "--global-batch-size", "8", | ||
| 102 | - "--disable-bias-linear", | ||
| 103 | - "--no-gradient-accumulation-fusion", | ||
| 104 | - "--fp16", | ||
| 105 | - "--attention-softmax-in-fp32", | ||
| 106 | - "--no-load-optim", | ||
| 107 | - "--no-load-rng", | ||
| 108 | - "--seq-length", "4096", | ||
| 109 | - "--seed", "42" | ||
| 110 | - ], | ||
| 111 | - | ||
| 112 | - "TRAINING_AUX": [ | ||
| 113 | - "--sequence-parallel", | ||
| 114 | - "--initial-loss-scale", "1024.0", | ||
| 115 | - "--use-fused-rmsnorm", | ||
| 116 | - "--init-method-std", "0.01", | ||
| 117 | - "--split", "100,0,0", | ||
| 118 | - "--log-interval", "1", | ||
| 119 | - "--save-interval", "10", | ||
| 120 | - "--eval-interval", "1000", | ||
| 121 | - "--eval-iters", "0", | ||
| 122 | - "--distributed-backend", "nccl", | ||
| 123 | - "--num-workers", "0" | ||
| 124 | - ], | ||
| 125 | - | ||
| 126 | - "CONVERT_WEIGHT_PARAM": [ | ||
| 127 | - "--model-type", "GPT", | ||
| 128 | - "--loader", "llama2_hf", | ||
| 129 | - "--saver", "megatron", | ||
| 130 | - "--target-tensor-parallel-size", "8", | ||
| 131 | - "--load-dir", "/home/dataset/baichuan-13B-hf", | ||
| 132 | - "--save-dir", "/home/dataset/baichuan-13B-tp8-pp1", | ||
| 133 | - "--tokenizer-model", "None", | ||
| 134 | - "--w-pack", "True" | ||
| 135 | - ] | ||
| 136 | -} | ||
| @@ -1,85 +0,0 @@ | |||
| 1 | -#!/bin/bash | ||
| 2 | - | ||
| 3 | -export CUDA_DEVICE_MAX_CONNECTIONS=1 | ||
| 4 | -export NPU_DETECT=0 | ||
| 5 | - | ||
| 6 | -GPUS_PER_NODE=8 | ||
| 7 | -MASTER_ADDR=localhost | ||
| 8 | -MASTER_PORT=6000 | ||
| 9 | -NNODES=1 | ||
| 10 | -NODE_RANK=0 | ||
| 11 | -WORLD_SIZE=$(($GPUS_PER_NODE*$NNODES)) | ||
| 12 | - | ||
| 13 | -DATA_PATH="/home/dataset/pretrain-dataset-baichuan-13B/alpaca_text_document" | ||
| 14 | -TOKENIZER_MODEL="/home/dataset/baichuan-13B-hf/tokenizer.model" | ||
| 15 | -CKPT_LOAD_DIR="/home/dataset/baichuan-13B-tp8-pp1" | ||
| 16 | - | ||
| 17 | -TP=8 | ||
| 18 | -PP=1 | ||
| 19 | - | ||
| 20 | -DISTRIBUTED_ARGS=" | ||
| 21 | - --nproc_per_node $GPUS_PER_NODE \ | ||
| 22 | - --nnodes $NNODES \ | ||
| 23 | - --node_rank $NODE_RANK \ | ||
| 24 | - --master_addr $MASTER_ADDR \ | ||
| 25 | - --master_port $MASTER_PORT | ||
| 26 | -" | ||
| 27 | - | ||
| 28 | -GPT_ARGS=" | ||
| 29 | - --tensor-model-parallel-size $TP \ | ||
| 30 | - --pipeline-model-parallel-size $PP \ | ||
| 31 | - --sequence-parallel \ | ||
| 32 | - --num-layers 40 \ | ||
| 33 | - --hidden-size 5120 \ | ||
| 34 | - --ffn-hidden-size 13696 \ | ||
| 35 | - --num-attention-heads 40 \ | ||
| 36 | - --tokenizer-type Llama2Tokenizer \ | ||
| 37 | - --tokenizer-model $TOKENIZER_MODEL \ | ||
| 38 | - --seq-length 4096 \ | ||
| 39 | - --disable-bias-linear \ | ||
| 40 | - --max-position-embeddings 4096 \ | ||
| 41 | - --micro-batch-size 1 \ | ||
| 42 | - --global-batch-size 32 \ | ||
| 43 | - --untie-embeddings-and-output-weights \ | ||
| 44 | - --make-vocab-size-divisible-by 64 \ | ||
| 45 | - --lr 1e-5 \ | ||
| 46 | - --no-gradient-accumulation-fusion \ | ||
| 47 | - --load ${CKPT_LOAD_DIR} \ | ||
| 48 | - --train-iters 2000 \ | ||
| 49 | - --lr-decay-style cosine \ | ||
| 50 | - --attention-dropout 0.0 \ | ||
| 51 | - --position-embedding-type alibi \ | ||
| 52 | - --hidden-dropout 0.0 \ | ||
| 53 | - --normalization RMSNorm \ | ||
| 54 | - --use-fused-rmsnorm \ | ||
| 55 | - --swiglu \ | ||
| 56 | - --attention-softmax-in-fp32 \ | ||
| 57 | - --min-lr 1e-7 \ | ||
| 58 | - --weight-decay 1e-1 \ | ||
| 59 | - --clip-grad 1.0 \ | ||
| 60 | - --adam-beta1 0.9 \ | ||
| 61 | - --initial-loss-scale 1024.0 \ | ||
| 62 | - --adam-beta2 0.95 \ | ||
| 63 | - --adam-eps 1.0e-5 \ | ||
| 64 | - --no-load-optim \ | ||
| 65 | - --no-load-rng \ | ||
| 66 | - --fp16 | ||
| 67 | -" | ||
| 68 | - | ||
| 69 | -DATA_ARGS=" | ||
| 70 | - --data-path $DATA_PATH \ | ||
| 71 | - --split 949,50,1 | ||
| 72 | -" | ||
| 73 | - | ||
| 74 | -OUTPUT_ARGS=" | ||
| 75 | - --log-interval 1 \ | ||
| 76 | - --save-interval 10000 \ | ||
| 77 | - --eval-interval 10000 \ | ||
| 78 | - --eval-iters 1 | ||
| 79 | -" | ||
| 80 | - | ||
| 81 | -torchrun $DISTRIBUTED_ARGS pretrain_gpt.py \ | ||
| 82 | - $GPT_ARGS \ | ||
| 83 | - $DATA_ARGS \ | ||
| 84 | - $OUTPUT_ARGS \ | ||
| 85 | - --distributed-backend nccl 2>&1 | tee /home/dataset/new_baichuan-13B.log | ||
| @@ -1,64 +0,0 @@ | |||
| 1 | -import unittest | ||
| 2 | -import sys | ||
| 3 | -import os | ||
| 4 | -import subprocess | ||
| 5 | -import glob | ||
| 6 | -from pathlib import Path | ||
| 7 | -import torch | ||
| 8 | -from utils import ParamConfig | ||
| 9 | - | ||
| 10 | -import modellink | ||
| 11 | - | ||
| 12 | - | ||
| 13 | -class TestConvertCkptFromHuggingface(unittest.TestCase): | ||
| 14 | - def setUp(self): | ||
| 15 | - # configure params, the index starts from 1 | ||
| 16 | - sys.argv = [sys.argv[0]] + ParamConfig.convert_weight_param | ||
| 17 | - | ||
| 18 | - def test_file_exsit(self): | ||
| 19 | - """ | ||
| 20 | - Test if the file in the `--load-dir` exsit, including `.bin`, `.json`... | ||
| 21 | - """ | ||
| 22 | - bin_file = glob.glob(os.path.join(sys.argv[10], "*.bin")) | ||
| 23 | - self.assertEqual(len(bin_file), 3) | ||
| 24 | - self.assertTrue(os.path.exists(os.path.join(sys.argv[10], "pytorch_model.bin.index.json"))) | ||
| 25 | - | ||
| 26 | - def test_convert_weights_form_huggingface(self): | ||
| 27 | - """ | ||
| 28 | - Test whether the weight to be converted as we want in `--save-dir`. We will check the model layer name, | ||
| 29 | - including embedding, final_norm, output and encoder. In the encoder, there will be some different layers | ||
| 30 | - to compose the unique transformer layer and all these layer stack to compose the entity of the model. | ||
| 31 | - """ | ||
| 32 | - base_dir = Path(__file__).absolute().parent.parent.parent.parent | ||
| 33 | - file_path = os.path.join(base_dir, "convert_ckpt.py") | ||
| 34 | - arguments = sys.argv[1:] | ||
| 35 | - subprocess.run(["python", file_path] + arguments) | ||
| 36 | - output_dir = os.path.join(sys.argv[12], "iter_0000001") | ||
| 37 | - weight_content = torch.load(os.path.join(output_dir, "mp_rank_00/model_optim_rng.pt")) | ||
| 38 | - weight_common_content = weight_content['model']['language_model'] # extract commmon content | ||
| 39 | - | ||
| 40 | - # embedding, encoder, output_layer is three out layers. | ||
| 41 | - self.assertEqual(len(os.listdir(output_dir)), int(sys.argv[8])) | ||
| 42 | - self.assertEqual(weight_common_content['embedding']['word_embeddings']['weight'].size(), | ||
| 43 | - torch.Size([8000, 5120])) | ||
| 44 | - self.assertEqual(weight_common_content['encoder']['final_norm.weight'].size(), torch.Size([5120])) | ||
| 45 | - | ||
| 46 | - # encoder has a common final_norm and each one has folliowing six layers | ||
| 47 | - weight_common_content['encoder'].pop('final_norm.weight') | ||
| 48 | - self.assertEqual(weight_common_content['encoder']['layers.0.self_attention.query_key_value.weight'].size(), | ||
| 49 | - torch.Size([1920, 5120])) | ||
| 50 | - self.assertEqual(weight_common_content['encoder']['layers.0.self_attention.dense.weight'].size(), | ||
| 51 | - torch.Size([5120, 640])) | ||
| 52 | - self.assertEqual(weight_common_content['encoder']['layers.0.mlp.dense_h_to_4h.weight'].size(), | ||
| 53 | - torch.Size([3424, 5120])) | ||
| 54 | - self.assertEqual(weight_common_content['encoder']['layers.0.mlp.dense_4h_to_h.weight'].size(), | ||
| 55 | - torch.Size([5120, 1712])) | ||
| 56 | - self.assertEqual(weight_common_content['encoder']['layers.0.input_norm.weight'].size(), torch.Size([5120])) | ||
| 57 | - self.assertEqual(weight_common_content['encoder']['layers.0.post_attention_norm.weight'].size(), | ||
| 58 | - torch.Size([5120])) | ||
| 59 | - | ||
| 60 | - self.assertEqual(weight_common_content['output_layer']['weight'].size(), torch.Size([8000, 5120])) | ||
| 61 | - | ||
| 62 | - | ||
| 63 | -if __name__ == "__main__": | ||
| 64 | - unittest.main() | ||
| @@ -1,100 +0,0 @@ | |||
| 1 | -import sys | ||
| 2 | -import os | ||
| 3 | -import json | ||
| 4 | -import torch | ||
| 5 | -import tqdm | ||
| 6 | -import torch_npu | ||
| 7 | -from utils import ParamConfig, assert_judge | ||
| 8 | -from transformers import AutoTokenizer | ||
| 9 | -import modellink | ||
| 10 | -from megatron.legacy.model import GPTModel | ||
| 11 | -from tests.common import DistributedTest | ||
| 12 | -from modellink.tasks.evaluation.utils import add_text_generate_args | ||
| 13 | - | ||
| 14 | - | ||
| 15 | -class TestEvaluation(DistributedTest): | ||
| 16 | - world_size = 8 | ||
| 17 | - | ||
| 18 | - def init(self, config=ParamConfig): | ||
| 19 | - sys.argv = [sys.argv[0]] + config.distributed_param + config.network_size + \ | ||
| 20 | - config.evaluation_param + config.auxiliary_param + config.tokenizer_param | ||
| 21 | - from megatron.training.initialize import initialize_megatron | ||
| 22 | - os.environ.update({"CUDA_DEVICE_MAX_CONNECTIONS": "1"}) | ||
| 23 | - initialize_megatron(extra_args_provider=add_text_generate_args, | ||
| 24 | - args_defaults={'no_load_rng': True, | ||
| 25 | - 'no_load_optim': True}) | ||
| 26 | - from megatron.training import get_args | ||
| 27 | - self.args = get_args() | ||
| 28 | - | ||
| 29 | - def get_result(self, tokenizer, result): | ||
| 30 | - if result: | ||
| 31 | - final_result = [result[0]] | ||
| 32 | - if result[1][0][tokenizer.encode("Yes")[-1]] >= result[1][0][tokenizer.encode("No")[-1]]: | ||
| 33 | - final_result.append("T") | ||
| 34 | - else: | ||
| 35 | - final_result.append("F") | ||
| 36 | - else: | ||
| 37 | - final_result = None | ||
| 38 | - return final_result | ||
| 39 | - | ||
| 40 | - def test_boolq_evaluation(self): | ||
| 41 | - self.init(config=ParamConfig) | ||
| 42 | - from evaluation import model_provider | ||
| 43 | - model = GPTModel.from_pretrained( | ||
| 44 | - model_provider=model_provider, | ||
| 45 | - pretrained_model_name_or_path=self.args.load | ||
| 46 | - ) | ||
| 47 | - tokenizer = AutoTokenizer.from_pretrained(self.args.tokenizer_name_or_path, trust_remote_code=True) | ||
| 48 | - max_new_tokens = self.args.max_new_tokens | ||
| 49 | - | ||
| 50 | - instruction_template = "{passage}\nQuestion: {question}?\nAnswer:" | ||
| 51 | - | ||
| 52 | - answer_result = {} | ||
| 53 | - total_acc_n = 0 | ||
| 54 | - total_n = 0 | ||
| 55 | - test_dir = None | ||
| 56 | - for path in self.args.task_data_path: | ||
| 57 | - if "boolq" in path: | ||
| 58 | - test_dir = path | ||
| 59 | - for file in tqdm.tqdm(os.listdir(test_dir)): | ||
| 60 | - file_path = os.path.join(test_dir, file) | ||
| 61 | - with open(file_path, encoding='utf-8') as f: | ||
| 62 | - boolq_question_list = [] | ||
| 63 | - for line in f.readlines(): | ||
| 64 | - boolq_question_list.append(json.loads(line)) | ||
| 65 | - boolq_question_list = boolq_question_list[:60] | ||
| 66 | - subject_result = {} | ||
| 67 | - acc_n = 0 | ||
| 68 | - for index, item in enumerate(boolq_question_list): | ||
| 69 | - instruction = instruction_template.format(passage=item['passage'], question=item['question']) | ||
| 70 | - result = model.generate( | ||
| 71 | - instruction, | ||
| 72 | - do_sample=False, | ||
| 73 | - max_new_tokens=max_new_tokens, | ||
| 74 | - tokenizer=tokenizer, | ||
| 75 | - stream=False, | ||
| 76 | - return_output_log_probs=True | ||
| 77 | - ) | ||
| 78 | - result = self.get_result(tokenizer, result) | ||
| 79 | - if result: | ||
| 80 | - answer = result[1] | ||
| 81 | - else: | ||
| 82 | - answer = None | ||
| 83 | - try: | ||
| 84 | - if torch.distributed.get_rank() == 0: | ||
| 85 | - subject_result[str(index)] = answer | ||
| 86 | - if subject_result[str(index)] == str(item['answer'])[0]: | ||
| 87 | - acc_n += 1 | ||
| 88 | - except Exception as e: | ||
| 89 | - raise e | ||
| 90 | - if torch.distributed.get_rank() == 0: | ||
| 91 | - total_n += len(boolq_question_list) | ||
| 92 | - total_acc_n += acc_n | ||
| 93 | - answer_result['Boolq_dataset'] = subject_result | ||
| 94 | - if torch.distributed.get_rank() == 0: | ||
| 95 | - try: | ||
| 96 | - final_acc = total_acc_n / total_n | ||
| 97 | - except ZeroDivisionError as e: | ||
| 98 | - raise e | ||
| 99 | - print(final_acc) | ||
| 100 | - assert_judge(abs(final_acc - 0.71) < 0.03) | ||
| @@ -1,97 +0,0 @@ | |||
| 1 | -import sys | ||
| 2 | -import os | ||
| 3 | -import torch | ||
| 4 | -import torch_npu | ||
| 5 | -from utils import ParamConfig, assert_judge | ||
| 6 | -import modellink | ||
| 7 | -from megatron.legacy.model import GPTModel | ||
| 8 | -from modellink.tasks.inference.text_generation.infer_base import add_text_generate_args | ||
| 9 | -from tests.common import DistributedTest | ||
| 10 | - | ||
| 11 | - | ||
| 12 | -class TestGeneration(DistributedTest): | ||
| 13 | - world_size = 8 | ||
| 14 | - | ||
| 15 | - def init(self, config=ParamConfig): | ||
| 16 | - """ | ||
| 17 | - initialize the environment and arguments | ||
| 18 | - """ | ||
| 19 | - sys.argv = [sys.argv[0]] + config.distributed_param + config.network_size + \ | ||
| 20 | - config.inference_param + config.auxiliary_param + config.tokenizer_param | ||
| 21 | - from megatron.training.initialize import initialize_megatron | ||
| 22 | - os.environ.update({"CUDA_DEVICE_MAX_CONNECTIONS": "1"}) | ||
| 23 | - initialize_megatron(extra_args_provider=add_text_generate_args, | ||
| 24 | - args_defaults={'no_load_rng': True, | ||
| 25 | - 'no_load_optim': True}) | ||
| 26 | - from megatron.training import get_args | ||
| 27 | - self.args = get_args() | ||
| 28 | - | ||
| 29 | - def test_greedy_search(self): | ||
| 30 | - """ | ||
| 31 | - load weight to get model and construct the prompts to generate output, | ||
| 32 | - and compare with expected for `greedy search`. | ||
| 33 | - """ | ||
| 34 | - self.init(config=ParamConfig) | ||
| 35 | - from inference import model_provider | ||
| 36 | - model = GPTModel.from_pretrained( | ||
| 37 | - model_provider=model_provider, | ||
| 38 | - pretrained_model_name_or_path=self.args.load | ||
| 39 | - ) | ||
| 40 | - instruction = ["解释一下“温故而知新”"] | ||
| 41 | - output = model.generate(instruction, detokenize=False) | ||
| 42 | - expected_output = [5, 31694, 31829, 31290, 31356, 31226, 31125, 1231, 31178, 31387, | ||
| 43 | - 34360, 73, 31106, 5, 31106, 84, 31442, 32369, 85, 31106, | ||
| 44 | - 5, 31106, 53, 31694, 31143, 31694, 31434, 73, 31106, 5, | ||
| 45 | - 31106, 54, 31829, 31143, 32363, 31135, 3317, 73, 31106, 5, | ||
| 46 | - 31106, 55, 31226, 31143, 5916, 3317, 73, 31106, 5, 31106] | ||
| 47 | - | ||
| 48 | - if torch.distributed.get_rank() == 0: | ||
| 49 | - print(output) | ||
| 50 | - similarity = torch.nn.CosineSimilarity(dim=1) | ||
| 51 | - cos_sim = similarity(torch.tensor(expected_output)[:20].unsqueeze(0).float().npu(), | ||
| 52 | - output[:20].unsqueeze(0).float()) | ||
| 53 | - print(cos_sim) | ||
| 54 | - assert_judge(cos_sim > 0.80) | ||
| 55 | - | ||
| 56 | - def test_beam_search(self): | ||
| 57 | - """ | ||
| 58 | - load weight to get model and construct the prompts to generate output, | ||
| 59 | - and compare with expected for `beam search`. | ||
| 60 | - """ | ||
| 61 | - self.init(config=ParamConfig) | ||
| 62 | - from inference import model_provider | ||
| 63 | - model = GPTModel.from_pretrained( | ||
| 64 | - model_provider=model_provider, | ||
| 65 | - pretrained_model_name_or_path=self.args.load | ||
| 66 | - ) | ||
| 67 | - | ||
| 68 | - max_new_tokens = self.args.max_new_tokens | ||
| 69 | - prompt = "解释一下“温故而知新”" | ||
| 70 | - system_template = "" | ||
| 71 | - dialog_template = "{instruction}" | ||
| 72 | - template = system_template + dialog_template | ||
| 73 | - instruction = template.format(instruction=prompt) | ||
| 74 | - | ||
| 75 | - output = model.generate( | ||
| 76 | - instruction, | ||
| 77 | - num_beams=2, | ||
| 78 | - top_k=self.args.top_k, | ||
| 79 | - top_p=self.args.top_p, | ||
| 80 | - max_new_tokens=max_new_tokens, | ||
| 81 | - tokenizer=None, | ||
| 82 | - stream=False, | ||
| 83 | - detokenize=False | ||
| 84 | - ) | ||
| 85 | - expected_output = [5, 31694, 31829, 31290, 31356, 31226, 31125, 1231, 31178, 31387, | ||
| 86 | - 34360, 73, 31106, 5, 31106, 84, 31442, 32369, 85, 31106, | ||
| 87 | - 5, 31106, 53, 31694, 31143, 31694, 31434, 73, 31106, 5, | ||
| 88 | - 31106, 54, 31829, 31143, 32363, 31135, 3317, 73, 31106, 5, | ||
| 89 | - 31106, 55, 31226, 31143, 5916, 3317, 73, 31106, 5, 31106] | ||
| 90 | - | ||
| 91 | - if torch.distributed.get_rank() == 0: | ||
| 92 | - print(output) | ||
| 93 | - similarity = torch.nn.CosineSimilarity(dim=1) | ||
| 94 | - cos_sim = similarity(torch.tensor(expected_output)[:20].unsqueeze(0).float().npu(), | ||
| 95 | - output[:20].unsqueeze(0).float()) | ||
| 96 | - print(cos_sim) | ||
| 97 | - assert_judge(cos_sim > 0.80) | ||
| @@ -1,134 +0,0 @@ | |||
| 1 | -import sys | ||
| 2 | -import os | ||
| 3 | -import torch | ||
| 4 | -import torch_npu | ||
| 5 | -from utils import ParamConfig, assert_judge | ||
| 6 | -import modellink | ||
| 7 | -from megatron.legacy.model import GPTModel | ||
| 8 | -from megatron.core.enums import ModelType | ||
| 9 | -from megatron.core.utils import get_model_config | ||
| 10 | -from megatron.training.training import setup_model_and_optimizer, build_train_valid_test_data_iterators | ||
| 11 | -from tests.common import DistributedTest | ||
| 12 | - | ||
| 13 | - | ||
| 14 | -class TestLora(DistributedTest): | ||
| 15 | - world_size = 8 | ||
| 16 | - | ||
| 17 | - def init(self, config=ParamConfig): | ||
| 18 | - sys.argv = [sys.argv[0]] + config.network_size + config.tokenizer_param \ | ||
| 19 | - + config.auxiliary_param + config.lora_param + config.regularization \ | ||
| 20 | - + config.learning_rate_param + config.training_aux + config.distributed_param | ||
| 21 | - from megatron.training.initialize import initialize_megatron | ||
| 22 | - os.environ.update({"CUDA_DEVICE_MAX_CONNECTIONS": "1"}) | ||
| 23 | - initialize_megatron(extra_args_provider=None, | ||
| 24 | - args_defaults={'no_load_rng': True, | ||
| 25 | - 'no_load_optim': True}) | ||
| 26 | - from megatron.training import get_args | ||
| 27 | - self.args = get_args() | ||
| 28 | - | ||
| 29 | - def test_megatron_lora_module(self): | ||
| 30 | - self.init(config=ParamConfig) | ||
| 31 | - from megatron.core import tensor_parallel | ||
| 32 | - from pretrain_gpt import model_provider | ||
| 33 | - model, optimizer, lr_scheduler = setup_model_and_optimizer( | ||
| 34 | - model_provider, ModelType.encoder_or_decoder | ||
| 35 | - ) | ||
| 36 | - model = model[0] | ||
| 37 | - for name, module in model.named_modules(): | ||
| 38 | - if name.endswith("query_key_value.lora_A.default"): | ||
| 39 | - assert_judge(isinstance(module, torch.nn.Linear)) | ||
| 40 | - if name.endswith("query_key_value.lora_B.default"): | ||
| 41 | - assert_judge(isinstance(module, tensor_parallel.ColumnParallelLinear)) | ||
| 42 | - | ||
| 43 | - if name.endswith("dense.lora_A.default"): | ||
| 44 | - assert_judge(isinstance(module, tensor_parallel.RowParallelLinear)) | ||
| 45 | - if name.endswith("dense.lora_B.default"): | ||
| 46 | - assert_judge(isinstance(module, torch.nn.Linear)) | ||
| 47 | - | ||
| 48 | - if name.endswith("dense_h_to_4h.lora_A.default"): | ||
| 49 | - assert_judge(isinstance(module, torch.nn.Linear)) | ||
| 50 | - if name.endswith("dense_h_to_4h.lora_B.default"): | ||
| 51 | - assert_judge(isinstance(module, tensor_parallel.ColumnParallelLinear)) | ||
| 52 | - | ||
| 53 | - if name.endswith("dense_4h_to_h.lora_A.default"): | ||
| 54 | - assert_judge(isinstance(module, tensor_parallel.RowParallelLinear)) | ||
| 55 | - if name.endswith("dense_4h_to_h.lora_B.default"): | ||
| 56 | - assert_judge(isinstance(module, torch.nn.Linear)) | ||
| 57 | - | ||
| 58 | - def test_lora(self): | ||
| 59 | - self.init(config=ParamConfig) | ||
| 60 | - torch.npu.set_compile_mode(jit_compile=True) | ||
| 61 | - from pretrain_gpt import model_provider, forward_step | ||
| 62 | - from pretrain_gpt import train_valid_test_datasets_provider | ||
| 63 | - from megatron.training.global_vars import update_num_microbatches, get_num_microbatches, get_timers | ||
| 64 | - from megatron.training.training import train_step, training_log, save_checkpoint_and_time, num_floating_point_operations | ||
| 65 | - from megatron.core import mpu | ||
| 66 | - model, optimizer, lr_scheduler = setup_model_and_optimizer( | ||
| 67 | - model_provider, ModelType.encoder_or_decoder | ||
| 68 | - ) | ||
| 69 | - assert_judge(isinstance(model, list)) | ||
| 70 | - | ||
| 71 | - config = get_model_config(model[0]) | ||
| 72 | - train_valid_test_datasets_provider.is_distributed = True | ||
| 73 | - train_data_iterator, valid_data_iterator, test_data_iterator \ | ||
| 74 | - = build_train_valid_test_data_iterators( | ||
| 75 | - train_valid_test_datasets_provider | ||
| 76 | - ) | ||
| 77 | - if self.args.eval_iters == 0: | ||
| 78 | - assert_judge(valid_data_iterator is None) | ||
| 79 | - assert_judge(test_data_iterator is None) | ||
| 80 | - | ||
| 81 | - for model_module in model: | ||
| 82 | - model_module.train() | ||
| 83 | - | ||
| 84 | - timers = get_timers() | ||
| 85 | - total_loss_dict = {} | ||
| 86 | - iteration = self.args.iteration | ||
| 87 | - config.grad_scale_func = optimizer.scale_loss | ||
| 88 | - config.timers = timers | ||
| 89 | - report_memory_flag = True | ||
| 90 | - timers('interval-time', log_level=0).start(barrier=True) | ||
| 91 | - saved_checkpoint = False | ||
| 92 | - num_floating_point_operations_so_far = 0 | ||
| 93 | - while iteration < self.args.train_iters: | ||
| 94 | - update_num_microbatches(self.args.consumed_train_samples) | ||
| 95 | - self.args.curr_iteration = iteration | ||
| 96 | - loss_dict, skipped_iter, grad_norm, num_zeros_in_grad = \ | ||
| 97 | - train_step(forward_step, | ||
| 98 | - train_data_iterator, | ||
| 99 | - model, | ||
| 100 | - optimizer, | ||
| 101 | - lr_scheduler, | ||
| 102 | - config) | ||
| 103 | - iteration += 1 | ||
| 104 | - batch_size = mpu.get_data_parallel_world_size() * \ | ||
| 105 | - self.args.micro_batch_size * \ | ||
| 106 | - get_num_microbatches() | ||
| 107 | - self.args.consumed_train_samples += batch_size | ||
| 108 | - num_floating_point_operations_so_far += num_floating_point_operations(self.args, batch_size) | ||
| 109 | - loss_scale = optimizer.get_loss_scale().item() | ||
| 110 | - params_norm = None | ||
| 111 | - learning_rate = None | ||
| 112 | - decoupled_learning_rate = None | ||
| 113 | - for param_group in optimizer.param_groups: | ||
| 114 | - if param_group['is_decoupled_lr']: | ||
| 115 | - decoupled_learning_rate = param_group['lr'] | ||
| 116 | - else: | ||
| 117 | - learning_rate = param_group['lr'] | ||
| 118 | - report_memory_flag = training_log(loss_dict, total_loss_dict, learning_rate, | ||
| 119 | - decoupled_learning_rate, | ||
| 120 | - iteration, loss_scale, | ||
| 121 | - report_memory_flag, skipped_iter, | ||
| 122 | - grad_norm, params_norm, num_zeros_in_grad) | ||
| 123 | - | ||
| 124 | - if self.args.save and self.args.save_interval and \ | ||
| 125 | - iteration % self.args.save_interval == 0: | ||
| 126 | - save_checkpoint_and_time(iteration, model, optimizer, lr_scheduler, num_floating_point_operations_so_far) | ||
| 127 | - saved_checkpoint = True | ||
| 128 | - if saved_checkpoint: | ||
| 129 | - for file_name in os.listdir(self.args.save): | ||
| 130 | - file_path = os.path.join(self.args.save, file_name) | ||
| 131 | - if os.path.isfile(file_path): | ||
| 132 | - assert_judge(file_path.endswith(".txt")) | ||
| 133 | - else: | ||
| 134 | - assert_judge(len(os.listdir(file_path)) == self.args.tensor_model_parallel_size) | ||
| @@ -1,81 +0,0 @@ | |||
| 1 | -import unittest | ||
| 2 | -import sys | ||
| 3 | -import os | ||
| 4 | -import glob | ||
| 5 | -from utils import ParamConfig | ||
| 6 | -from modellink.tokenizer import build_tokenizer | ||
| 7 | -from modellink.tokenizer.tokenizer import _AutoTokenizer | ||
| 8 | -from modellink.tasks.preprocess.data_handler import GeneralInstructionHandler | ||
| 9 | -from modellink.tasks.preprocess.data_handler import build_dataset, get_dataset_handler | ||
| 10 | -from preprocess_data import get_args, build_splitter | ||
| 11 | - | ||
| 12 | - | ||
| 13 | -class TestProcessInstructionData(unittest.TestCase): | ||
| 14 | - @classmethod | ||
| 15 | - def setUpClass(self): | ||
| 16 | - # configure params, the index starts from 1 | ||
| 17 | - sys.argv = [sys.argv[0]] + ParamConfig.tokenizer_param + ParamConfig.process_data_input_path \ | ||
| 18 | - + ParamConfig.process_instruction_data_param | ||
| 19 | - self.args = get_args() | ||
| 20 | - self.tokenizer = build_tokenizer(self.args) | ||
| 21 | - self.splitter = build_splitter(self.args) | ||
| 22 | - self.raw_dataset = build_dataset(self.args) | ||
| 23 | - self.handler = get_dataset_handler(self.args, self.raw_dataset, self.tokenizer, self.splitter) | ||
| 24 | - | ||
| 25 | - def test_build_tokenizer(self): | ||
| 26 | - """ | ||
| 27 | - Test normal function of the tokenizer: | ||
| 28 | - the instance of tokenizer | ||
| 29 | - the length of vocabulary | ||
| 30 | - the encode function | ||
| 31 | - the decode function | ||
| 32 | - the eod append | ||
| 33 | - ...(If missed something else, welcome to add) | ||
| 34 | - """ | ||
| 35 | - self.assertIsInstance(self.tokenizer, _AutoTokenizer) | ||
| 36 | - self.assertEqual(self.tokenizer.vocab_size, 64000) | ||
| 37 | - self.assertEqual(self.tokenizer.tokenize('<0xF7>'), [1557, 52, 31141, 31150, 59, 31219]) | ||
| 38 | - self.assertEqual(self.tokenizer.detokenize(31338), '度') | ||
| 39 | - self.assertEqual(self.tokenizer.detokenize(self.tokenizer.eod), '</s>') | ||
| 40 | - | ||
| 41 | - def test_build_splitter(self): | ||
| 42 | - """ | ||
| 43 | - If there's no split_sentence, default process is `IdentitySplitter()`. | ||
| 44 | - """ | ||
| 45 | - pass | ||
| 46 | - | ||
| 47 | - def test_build_dataset(self): | ||
| 48 | - """ | ||
| 49 | - Test the raw_dataset, need to test number of columns and rows | ||
| 50 | - """ | ||
| 51 | - self.assertEqual(len(self.raw_dataset.__getitem__("instruction")), 52002) | ||
| 52 | - self.assertEqual(len(self.raw_dataset.__getitem__("input")), 52002) | ||
| 53 | - self.assertEqual(len(self.raw_dataset.__getitem__("output")), 52002) | ||
| 54 | - self.assertEqual(len(self.raw_dataset.__getitem__("text")), 52002) | ||
| 55 | - | ||
| 56 | - def test_get_dataset_handler(self): | ||
| 57 | - """ | ||
| 58 | - Test if get the right data handler for pretrain | ||
| 59 | - """ | ||
| 60 | - self.assertIsInstance(self.handler, GeneralInstructionHandler) | ||
| 61 | - | ||
| 62 | - def test_serialize_to_disk(self): | ||
| 63 | - """ | ||
| 64 | - Test generate pretrain object files and files are not None(MB). | ||
| 65 | - """ | ||
| 66 | - self.handler.serialize_to_disk() | ||
| 67 | - folder_path = sys.argv[8].replace("/alpaca", "") | ||
| 68 | - bin_file = glob.glob(os.path.join(folder_path, "*.bin")) | ||
| 69 | - idx_file = glob.glob(os.path.join(folder_path, "*.idx")) | ||
| 70 | - total_size = 0 | ||
| 71 | - for file_name in os.listdir(folder_path): | ||
| 72 | - file_path = os.path.join(folder_path, file_name) | ||
| 73 | - if os.path.isfile(file_path): | ||
| 74 | - total_size += os.path.getsize(file_path) | ||
| 75 | - self.assertEqual(len(bin_file), 3) | ||
| 76 | - self.assertEqual(len(idx_file), 3) | ||
| 77 | - self.assertAlmostEqual((total_size / (1024 * 1024)), 90, delta=1) | ||
| 78 | - | ||
| 79 | - | ||
| 80 | -if __name__ == "__main__": | ||
| 81 | - unittest.main() | ||
| @@ -1,81 +0,0 @@ | |||
| 1 | -import unittest | ||
| 2 | -import sys | ||
| 3 | -import os | ||
| 4 | -import glob | ||
| 5 | -from utils import ParamConfig | ||
| 6 | -from modellink.tokenizer import build_tokenizer | ||
| 7 | -from modellink.tokenizer.tokenizer import _AutoTokenizer | ||
| 8 | -from modellink.tasks.preprocess.data_handler import GeneralPretrainHandler | ||
| 9 | -from modellink.tasks.preprocess.data_handler import build_dataset, get_dataset_handler | ||
| 10 | -from preprocess_data import get_args, build_splitter | ||
| 11 | - | ||
| 12 | - | ||
| 13 | -class TestProcessPretrainData(unittest.TestCase): | ||
| 14 | - @classmethod | ||
| 15 | - def setUpClass(self): | ||
| 16 | - # configure params, the index starts from 1 | ||
| 17 | - sys.argv = [sys.argv[0]] + ParamConfig.tokenizer_param + ParamConfig.process_data_input_path \ | ||
| 18 | - + ParamConfig.process_pretrain_data_param | ||
| 19 | - self.args = get_args() | ||
| 20 | - self.tokenizer = build_tokenizer(self.args) | ||
| 21 | - self.splitter = build_splitter(self.args) | ||
| 22 | - self.raw_dataset = build_dataset(self.args) | ||
| 23 | - self.handler = get_dataset_handler(self.args, self.raw_dataset, self.tokenizer, self.splitter) | ||
| 24 | - | ||
| 25 | - def test_build_tokenizer(self): | ||
| 26 | - """ | ||
| 27 | - Test normal function of the tokenizer: | ||
| 28 | - the instance of tokenizer | ||
| 29 | - the length of vocabulary | ||
| 30 | - the encode function | ||
| 31 | - the decode function | ||
| 32 | - the eos append | ||
| 33 | - ...(If missed something else, welcome to add) | ||
| 34 | - """ | ||
| 35 | - self.assertIsInstance(self.tokenizer, _AutoTokenizer) | ||
| 36 | - self.assertEqual(self.tokenizer.vocab_size, 64000) | ||
| 37 | - self.assertEqual(self.tokenizer.tokenize('bug'), [15498]) | ||
| 38 | - self.assertEqual(self.tokenizer.detokenize(23961), 'prolong') | ||
| 39 | - self.assertEqual(self.tokenizer.detokenize(self.tokenizer.eos), '</s>') | ||
| 40 | - | ||
| 41 | - def test_build_splitter(self): | ||
| 42 | - """ | ||
| 43 | - If there's no split_sentence, default process is `IdentitySplitter()`. | ||
| 44 | - """ | ||
| 45 | - pass | ||
| 46 | - | ||
| 47 | - def test_build_dataset(self): | ||
| 48 | - """ | ||
| 49 | - Test the raw_dataset, need to test number of columns and rows | ||
| 50 | - """ | ||
| 51 | - self.assertEqual(len(self.raw_dataset.__getitem__("instruction")), 52002) | ||
| 52 | - self.assertEqual(len(self.raw_dataset.__getitem__("input")), 52002) | ||
| 53 | - self.assertEqual(len(self.raw_dataset.__getitem__("output")), 52002) | ||
| 54 | - self.assertEqual(len(self.raw_dataset.__getitem__("text")), 52002) | ||
| 55 | - | ||
| 56 | - def test_get_dataset_handler(self): | ||
| 57 | - """ | ||
| 58 | - Test if get the right data handler for pretrain | ||
| 59 | - """ | ||
| 60 | - self.assertIsInstance(self.handler, GeneralPretrainHandler) | ||
| 61 | - | ||
| 62 | - def test_serialize_to_disk(self): | ||
| 63 | - """ | ||
| 64 | - Test generate pretrain object files and files are not None(MB). | ||
| 65 | - """ | ||
| 66 | - self.handler.serialize_to_disk() | ||
| 67 | - folder_path = sys.argv[8].replace("/alpaca", "") | ||
| 68 | - bin_file = glob.glob(os.path.join(folder_path, "*.bin")) | ||
| 69 | - idx_file = glob.glob(os.path.join(folder_path, "*.idx")) | ||
| 70 | - total_size = 0 | ||
| 71 | - for file_name in os.listdir(folder_path): | ||
| 72 | - file_path = os.path.join(folder_path, file_name) | ||
| 73 | - if os.path.isfile(file_path): | ||
| 74 | - total_size += os.path.getsize(file_path) | ||
| 75 | - self.assertEqual(len(bin_file), 1) | ||
| 76 | - self.assertEqual(len(idx_file), 1) | ||
| 77 | - self.assertAlmostEqual((total_size / (1024 * 1024)), 25, delta=1) | ||
| 78 | - | ||
| 79 | - | ||
| 80 | -if __name__ == "__main__": | ||
| 81 | - unittest.main() | ||
| @@ -1,152 +0,0 @@ | |||
| 1 | -import sys | ||
| 2 | -import os | ||
| 3 | -import torch | ||
| 4 | -import torch_npu | ||
| 5 | -from utils import ParamConfig, assert_judge | ||
| 6 | -import modellink | ||
| 7 | -from megatron.core.enums import ModelType | ||
| 8 | -from megatron.core.utils import get_model_config | ||
| 9 | -from megatron.training.training import setup_model_and_optimizer, build_train_valid_test_data_iterators, num_floating_point_operations | ||
| 10 | -from tests.common import DistributedTest | ||
| 11 | - | ||
| 12 | - | ||
| 13 | -class TestTraining(DistributedTest): | ||
| 14 | - world_size = 8 | ||
| 15 | - | ||
| 16 | - def init(self, config=ParamConfig): | ||
| 17 | - sys.argv = [sys.argv[0]] + config.network_size + config.auxiliary_param \ | ||
| 18 | - + config.regularization + config.learning_rate_param \ | ||
| 19 | - + config.training_aux + config.distributed_param + config.training_param | ||
| 20 | - from megatron.training.initialize import initialize_megatron | ||
| 21 | - os.environ.update({"CUDA_DEVICE_MAX_CONNECTIONS": "1"}) | ||
| 22 | - initialize_megatron(extra_args_provider=None, | ||
| 23 | - args_defaults={'no_load_rng': True, | ||
| 24 | - 'no_load_optim': True}) | ||
| 25 | - from megatron.training import get_args | ||
| 26 | - self.args = get_args() | ||
| 27 | - | ||
| 28 | - def test_training(self): | ||
| 29 | - self.init(config=ParamConfig) | ||
| 30 | - torch.npu.set_compile_mode(jit_compile=True) | ||
| 31 | - from pretrain_gpt import model_provider, forward_step | ||
| 32 | - from pretrain_gpt import train_valid_test_datasets_provider | ||
| 33 | - from megatron.training.global_vars import update_num_microbatches, get_num_microbatches, get_timers | ||
| 34 | - from megatron.training.training import train_step, training_log, save_checkpoint_and_time | ||
| 35 | - from megatron.core import mpu | ||
| 36 | - model, optimizer, lr_scheduler = setup_model_and_optimizer( | ||
| 37 | - model_provider, ModelType.encoder_or_decoder) | ||
| 38 | - | ||
| 39 | - config = get_model_config(model[0]) | ||
| 40 | - train_valid_test_datasets_provider.is_distributed = True | ||
| 41 | - train_data_iterator, valid_data_iterator, test_data_iterator \ | ||
| 42 | - = build_train_valid_test_data_iterators( | ||
| 43 | - train_valid_test_datasets_provider | ||
| 44 | - ) | ||
| 45 | - if self.args.eval_iters == 0: | ||
| 46 | - assert_judge(valid_data_iterator is None) | ||
| 47 | - assert_judge(test_data_iterator is None) | ||
| 48 | - | ||
| 49 | - for model_module in model: | ||
| 50 | - model_module.train() | ||
| 51 | - | ||
| 52 | - timers = get_timers() | ||
| 53 | - total_loss_dict = {} | ||
| 54 | - iteration = self.args.iteration | ||
| 55 | - config.grad_scale_func = optimizer.scale_loss | ||
| 56 | - config.timers = timers | ||
| 57 | - report_memory_flag = True | ||
| 58 | - timers('interval-time', log_level=0).start(barrier=True) | ||
| 59 | - saved_checkpoint = False | ||
| 60 | - num_floating_point_operations_so_far = 0 | ||
| 61 | - while iteration < self.args.train_iters: | ||
| 62 | - update_num_microbatches(self.args.consumed_train_samples) | ||
| 63 | - self.args.curr_iteration = iteration | ||
| 64 | - loss_dict, skipped_iter, grad_norm, num_zeros_in_grad = \ | ||
| 65 | - train_step(forward_step, | ||
| 66 | - train_data_iterator, | ||
| 67 | - model, | ||
| 68 | - optimizer, | ||
| 69 | - lr_scheduler, | ||
| 70 | - config) | ||
| 71 | - iteration += 1 | ||
| 72 | - batch_size = mpu.get_data_parallel_world_size() * \ | ||
| 73 | - self.args.micro_batch_size * \ | ||
| 74 | - get_num_microbatches() | ||
| 75 | - self.args.consumed_train_samples += batch_size | ||
| 76 | - num_floating_point_operations_so_far += num_floating_point_operations(self.args, batch_size) | ||
| 77 | - loss_scale = optimizer.get_loss_scale().item() | ||
| 78 | - params_norm = None | ||
| 79 | - learning_rate = None | ||
| 80 | - decoupled_learning_rate = None | ||
| 81 | - for param_group in optimizer.param_groups: | ||
| 82 | - if param_group['is_decoupled_lr']: | ||
| 83 | - decoupled_learning_rate = param_group['lr'] | ||
| 84 | - else: | ||
| 85 | - learning_rate = param_group['lr'] | ||
| 86 | - report_memory_flag = training_log(loss_dict, total_loss_dict, learning_rate, | ||
| 87 | - decoupled_learning_rate, | ||
| 88 | - iteration, loss_scale, | ||
| 89 | - report_memory_flag, skipped_iter, | ||
| 90 | - grad_norm, params_norm, num_zeros_in_grad) | ||
| 91 | - | ||
| 92 | - if self.args.save and self.args.save_interval and \ | ||
| 93 | - iteration % self.args.save_interval == 0: | ||
| 94 | - save_checkpoint_and_time(iteration, model, optimizer, lr_scheduler, num_floating_point_operations_so_far) | ||
| 95 | - saved_checkpoint = True | ||
| 96 | - break | ||
| 97 | - | ||
| 98 | - if saved_checkpoint: | ||
| 99 | - for file_name in os.listdir(self.args.save): | ||
| 100 | - file_path = os.path.join(self.args.save, file_name) | ||
| 101 | - if os.path.isfile(file_path): | ||
| 102 | - assert_judge(file_path.endswith(".txt")) | ||
| 103 | - else: | ||
| 104 | - assert_judge(len(os.listdir(file_path)) == self.args.tensor_model_parallel_size) | ||
| 105 | - | ||
| 106 | - def test_breakpoint_renewal_training(self): | ||
| 107 | - self.init(config=ParamConfig) | ||
| 108 | - self.args.load = self.args.save | ||
| 109 | - torch.npu.set_compile_mode(jit_compile=True) | ||
| 110 | - from pretrain_gpt import model_provider, forward_step | ||
| 111 | - from pretrain_gpt import train_valid_test_datasets_provider | ||
| 112 | - from megatron.training.global_vars import update_num_microbatches, get_timers | ||
| 113 | - from megatron.training.training import train_step | ||
| 114 | - if self.args.load == self.args.save: # We can regard it as Breakpoint Renewal Training situation | ||
| 115 | - model, optimizer, lr_scheduler = setup_model_and_optimizer( | ||
| 116 | - model_provider, ModelType.encoder_or_decoder) | ||
| 117 | - config = get_model_config(model[0]) | ||
| 118 | - train_valid_test_datasets_provider.is_distributed = True | ||
| 119 | - train_data_iterator, valid_data_iterator, test_data_iterator \ | ||
| 120 | - = build_train_valid_test_data_iterators( | ||
| 121 | - train_valid_test_datasets_provider | ||
| 122 | - ) | ||
| 123 | - if self.args.eval_iters == 0: | ||
| 124 | - assert_judge(valid_data_iterator is None) | ||
| 125 | - assert_judge(test_data_iterator is None) | ||
| 126 | - | ||
| 127 | - for model_module in model: | ||
| 128 | - model_module.train() | ||
| 129 | - | ||
| 130 | - timers = get_timers() | ||
| 131 | - iteration = self.args.iteration | ||
| 132 | - if torch.distributed.get_rank() == 0: | ||
| 133 | - print(f"iteration:{iteration}") | ||
| 134 | - assert_judge(iteration == 10) | ||
| 135 | - config.grad_scale_func = optimizer.scale_loss | ||
| 136 | - config.timers = timers | ||
| 137 | - timers('interval-time', log_level=0).start(barrier=True) | ||
| 138 | - | ||
| 139 | - if iteration < self.args.train_iters: | ||
| 140 | - update_num_microbatches(self.args.consumed_train_samples) | ||
| 141 | - self.args.curr_iteration = iteration | ||
| 142 | - loss_dict, skipped_iter, grad_norm, num_zeros_in_grad = \ | ||
| 143 | - train_step(forward_step, | ||
| 144 | - train_data_iterator, | ||
| 145 | - model, | ||
| 146 | - optimizer, | ||
| 147 | - lr_scheduler, | ||
| 148 | - config) | ||
| 149 | - if 'lm loss' in loss_dict.keys(): | ||
| 150 | - if torch.distributed.get_rank() == 0: | ||
| 151 | - print(f"loss:{loss_dict['lm loss']}") | ||
| 152 | - assert_judge(abs(loss_dict['lm loss'] - 0.97) < 0.1) | ||
| @@ -1,47 +0,0 @@ | |||
| 1 | -import json | ||
| 2 | -import os | ||
| 3 | -from pathlib import Path | ||
| 4 | -from dataclasses import dataclass | ||
| 5 | - | ||
| 6 | - | ||
| 7 | -@dataclass | ||
| 8 | -class ParamConfig: | ||
| 9 | - """ | ||
| 10 | - We can config the params in the `.json` file including: | ||
| 11 | - distributed_param, | ||
| 12 | - network_size, | ||
| 13 | - inference_param, | ||
| 14 | - evaluation_param, | ||
| 15 | - lora_param, | ||
| 16 | - training_param, | ||
| 17 | - training_auxiliary, | ||
| 18 | - learning_rate, | ||
| 19 | - regularization, | ||
| 20 | - and other auxiliary_param. | ||
| 21 | - """ | ||
| 22 | - base_dir = Path(__file__).absolute().parent | ||
| 23 | - param_config = os.path.join(base_dir, "param_config.json") | ||
| 24 | - with open(param_config) as f: | ||
| 25 | - config_file = json.load(f) | ||
| 26 | - | ||
| 27 | - distributed_param = config_file["DISTRIBUTED_PARAM"] | ||
| 28 | - network_size = config_file["NETWORK_SIZE"] | ||
| 29 | - inference_param = config_file["INFERENCE_PARAM"] | ||
| 30 | - evaluation_param = config_file["EVALUATION_PARAM"] | ||
| 31 | - lora_param = config_file["LORA_PARAM"] | ||
| 32 | - training_param = config_file["TRAINING_PARAM"] | ||
| 33 | - training_aux = config_file["TRAINING_AUX"] | ||
| 34 | - learning_rate_param = config_file["LEARNING_RATE"] | ||
| 35 | - regularization = config_file["REGULARIZATION"] | ||
| 36 | - auxiliary_param = config_file["AUXILIARY_PARAM"] | ||
| 37 | - tokenizer_param = config_file["TOKENIZER_PARAM"] | ||
| 38 | - process_pretrain_data_param = config_file["PROCESS_PRETRAIN_DATA_PARAM"] | ||
| 39 | - process_instruction_data_param = config_file["PROCESS_INSTRUCTION_DATA_PARAM"] | ||
| 40 | - process_data_input_path = config_file["PROCESS_DATA_INPUT_PATH"] | ||
| 41 | - lora_inference_param = config_file["LORA_INFERENCE_PARAM"] | ||
| 42 | - convert_weight_param = config_file["CONVERT_WEIGHT_PARAM"] | ||
| 43 | - | ||
| 44 | - | ||
| 45 | -def assert_judge(expression): | ||
| 46 | - if not expression: | ||
| 47 | - raise AssertionError | ||
| @@ -1,5 +0,0 @@ | |||
| 1 | -# Provide uniform access for piepline. | ||
| 2 | - | ||
| 3 | -python ./tests/pipeline/baichuan-7B/test_process_pretrain_data.py | ||
| 4 | -python ./tests/pipeline/baichuan-7B/test_process_instruction_data.py | ||
| 5 | - | ||
| @@ -1,21 +0,0 @@ | |||
| 1 | -{ | ||
| 2 | - "PROCESS_PRETRAIN_DATA": [ | ||
| 3 | - "--input", "/home/dataset/train-00000-of-00001-a09b74b3ef9c3b56.parquet", | ||
| 4 | - "--tokenizer-type", "PretrainedFromHF", | ||
| 5 | - "--output-prefix", "/home/dataset/pretrain-dataset-baichuan-7B/alpaca", | ||
| 6 | - "--tokenizer-name-or-path", "/home/dataset/baichuan-7B-hf", | ||
| 7 | - "--workers", "4", | ||
| 8 | - "--log-interval", "1000" | ||
| 9 | - ], | ||
| 10 | - | ||
| 11 | - "PROCESS_INSTRUCTION_DATA": [ | ||
| 12 | - "--input", "/home/dataset/train-00000-of-00001-a09b74b3ef9c3b56.parquet", | ||
| 13 | - "--tokenizer-type", "PretrainedFromHF", | ||
| 14 | - "--handler-name", "GeneralInstructionHandler", | ||
| 15 | - "--output-prefix", "/home/dataset/tune-dataset-baichuan-7B/alpaca", | ||
| 16 | - "--tokenizer-name-or-path", "/home/dataset/baichuan-7B-hf", | ||
| 17 | - "--workers", "4", | ||
| 18 | - "--log-interval", "1000", | ||
| 19 | - "--append-eod" | ||
| 20 | - ] | ||
| 21 | -} | ||
| @@ -1,84 +0,0 @@ | |||
| 1 | -import unittest | ||
| 2 | -import sys | ||
| 3 | -import os | ||
| 4 | -import glob | ||
| 5 | - | ||
| 6 | -from utils import ParamConfig | ||
| 7 | - | ||
| 8 | -from modellink.tokenizer import build_tokenizer | ||
| 9 | -from modellink.tokenizer.tokenizer import _AutoTokenizer | ||
| 10 | -from modellink.tasks.preprocess.data_handler import GeneralInstructionHandler | ||
| 11 | -from modellink.tasks.preprocess.data_handler import build_dataset, get_dataset_handler | ||
| 12 | -from preprocess_data import get_args, build_splitter | ||
| 13 | - | ||
| 14 | - | ||
| 15 | -class TestProcessInstructionData(unittest.TestCase): | ||
| 16 | - @classmethod | ||
| 17 | - def setUpClass(self): | ||
| 18 | - # configure params, the index starts from 1 | ||
| 19 | - self.config = ParamConfig | ||
| 20 | - sys.argv = [sys.argv[0]] + self.config.instruction_data_param | ||
| 21 | - self.args = get_args() | ||
| 22 | - self.tokenizer = build_tokenizer(self.args) | ||
| 23 | - self.splitter = build_splitter(self.args) | ||
| 24 | - self.raw_dataset = build_dataset(self.args) | ||
| 25 | - self.handler = get_dataset_handler(self.args, self.raw_dataset, self.tokenizer, self.splitter) | ||
| 26 | - | ||
| 27 | - | ||
| 28 | - def test_build_tokenizer(self): | ||
| 29 | - """ | ||
| 30 | - Test normal function of the tokenizer: | ||
| 31 | - the instance of tokenizer | ||
| 32 | - the length of vocabulary | ||
| 33 | - the encode function | ||
| 34 | - the decode function | ||
| 35 | - the eod append | ||
| 36 | - ...(If missed something else, welcome to add) | ||
| 37 | - """ | ||
| 38 | - self.assertIsInstance(self.tokenizer, _AutoTokenizer) | ||
| 39 | - self.assertEqual(self.tokenizer.vocab_size, 64000) | ||
| 40 | - self.assertEqual(self.tokenizer.tokenize('<0xF7>'), [1557, 52, 31141, 31150, 59, 31219]) | ||
| 41 | - self.assertEqual(self.tokenizer.detokenize(31338), '度') | ||
| 42 | - self.assertEqual(self.tokenizer.detokenize(self.tokenizer.eod), '</s>') | ||
| 43 | - | ||
| 44 | - def test_build_splitter(self): | ||
| 45 | - """ | ||
| 46 | - If there's no split_sentence, default process is `IdentitySplitter()`. | ||
| 47 | - """ | ||
| 48 | - pass | ||
| 49 | - | ||
| 50 | - def test_build_dataset(self): | ||
| 51 | - """ | ||
| 52 | - Test the raw_dataset, need to test number of columns and rows | ||
| 53 | - """ | ||
| 54 | - self.assertEqual(len(self.raw_dataset.__getitem__("instruction")), 52002) | ||
| 55 | - self.assertEqual(len(self.raw_dataset.__getitem__("input")), 52002) | ||
| 56 | - self.assertEqual(len(self.raw_dataset.__getitem__("output")), 52002) | ||
| 57 | - self.assertEqual(len(self.raw_dataset.__getitem__("text")), 52002) | ||
| 58 | - | ||
| 59 | - def test_get_dataset_handler(self): | ||
| 60 | - """ | ||
| 61 | - Test if get the right data handler for pretrain | ||
| 62 | - """ | ||
| 63 | - self.assertIsInstance(self.handler, GeneralInstructionHandler) | ||
| 64 | - | ||
| 65 | - def test_serialize_to_disk(self): | ||
| 66 | - """ | ||
| 67 | - Test generate pretrain object files and files are not None(MB). | ||
| 68 | - """ | ||
| 69 | - self.handler.serialize_to_disk() | ||
| 70 | - folder_path = self.config.instruction_data_param[7].replace("/alpaca", "") | ||
| 71 | - bin_file = glob.glob(os.path.join(folder_path, "*.bin")) | ||
| 72 | - idx_file = glob.glob(os.path.join(folder_path, "*.idx")) | ||
| 73 | - total_size = 0 | ||
| 74 | - for file_name in os.listdir(folder_path): | ||
| 75 | - file_path = os.path.join(folder_path, file_name) | ||
| 76 | - if os.path.isfile(file_path): | ||
| 77 | - total_size += os.path.getsize(file_path) | ||
| 78 | - self.assertEqual(len(bin_file), 3) | ||
| 79 | - self.assertEqual(len(idx_file), 3) | ||
| 80 | - self.assertAlmostEqual((total_size / (1024 * 1024)), 89, delta=1) | ||
| 81 | - | ||
| 82 | - | ||
| 83 | -if __name__ == "__main__": | ||
| 84 | - unittest.main() | ||
| @@ -1,88 +0,0 @@ | |||
| 1 | -import unittest | ||
| 2 | -import sys | ||
| 3 | -import os | ||
| 4 | -import glob | ||
| 5 | - | ||
| 6 | -from utils import ParamConfig | ||
| 7 | - | ||
| 8 | -from modellink.tokenizer import build_tokenizer | ||
| 9 | -from modellink.tokenizer.tokenizer import _AutoTokenizer | ||
| 10 | -from modellink.tasks.preprocess.data_handler import GeneralPretrainHandler | ||
| 11 | -from modellink.tasks.preprocess.data_handler import build_dataset, get_dataset_handler | ||
| 12 | -from preprocess_data import get_args, build_splitter | ||
| 13 | - | ||
| 14 | - | ||
| 15 | -class TestProcessPretrainData(unittest.TestCase): | ||
| 16 | - @classmethod | ||
| 17 | - def setUpClass(self): | ||
| 18 | - # configure params, the index starts from 1 | ||
| 19 | - self.config = ParamConfig | ||
| 20 | - sys.argv = [sys.argv[0]] + self.config.pretrain_data_param | ||
| 21 | - self.args = get_args() | ||
| 22 | - self.tokenizer = build_tokenizer(self.args) | ||
| 23 | - self.splitter = build_splitter(self.args) | ||
| 24 | - self.raw_dataset = build_dataset(self.args) | ||
| 25 | - self.handler = get_dataset_handler(self.args, self.raw_dataset, self.tokenizer, self.splitter) | ||
| 26 | - | ||
| 27 | - | ||
| 28 | - def test_build_tokenizer(self): | ||
| 29 | - """ | ||
| 30 | - Test normal function of the tokenizer: | ||
| 31 | - the instance of tokenizer | ||
| 32 | - the length of vocabulary | ||
| 33 | - the encode function | ||
| 34 | - the decode function | ||
| 35 | - the eos append | ||
| 36 | - ...(If missed something else, welcome to add) | ||
| 37 | - """ | ||
| 38 | - self.assertIsInstance(self.tokenizer, _AutoTokenizer) | ||
| 39 | - self.assertEqual(self.tokenizer.vocab_size, 64000) | ||
| 40 | - self.assertEqual(self.tokenizer.tokenize('bug'), [15498]) | ||
| 41 | - self.assertEqual(self.tokenizer.detokenize(23961), 'prolong') | ||
| 42 | - self.assertEqual(self.tokenizer.detokenize(self.tokenizer.eos), '</s>') | ||
| 43 | - | ||
| 44 | - | ||
| 45 | - def test_build_splitter(self): | ||
| 46 | - """ | ||
| 47 | - If there's no split_sentence, default process is `IdentitySplitter()`. | ||
| 48 | - """ | ||
| 49 | - pass | ||
| 50 | - | ||
| 51 | - | ||
| 52 | - def test_build_dataset(self): | ||
| 53 | - """ | ||
| 54 | - Test the raw_dataset, need to test number of columns and rows | ||
| 55 | - """ | ||
| 56 | - self.assertEqual(len(self.raw_dataset.__getitem__("instruction")), 52002) | ||
| 57 | - self.assertEqual(len(self.raw_dataset.__getitem__("input")), 52002) | ||
| 58 | - self.assertEqual(len(self.raw_dataset.__getitem__("output")), 52002) | ||
| 59 | - self.assertEqual(len(self.raw_dataset.__getitem__("text")), 52002) | ||
| 60 | - | ||
| 61 | - | ||
| 62 | - def test_get_dataset_handler(self): | ||
| 63 | - """ | ||
| 64 | - Test if get the right data handler for pretrain | ||
| 65 | - """ | ||
| 66 | - self.assertIsInstance(self.handler, GeneralPretrainHandler) | ||
| 67 | - | ||
| 68 | - | ||
| 69 | - def test_serialize_to_disk(self): | ||
| 70 | - """ | ||
| 71 | - Test generate pretrain object files and files are not None(MB). | ||
| 72 | - """ | ||
| 73 | - self.handler.serialize_to_disk() | ||
| 74 | - folder_path = self.config.pretrain_data_param[5].replace("/alpaca", "") | ||
| 75 | - bin_file = glob.glob(os.path.join(folder_path, "*.bin")) | ||
| 76 | - idx_file = glob.glob(os.path.join(folder_path, "*.idx")) | ||
| 77 | - total_size = 0 | ||
| 78 | - for file_name in os.listdir(folder_path): | ||
| 79 | - file_path = os.path.join(folder_path, file_name) | ||
| 80 | - if os.path.isfile(file_path): | ||
| 81 | - total_size += os.path.getsize(file_path) | ||
| 82 | - self.assertEqual(len(bin_file), 1) | ||
| 83 | - self.assertEqual(len(idx_file), 1) | ||
| 84 | - self.assertAlmostEqual((total_size / (1024 * 1024)), 25, delta=1) | ||
| 85 | - | ||
| 86 | - | ||
| 87 | -if __name__ == "__main__": | ||
| 88 | - unittest.main() | ||
| @@ -1,20 +0,0 @@ | |||
| 1 | -import json | ||
| 2 | -import os | ||
| 3 | -from pathlib import Path | ||
| 4 | -from dataclasses import dataclass | ||
| 5 | - | ||
| 6 | - | ||
| 7 | -@dataclass | ||
| 8 | -class ParamConfig: | ||
| 9 | - base_dir = Path(__file__).absolute().parent | ||
| 10 | - param_config = os.path.join(base_dir, "param_config.json") | ||
| 11 | - with open(param_config) as f: | ||
| 12 | - config_file = json.load(f) | ||
| 13 | - | ||
| 14 | - pretrain_data_param = config_file["PROCESS_PRETRAIN_DATA"] | ||
| 15 | - instruction_data_param = config_file["PROCESS_INSTRUCTION_DATA"] | ||
| 16 | - | ||
| 17 | - | ||
| 18 | -def assert_judge(expression): | ||
| 19 | - if not expression: | ||
| 20 | - raise AssertionError | ||
| @@ -8,7 +8,7 @@ from utils import ParamConfig, assert_judge | |||
| 8 | from transformers import AutoTokenizer | 8 | from transformers import AutoTokenizer |
| 9 | import modellink | 9 | import modellink |
| 10 | from megatron.legacy.model import GPTModel | 10 | from megatron.legacy.model import GPTModel |
| 11 | -from tests.common import DistributedTest | 11 | +from tests.test_tools.dist_test import DistributedTest |
| 12 | from modellink.tasks.evaluation.utils import add_text_generate_args | 12 | from modellink.tasks.evaluation.utils import add_text_generate_args |
| 13 | 13 | ||
| 14 | 14 | ||
| @@ -6,7 +6,7 @@ from utils import ParamConfig, assert_judge | |||
| 6 | import modellink | 6 | import modellink |
| 7 | from megatron.legacy.model import GPTModel | 7 | from megatron.legacy.model import GPTModel |
| 8 | from modellink.tasks.inference.text_generation.infer_base import add_text_generate_args | 8 | from modellink.tasks.inference.text_generation.infer_base import add_text_generate_args |
| 9 | -from tests.common import DistributedTest | 9 | +from tests.test_tools.dist_test import DistributedTest |
| 10 | 10 | ||
| 11 | 11 | ||
| 12 | class TestGeneration(DistributedTest): | 12 | class TestGeneration(DistributedTest): |
| @@ -1,5 +0,0 @@ | |||
| 1 | -# Provide uniform access for piepline. | ||
| 2 | - | ||
| 3 | -python ./tests/pipeline/baichuan2-7B/test_process_pretrain_data.py | ||
| 4 | -python ./tests/pipeline/baichuan2-7B/test_process_instruction_data.py | ||
| 5 | - | ||
| @@ -1,21 +0,0 @@ | |||
| 1 | -{ | ||
| 2 | - "PROCESS_PRETRAIN_DATA": [ | ||
| 3 | - "--input", "/home/dataset/train-00000-of-00001-a09b74b3ef9c3b56.parquet", | ||
| 4 | - "--tokenizer-type", "PretrainedFromHF", | ||
| 5 | - "--output-prefix", "/home/dataset/pretrain-dataset-baichuan2-7B/alpaca", | ||
| 6 | - "--tokenizer-name-or-path", "/home/dataset/baichuan2-7B-hf", | ||
| 7 | - "--workers", "4", | ||
| 8 | - "--log-interval", "1000" | ||
| 9 | - ], | ||
| 10 | - | ||
| 11 | - "PROCESS_INSTRUCTION_DATA": [ | ||
| 12 | - "--input", "/home/dataset/train-00000-of-00001-a09b74b3ef9c3b56.parquet", | ||
| 13 | - "--tokenizer-type", "PretrainedFromHF", | ||
| 14 | - "--handler-name", "GeneralInstructionHandler", | ||
| 15 | - "--output-prefix", "/home/dataset/tune-dataset-baichuan2-7B/alpaca", | ||
| 16 | - "--tokenizer-name-or-path", "/home/dataset/baichuan2-7B-hf", | ||
| 17 | - "--workers", "4", | ||
| 18 | - "--log-interval", "1000", | ||
| 19 | - "--append-eod" | ||
| 20 | - ] | ||
| 21 | -} | ||
| @@ -1,83 +0,0 @@ | |||
| 1 | -import unittest | ||
| 2 | -import sys | ||
| 3 | -import os | ||
| 4 | -import glob | ||
| 5 | - | ||
| 6 | -from utils import ParamConfig | ||
| 7 | - | ||
| 8 | -from modellink.tokenizer import build_tokenizer | ||
| 9 | -from modellink.tokenizer.tokenizer import _AutoTokenizer | ||
| 10 | -from modellink.tasks.preprocess.data_handler import GeneralInstructionHandler | ||
| 11 | -from modellink.tasks.preprocess.data_handler import build_dataset, get_dataset_handler | ||
| 12 | -from preprocess_data import get_args, build_splitter | ||
| 13 | - | ||
| 14 | - | ||
| 15 | -class TestProcessInstructionData(unittest.TestCase): | ||
| 16 | - @classmethod | ||
| 17 | - def setUpClass(self): | ||
| 18 | - # configure params, the index starts from 1 | ||
| 19 | - self.config = ParamConfig | ||
| 20 | - sys.argv = [sys.argv[0]] + self.config.instruction_data_param | ||
| 21 | - self.args = get_args() | ||
| 22 | - self.tokenizer = build_tokenizer(self.args) | ||
| 23 | - self.splitter = build_splitter(self.args) | ||
| 24 | - self.raw_dataset = build_dataset(self.args) | ||
| 25 | - self.handler = get_dataset_handler(self.args, self.raw_dataset, self.tokenizer, self.splitter) | ||
| 26 | - | ||
| 27 | - def test_build_tokenizer(self): | ||
| 28 | - """ | ||
| 29 | - Test normal function of the tokenizer: | ||
| 30 | - the instance of tokenizer | ||
| 31 | - the length of vocabulary | ||
| 32 | - the encode function | ||
| 33 | - the decode function | ||
| 34 | - the eod append | ||
| 35 | - ...(If missed something else, welcome to add) | ||
| 36 | - """ | ||
| 37 | - self.assertIsInstance(self.tokenizer, _AutoTokenizer) | ||
| 38 | - self.assertEqual(self.tokenizer.vocab_size, 125696) | ||
| 39 | - self.assertEqual(self.tokenizer.tokenize('<0xF7>'), [92655, 92335, 92365, 92379, 92383, 92574]) | ||
| 40 | - self.assertEqual(self.tokenizer.detokenize(31338), ' Norman') | ||
| 41 | - self.assertEqual(self.tokenizer.detokenize(self.tokenizer.eod), '</s>') | ||
| 42 | - | ||
| 43 | - def test_build_splitter(self): | ||
| 44 | - """ | ||
| 45 | - If there's no split_sentence, default process is `IdentitySplitter()`. | ||
| 46 | - """ | ||
| 47 | - pass | ||
| 48 | - | ||
| 49 | - def test_build_dataset(self): | ||
| 50 | - """ | ||
| 51 | - Test the raw_dataset, need to test number of columns and rows | ||
| 52 | - """ | ||
| 53 | - self.assertEqual(len(self.raw_dataset.__getitem__("instruction")), 52002) | ||
| 54 | - self.assertEqual(len(self.raw_dataset.__getitem__("input")), 52002) | ||
| 55 | - self.assertEqual(len(self.raw_dataset.__getitem__("output")), 52002) | ||
| 56 | - self.assertEqual(len(self.raw_dataset.__getitem__("text")), 52002) | ||
| 57 | - | ||
| 58 | - def test_get_dataset_handler(self): | ||
| 59 | - """ | ||
| 60 | - Test if get the right data handler for pretrain | ||
| 61 | - """ | ||
| 62 | - self.assertIsInstance(self.handler, GeneralInstructionHandler) | ||
| 63 | - | ||
| 64 | - def test_serialize_to_disk(self): | ||
| 65 | - """ | ||
| 66 | - Test generate pretrain object files and files are not None(MB). | ||
| 67 | - """ | ||
| 68 | - self.handler.serialize_to_disk() | ||
| 69 | - folder_path = self.config.instruction_data_param[7].replace("/alpaca", "") | ||
| 70 | - bin_file = glob.glob(os.path.join(folder_path, "*.bin")) | ||
| 71 | - idx_file = glob.glob(os.path.join(folder_path, "*.idx")) | ||
| 72 | - total_size = 0 | ||
| 73 | - for file_name in os.listdir(folder_path): | ||
| 74 | - file_path = os.path.join(folder_path, file_name) | ||
| 75 | - if os.path.isfile(file_path): | ||
| 76 | - total_size += os.path.getsize(file_path) | ||
| 77 | - self.assertEqual(len(bin_file), 3) | ||
| 78 | - self.assertEqual(len(idx_file), 3) | ||
| 79 | - self.assertAlmostEqual((total_size / (1024 * 1024)), 83, delta=1) | ||
| 80 | - | ||
| 81 | - | ||
| 82 | -if __name__ == "__main__": | ||
| 83 | - unittest.main() | ||
| @@ -1,83 +0,0 @@ | |||
| 1 | -import unittest | ||
| 2 | -import sys | ||
| 3 | -import os | ||
| 4 | -import glob | ||
| 5 | - | ||
| 6 | -from utils import ParamConfig | ||
| 7 | - | ||
| 8 | -from modellink.tokenizer import build_tokenizer | ||
| 9 | -from modellink.tokenizer.tokenizer import _AutoTokenizer | ||
| 10 | -from modellink.tasks.preprocess.data_handler import GeneralPretrainHandler | ||
| 11 | -from modellink.tasks.preprocess.data_handler import build_dataset, get_dataset_handler | ||
| 12 | -from preprocess_data import get_args, build_splitter | ||
| 13 | - | ||
| 14 | - | ||
| 15 | -class TestProcessPretrainData(unittest.TestCase): | ||
| 16 | - @classmethod | ||
| 17 | - def setUpClass(self): | ||
| 18 | - # configure params, the index starts from 1 | ||
| 19 | - self.config = ParamConfig | ||
| 20 | - sys.argv = [sys.argv[0]] + self.config.pretrain_data_param | ||
| 21 | - self.args = get_args() | ||
| 22 | - self.tokenizer = build_tokenizer(self.args) | ||
| 23 | - self.splitter = build_splitter(self.args) | ||
| 24 | - self.raw_dataset = build_dataset(self.args) | ||
| 25 | - self.handler = get_dataset_handler(self.args, self.raw_dataset, self.tokenizer, self.splitter) | ||
| 26 | - | ||
| 27 | - def test_build_tokenizer(self): | ||
| 28 | - """ | ||
| 29 | - Test normal function of the tokenizer: | ||
| 30 | - the instance of tokenizer | ||
| 31 | - the length of vocabulary | ||
| 32 | - the encode function | ||
| 33 | - the decode function | ||
| 34 | - the eos append | ||
| 35 | - ...(If missed something else, welcome to add) | ||
| 36 | - """ | ||
| 37 | - self.assertIsInstance(self.tokenizer, _AutoTokenizer) | ||
| 38 | - self.assertEqual(self.tokenizer.vocab_size, 125696) | ||
| 39 | - self.assertEqual(self.tokenizer.tokenize('bug'), [44985]) | ||
| 40 | - self.assertEqual(self.tokenizer.detokenize(23961), '为孩子') | ||
| 41 | - self.assertEqual(self.tokenizer.detokenize(self.tokenizer.eos), '</s>') | ||
| 42 | - | ||
| 43 | - def test_build_splitter(self): | ||
| 44 | - """ | ||
| 45 | - If there's no split_sentence, default process is `IdentitySplitter()`. | ||
| 46 | - """ | ||
| 47 | - pass | ||
| 48 | - | ||
| 49 | - def test_build_dataset(self): | ||
| 50 | - """ | ||
| 51 | - Test the raw_dataset, need to test number of columns and rows | ||
| 52 | - """ | ||
| 53 | - self.assertEqual(len(self.raw_dataset.__getitem__("instruction")), 52002) | ||
| 54 | - self.assertEqual(len(self.raw_dataset.__getitem__("input")), 52002) | ||
| 55 | - self.assertEqual(len(self.raw_dataset.__getitem__("output")), 52002) | ||
| 56 | - self.assertEqual(len(self.raw_dataset.__getitem__("text")), 52002) | ||
| 57 | - | ||
| 58 | - def test_get_dataset_handler(self): | ||
| 59 | - """ | ||
| 60 | - Test if get the right data handler for pretrain | ||
| 61 | - """ | ||
| 62 | - self.assertIsInstance(self.handler, GeneralPretrainHandler) | ||
| 63 | - | ||
| 64 | - def test_serialize_to_disk(self): | ||
| 65 | - """ | ||
| 66 | - Test generate pretrain object files and files are not None(MB). | ||
| 67 | - """ | ||
| 68 | - self.handler.serialize_to_disk() | ||
| 69 | - folder_path = self.config.pretrain_data_param[5].replace("/alpaca", "") | ||
| 70 | - bin_file = glob.glob(os.path.join(folder_path, "*.bin")) | ||
| 71 | - idx_file = glob.glob(os.path.join(folder_path, "*.idx")) | ||
| 72 | - total_size = 0 | ||
| 73 | - for file_name in os.listdir(folder_path): | ||
| 74 | - file_path = os.path.join(folder_path, file_name) | ||
| 75 | - if os.path.isfile(file_path): | ||
| 76 | - total_size += os.path.getsize(file_path) | ||
| 77 | - self.assertEqual(len(bin_file), 1) | ||
| 78 | - self.assertEqual(len(idx_file), 1) | ||
| 79 | - self.assertAlmostEqual((total_size / (1024 * 1024)), 23, delta=1) | ||
| 80 | - | ||
| 81 | - | ||
| 82 | -if __name__ == "__main__": | ||
| 83 | - unittest.main() | ||
| @@ -1,20 +0,0 @@ | |||
| 1 | -import json | ||
| 2 | -import os | ||
| 3 | -from pathlib import Path | ||
| 4 | -from dataclasses import dataclass | ||
| 5 | - | ||
| 6 | - | ||
| 7 | -@dataclass | ||
| 8 | -class ParamConfig: | ||
| 9 | - base_dir = Path(__file__).absolute().parent | ||
| 10 | - param_config = os.path.join(base_dir, "param_config.json") | ||
| 11 | - with open(param_config) as f: | ||
| 12 | - config_file = json.load(f) | ||
| 13 | - | ||
| 14 | - pretrain_data_param = config_file["PROCESS_PRETRAIN_DATA"] | ||
| 15 | - instruction_data_param = config_file["PROCESS_INSTRUCTION_DATA"] | ||
| 16 | - | ||
| 17 | - | ||
| 18 | -def assert_judge(expression): | ||
| 19 | - if not expression: | ||
| 20 | - raise AssertionError | ||
| @@ -7,7 +7,7 @@ import pandas as pd | |||
| 7 | import torch | 7 | import torch |
| 8 | import torch_npu | 8 | import torch_npu |
| 9 | from transformers import AutoTokenizer | 9 | from transformers import AutoTokenizer |
| 10 | -from tests.common import DistributedTest | 10 | +from tests.test_tools.dist_test import DistributedTest |
| 11 | from utils import ParamConfig, assert_judge | 11 | from utils import ParamConfig, assert_judge |
| 12 | import modellink | 12 | import modellink |
| 13 | from megatron.legacy.model import GPTModel | 13 | from megatron.legacy.model import GPTModel |
| @@ -3,7 +3,7 @@ import os | |||
| 3 | import nltk | 3 | import nltk |
| 4 | import torch | 4 | import torch |
| 5 | import torch_npu | 5 | import torch_npu |
| 6 | -from tests.common import DistributedTest | 6 | +from tests.test_tools.dist_test import DistributedTest |
| 7 | from utils import ParamConfig, assert_judge | 7 | from utils import ParamConfig, assert_judge |
| 8 | import modellink | 8 | import modellink |
| 9 | from megatron.legacy.model import GPTModel | 9 | from megatron.legacy.model import GPTModel |
| @@ -3,7 +3,7 @@ import os | |||
| 3 | import nltk | 3 | import nltk |
| 4 | import torch | 4 | import torch |
| 5 | import torch_npu | 5 | import torch_npu |
| 6 | -from tests.common import DistributedTest | 6 | +from tests.test_tools.dist_test import DistributedTest |
| 7 | from utils import ParamConfig, assert_judge | 7 | from utils import ParamConfig, assert_judge |
| 8 | import modellink | 8 | import modellink |
| 9 | from megatron.legacy.model import GPTModel | 9 | from megatron.legacy.model import GPTModel |
| @@ -1,5 +0,0 @@ | |||
| 1 | -# Provide uniform access for piepline. | ||
| 2 | - | ||
| 3 | -python tests/pipeline/codellama-34B/test_convert_ckpt_from_huggingface.py | ||
| 4 | - | ||
| 5 | -pytest -s tests/pipeline/codellama-34B/test_generation.py | ||
| @@ -1,62 +0,0 @@ | |||
| 1 | -{ | ||
| 2 | - "CONVERT_CKPT_PARAM": [ | ||
| 3 | - "--model-type", "GPT", | ||
| 4 | - "--loader", "llama2_hf", | ||
| 5 | - "--saver", "megatron", | ||
| 6 | - "--load-dir", "/home/dataset/codellama-34B-hf", | ||
| 7 | - "--save-dir", "/home/dataset/codellama-34B-mt-t8p1", | ||
| 8 | - "--target-tensor-parallel-size", "8", | ||
| 9 | - "--target-pipeline-parallel-size", "1", | ||
| 10 | - "--tokenizer-model", "None" | ||
| 11 | - ], | ||
| 12 | - | ||
| 13 | - "NETWORK_SIZE": [ | ||
| 14 | - "--num-layers", "48", | ||
| 15 | - "--hidden-size", "8192", | ||
| 16 | - "--ffn-hidden-size", "22016", | ||
| 17 | - "--num-attention-heads", "64", | ||
| 18 | - "--max-position-embeddings", "16384", | ||
| 19 | - "--position-embedding-type", "rope", | ||
| 20 | - "--make-vocab-size-divisible-by", "1", | ||
| 21 | - "--normalization", "RMSNorm", | ||
| 22 | - "--swiglu", | ||
| 23 | - "--untie-embeddings-and-output-weights", | ||
| 24 | - "--load", "/home/dataset/codellama-34B-mt-t8p1" | ||
| 25 | - ], | ||
| 26 | - | ||
| 27 | - "TOKENIZER_PARAM": [ | ||
| 28 | - "--tokenizer-type", "PretrainedFromHF", | ||
| 29 | - "--tokenizer-name-or-path", "/home/dataset/codellama-34B-hf" | ||
| 30 | - ], | ||
| 31 | - | ||
| 32 | - "DISTRIBUTED_PARAM": [ | ||
| 33 | - "--tensor-model-parallel-size", "8", | ||
| 34 | - "--pipeline-model-parallel-size", "1" | ||
| 35 | - ], | ||
| 36 | - | ||
| 37 | - "INFERENCE_PARAM": [ | ||
| 38 | - "--max-new-tokens", "256", | ||
| 39 | - "--tokenizer-not-use-fast", | ||
| 40 | - "--exit-on-missing-checkpoint", | ||
| 41 | - "--attention-softmax-in-fp32" | ||
| 42 | - ], | ||
| 43 | - | ||
| 44 | - "AUXILIARY_PARAM": [ | ||
| 45 | - "--micro-batch-size", "1", | ||
| 46 | - "--global-batch-size", "16", | ||
| 47 | - "--no-masked-softmax-fusion", | ||
| 48 | - "--disable-bias-linear", | ||
| 49 | - "--no-gradient-accumulation-fusion", | ||
| 50 | - "--bf16", | ||
| 51 | - "--seed", "42", | ||
| 52 | - "--use-fused-rmsnorm", | ||
| 53 | - "--group-query-attention", | ||
| 54 | - "--no-load-optim", | ||
| 55 | - "--no-load-rng", | ||
| 56 | - "--seq-length", "4096", | ||
| 57 | - "--num-query-groups", "8", | ||
| 58 | - "--vocab-size", "32000", | ||
| 59 | - "--padded-vocab-size", "32000", | ||
| 60 | - "--rotary-base", "1000000" | ||
| 61 | - ] | ||
| 62 | -} | ||
| @@ -1,59 +0,0 @@ | |||
| 1 | -import unittest | ||
| 2 | -import sys | ||
| 3 | -import os | ||
| 4 | -import subprocess | ||
| 5 | -import glob | ||
| 6 | -from pathlib import Path | ||
| 7 | -from utils import ParamConfig | ||
| 8 | -import torch | ||
| 9 | - | ||
| 10 | -import modellink | ||
| 11 | - | ||
| 12 | - | ||
| 13 | -class TestConvertCkptFromHuggingface(unittest.TestCase): | ||
| 14 | - def setUp(self, config=ParamConfig): | ||
| 15 | - # configure params, the index starts from 1 | ||
| 16 | - self.config = config | ||
| 17 | - sys.argv = [sys.argv[0]] + self.config.convert_ckpt_param | ||
| 18 | - | ||
| 19 | - def test_file_exsit(self): | ||
| 20 | - """ | ||
| 21 | - Test if the file in the `--load-dir` exsit, including `.bin`, `.json`... | ||
| 22 | - """ | ||
| 23 | - bin_file = glob.glob(os.path.join(self.config.convert_ckpt_param[7], "*.bin")) | ||
| 24 | - self.assertEqual(len(bin_file), 7) | ||
| 25 | - self.assertTrue(os.path.exists(os.path.join(self.config.convert_ckpt_param[7], "pytorch_model.bin.index.json"))) | ||
| 26 | - | ||
| 27 | - def test_convert_weights_form_huggingface(self): | ||
| 28 | - """ | ||
| 29 | - Test whether the weight to be converted as we want in `--save-dir`. We will check the model layer name, | ||
| 30 | - including embedding, final_norm, output and encoder. In the encoder, there will be some different layers | ||
| 31 | - to compose the unique transformer layer and all these layer stack to compose the entity of the model. | ||
| 32 | - """ | ||
| 33 | - base_dir = Path(__file__).absolute().parent.parent.parent.parent | ||
| 34 | - file_path = os.path.join(base_dir, "convert_ckpt.py") | ||
| 35 | - arguments = sys.argv[1:] | ||
| 36 | - subprocess.run(["python", file_path] + arguments) | ||
| 37 | - output_dir = os.path.join(self.config.convert_ckpt_param[9], "iter_0000001") | ||
| 38 | - weight_content = torch.load(os.path.join(output_dir, "mp_rank_00/model_optim_rng.pt")) | ||
| 39 | - weight_common_content = weight_content['model']['language_model'] # extract commmon content | ||
| 40 | - | ||
| 41 | - # embedding, encoder, output_layer is three out layers. | ||
| 42 | - self.assertEqual(len(os.listdir(output_dir)), int(self.config.convert_ckpt_param[11])) | ||
| 43 | - self.assertEqual(weight_common_content['embedding']['word_embeddings']['weight'].size(), torch.Size([4000, 8192])) | ||
| 44 | - self.assertEqual(weight_common_content['encoder']['final_norm.weight'].size(), torch.Size([8192])) | ||
| 45 | - | ||
| 46 | - # encoder has a common final_norm and each one has folliowing six layers | ||
| 47 | - weight_common_content['encoder'].pop('final_norm.weight') | ||
| 48 | - self.assertEqual(weight_common_content['encoder']['layers.0.self_attention.query_key_value.weight'].size(), torch.Size([1280, 8192])) | ||
| 49 | - self.assertEqual(weight_common_content['encoder']['layers.0.self_attention.dense.weight'].size(), torch.Size([8192, 1024])) | ||
| 50 | - self.assertEqual(weight_common_content['encoder']['layers.0.mlp.dense_h_to_4h.weight'].size(), torch.Size([5504, 8192])) | ||
| 51 | - self.assertEqual(weight_common_content['encoder']['layers.0.mlp.dense_4h_to_h.weight'].size(), torch.Size([8192, 2752])) | ||
| 52 | - self.assertEqual(weight_common_content['encoder']['layers.0.input_norm.weight'].size(), torch.Size([8192])) | ||
| 53 | - self.assertEqual(weight_common_content['encoder']['layers.0.post_attention_norm.weight'].size(), torch.Size([8192])) | ||
| 54 | - | ||
| 55 | - self.assertEqual(weight_common_content['output_layer']['weight'].size(), torch.Size([4000, 8192])) | ||
| 56 | - | ||
| 57 | - | ||
| 58 | -if __name__ == "__main__": | ||
| 59 | - unittest.main() | ||
| @@ -1,100 +0,0 @@ | |||
| 1 | -import sys | ||
| 2 | -import os | ||
| 3 | -import torch | ||
| 4 | -import torch_npu | ||
| 5 | -from tests.common import DistributedTest | ||
| 6 | -from utils import ParamConfig, assert_judge | ||
| 7 | -import modellink | ||
| 8 | -from megatron.legacy.model import GPTModel | ||
| 9 | -from modellink.tasks.inference.text_generation.infer_base import add_text_generate_args | ||
| 10 | - | ||
| 11 | - | ||
| 12 | -class TestGeneration(DistributedTest): | ||
| 13 | - world_size = 8 | ||
| 14 | - | ||
| 15 | - def init(self, config=ParamConfig): | ||
| 16 | - """ | ||
| 17 | - initialize the environment and arguments | ||
| 18 | - """ | ||
| 19 | - sys.argv = [sys.argv[0]] + config.distributed_param + config.network_size + \ | ||
| 20 | - config.inference_param + config.auxiliary_param + config.tokenizer_param | ||
| 21 | - from megatron.training.initialize import initialize_megatron | ||
| 22 | - os.environ.update({"CUDA_DEVICE_MAX_CONNECTIONS": "1"}) | ||
| 23 | - initialize_megatron(extra_args_provider=add_text_generate_args, | ||
| 24 | - args_defaults={'no_load_rng': True, | ||
| 25 | - 'no_load_optim': True}) | ||
| 26 | - from megatron.training import get_args | ||
| 27 | - self.args = get_args() | ||
| 28 | - | ||
| 29 | - def test_greedy_search(self): | ||
| 30 | - """ | ||
| 31 | - load weight to get model and construct the prompts to generate output, | ||
| 32 | - and compare with expected for `greedy search`. | ||
| 33 | - """ | ||
| 34 | - self.init(config=ParamConfig) | ||
| 35 | - from inference import model_provider | ||
| 36 | - model = GPTModel.from_pretrained( | ||
| 37 | - model_provider=model_provider, | ||
| 38 | - pretrained_model_name_or_path=self.args.load | ||
| 39 | - ) | ||
| 40 | - instruction = ["import socket\n\ndef ping_exponential_backoff(host: str):"] | ||
| 41 | - output = model.generate(instruction, detokenize=False) | ||
| 42 | - expected_output1 = [13, 1678, 9995, 13, 1678, 349, 292, 263, 3495, 773, | ||
| 43 | - 25658, 1250, 2696, 29889, 13, 1678, 9995, 13, 1678, 363, | ||
| 44 | - 474, 297, 3464, 29898, 29896, 29892, 29871, 29896, 29900, 1125, | ||
| 45 | - 13, 4706, 1018, 29901, 13, 9651, 9909, 29889, 29887, 621, | ||
| 46 | - 520, 29890, 948, 420, 29898, 3069, 29897, 13, 9651, 736] | ||
| 47 | - expected_output2 = [13, 1678, 9995, 13, 1678, 349, 292, 263, 3495, 773, | ||
| 48 | - 25658, 1250, 2696, 29889, 13, 1678, 9995, 13, 1678, 9055, | ||
| 49 | - 353, 29871, 29896, 13, 1678, 1550, 5852, 29901, 13, 4706, | ||
| 50 | - 1018, 29901, 13, 9651, 9909, 29889, 29887, 621, 520, 29890, | ||
| 51 | - 948, 420, 29898, 3069, 29897, 13, 9651, 736, 13, 4706] | ||
| 52 | - | ||
| 53 | - if torch.distributed.get_rank() == 0: | ||
| 54 | - print(output) | ||
| 55 | - similarity = torch.nn.CosineSimilarity(dim=1) | ||
| 56 | - cos_sim1 = similarity(torch.tensor(expected_output1).unsqueeze(0).float().npu(), | ||
| 57 | - output[:50].unsqueeze(0).float()) | ||
| 58 | - cos_sim2 = similarity(torch.tensor(expected_output2).unsqueeze(0).float().npu(), | ||
| 59 | - output[:50].unsqueeze(0).float()) | ||
| 60 | - cos_sim = torch.max(cos_sim1, cos_sim2) | ||
| 61 | - print("similarity: ", cos_sim) | ||
| 62 | - assert_judge(cos_sim > 0.95) | ||
| 63 | - | ||
| 64 | - def test_beam_search(self): | ||
| 65 | - """ | ||
| 66 | - load weight to get model and construct the prompts to generate output, | ||
| 67 | - and compare with expected for `beam search`. | ||
| 68 | - """ | ||
| 69 | - self.init(config=ParamConfig) | ||
| 70 | - from inference import model_provider | ||
| 71 | - model = GPTModel.from_pretrained( | ||
| 72 | - model_provider=model_provider, | ||
| 73 | - pretrained_model_name_or_path=self.args.load | ||
| 74 | - ) | ||
| 75 | - | ||
| 76 | - max_new_tokens = self.args.max_new_tokens | ||
| 77 | - instruction = "def fibonacci(" | ||
| 78 | - output = model.generate( | ||
| 79 | - instruction, | ||
| 80 | - num_beams=2, | ||
| 81 | - top_k=self.args.top_k, | ||
| 82 | - top_p=self.args.top_p, | ||
| 83 | - max_new_tokens=max_new_tokens, | ||
| 84 | - tokenizer=None, | ||
| 85 | - stream=False, | ||
| 86 | - detokenize=False | ||
| 87 | - ) | ||
| 88 | - expected_output = [29876, 1125, 13, 1678, 565, 302, 1275, 29871, 29900, 29901, | ||
| 89 | - 13, 4706, 736, 29871, 29900, 13, 1678, 25342, 302, 1275, | ||
| 90 | - 29871, 29896, 29901, 13, 4706, 736, 29871, 29896, 13, 1678, | ||
| 91 | - 1683, 29901, 13, 4706, 736, 18755, 265, 21566, 29898, 29876, | ||
| 92 | - 448, 29871, 29896, 29897, 718, 18755, 265, 21566, 29898, 29876] | ||
| 93 | - | ||
| 94 | - if torch.distributed.get_rank() == 0: | ||
| 95 | - print(output) | ||
| 96 | - similarity = torch.nn.CosineSimilarity(dim=1) | ||
| 97 | - cos_sim = similarity(torch.tensor(expected_output).unsqueeze(0).float().npu(), | ||
| 98 | - output[:50].unsqueeze(0).float()) | ||
| 99 | - print("similarity: ", cos_sim) | ||
| 100 | - assert_judge(cos_sim > 0.95) | ||
| @@ -1,33 +0,0 @@ | |||
| 1 | -import json | ||
| 2 | -import os | ||
| 3 | -from pathlib import Path | ||
| 4 | -from dataclasses import dataclass | ||
| 5 | - | ||
| 6 | - | ||
| 7 | -@dataclass | ||
| 8 | -class ParamConfig: | ||
| 9 | - """ | ||
| 10 | - We can config the params in the `.json` file including: | ||
| 11 | - convert_ckpt_param, | ||
| 12 | - network_size, | ||
| 13 | - tokenizer_param, | ||
| 14 | - distributed_param, | ||
| 15 | - inference_param, | ||
| 16 | - and other auxiliary_param. | ||
| 17 | - """ | ||
| 18 | - base_dir = Path(__file__).absolute().parent | ||
| 19 | - param_config = os.path.join(base_dir, "param_config.json") | ||
| 20 | - with open(param_config) as f: | ||
| 21 | - config_file = json.load(f) | ||
| 22 | - | ||
| 23 | - convert_ckpt_param = config_file["CONVERT_CKPT_PARAM"] | ||
| 24 | - network_size = config_file["NETWORK_SIZE"] | ||
| 25 | - tokenizer_param = config_file["TOKENIZER_PARAM"] | ||
| 26 | - distributed_param = config_file["DISTRIBUTED_PARAM"] | ||
| 27 | - inference_param = config_file["INFERENCE_PARAM"] | ||
| 28 | - auxiliary_param = config_file["AUXILIARY_PARAM"] | ||
| 29 | - | ||
| 30 | - | ||
| 31 | -def assert_judge(expression): | ||
| 32 | - if not expression: | ||
| 33 | - raise AssertionError | ||
| @@ -1,130 +0,0 @@ | |||
| 1 | -import json | ||
| 2 | -import argparse | ||
| 3 | -from typing import Tuple, Optional | ||
| 4 | -import pandas as pd | ||
| 5 | - | ||
| 6 | -class Comparator: | ||
| 7 | - def __init__(self, | ||
| 8 | - base_path_prefix: str, | ||
| 9 | - test_path_prefix: str, | ||
| 10 | - loss_error_rate: float = 0.02, | ||
| 11 | - perf_error_rate: float = 0.03, | ||
| 12 | - mem_error_rate: float = 0.003, | ||
| 13 | - warm_up: int = 1, | ||
| 14 | - compute_steps: int = 2000): | ||
| 15 | - self.base_path_prefix = base_path_prefix | ||
| 16 | - self.test_path_prefix = test_path_prefix | ||
| 17 | - self.loss_error_rate = loss_error_rate | ||
| 18 | - self.perf_error_rate = perf_error_rate | ||
| 19 | - self.mem_error_rate = mem_error_rate | ||
| 20 | - self.compute_steps = compute_steps | ||
| 21 | - self.warm_up = warm_up | ||
| 22 | - | ||
| 23 | - def _read_check_loss_file(self) -> Optional[Tuple[pd.DataFrame, pd.DataFrame, int]]: | ||
| 24 | - base_loss_pd = pd.read_csv(f"{self.base_path_prefix}_loss.tsv", sep='\t') | ||
| 25 | - test_loss_pd = pd.read_csv(f"{self.test_path_prefix}_loss.tsv", sep='\t') | ||
| 26 | - if len(base_loss_pd) < self.compute_steps or len(test_loss_pd) < self.compute_steps: | ||
| 27 | - print("The log doesn't have enough steps to compute!") | ||
| 28 | - return None | ||
| 29 | - | ||
| 30 | - base_loss_start = base_loss_pd.loss.ne(float('inf')).argmax() | ||
| 31 | - test_loss_start = test_loss_pd.loss.ne(float('inf')).argmax() | ||
| 32 | - if base_loss_start != test_loss_start: | ||
| 33 | - print("The validate loss step is not equal!") | ||
| 34 | - return None | ||
| 35 | - | ||
| 36 | - return base_loss_pd, test_loss_pd, base_loss_start | ||
| 37 | - | ||
| 38 | - | ||
| 39 | - | ||
| 40 | - def compare_memory(self) -> bool: | ||
| 41 | - base_mem_pd = pd.read_csv(f"{self.base_path_prefix}_memory.tsv", sep='\t') | ||
| 42 | - test_mem_pd = pd.read_csv(f"{self.test_path_prefix}_memory.tsv", sep='\t') | ||
| 43 | - base_mem_mean = base_mem_pd.memory.mean() | ||
| 44 | - test_mem_mean = test_mem_pd.memory.mean() | ||
| 45 | - if base_mem_mean * (1 + self.mem_error_rate) < test_mem_mean: | ||
| 46 | - print("Memory test failed!") | ||
| 47 | - return False | ||
| 48 | - | ||
| 49 | - print("Memory test pass!") | ||
| 50 | - return True | ||
| 51 | - | ||
| 52 | - def compare_perf(self) -> bool: | ||
| 53 | - result = self._read_check_loss_file() | ||
| 54 | - if not result: | ||
| 55 | - return False | ||
| 56 | - | ||
| 57 | - base_loss_pd, test_loss_pd, loss_start = result | ||
| 58 | - | ||
| 59 | - with open(f"{self.base_path_prefix}_parameters.json") as f: | ||
| 60 | - base_params = json.load(f) | ||
| 61 | - | ||
| 62 | - with open(f"{self.test_path_prefix}_parameters.json") as f: | ||
| 63 | - test_params = json.load(f) | ||
| 64 | - | ||
| 65 | - if base_params != test_params: | ||
| 66 | - print("The parameters are not equal") | ||
| 67 | - return False | ||
| 68 | - | ||
| 69 | - global_batch_size = base_params.get("global_batch_size") or base_params.get("train_batch_size") | ||
| 70 | - seq_length = base_params.get("seq_length") or base_params.get("seq-length") | ||
| 71 | - world_size = base_params.get("world_size", 8) | ||
| 72 | - | ||
| 73 | - | ||
| 74 | - # Here we need to skip the first steps until the training is stable | ||
| 75 | - base_itertime_mean = base_loss_pd[self.warm_up:self.compute_steps].iter_time.mean() | ||
| 76 | - test_itertime_mean = test_loss_pd[self.warm_up:self.compute_steps].iter_time.mean() | ||
| 77 | - | ||
| 78 | - base_perf = global_batch_size * seq_length / world_size / base_itertime_mean | ||
| 79 | - test_perf = global_batch_size * seq_length / world_size / test_itertime_mean | ||
| 80 | - | ||
| 81 | - if (1 - self.perf_error_rate) * base_perf > test_perf: | ||
| 82 | - print("Perf test failed!") | ||
| 83 | - return False | ||
| 84 | - | ||
| 85 | - print("Perf test pass!") | ||
| 86 | - return True | ||
| 87 | - | ||
| 88 | - | ||
| 89 | - def compare_loss(self) -> bool: | ||
| 90 | - result = self._read_check_loss_file() | ||
| 91 | - if not result: | ||
| 92 | - return False | ||
| 93 | - | ||
| 94 | - base_loss_pd, test_loss_pd, loss_start = result | ||
| 95 | - | ||
| 96 | - loss_error_rates = (test_loss_pd[loss_start:self.compute_steps].loss - base_loss_pd[loss_start:self.compute_steps].loss) / base_loss_pd[loss_start:self.compute_steps].loss | ||
| 97 | - if abs(loss_error_rates.mean()) > self.loss_error_rate: | ||
| 98 | - print("Loss test failed!") | ||
| 99 | - return False | ||
| 100 | - | ||
| 101 | - print("Loss test pass!") | ||
| 102 | - return True | ||
| 103 | - | ||
| 104 | - | ||
| 105 | - def __call__(self) -> None: | ||
| 106 | - self.compare_loss() | ||
| 107 | - self.compare_perf() | ||
| 108 | - self.compare_memory() | ||
| 109 | - | ||
| 110 | -def main(args): | ||
| 111 | - Comparator(args.base_path_prefix, | ||
| 112 | - args.test_path_prefix, | ||
| 113 | - args.loss_error_rate, | ||
| 114 | - args.perf_error_rate, | ||
| 115 | - args.mem_error_rate, | ||
| 116 | - args.warm_up, | ||
| 117 | - args.compute_steps)() | ||
| 118 | - | ||
| 119 | - | ||
| 120 | -if __name__ == '__main__': | ||
| 121 | - parser = argparse.ArgumentParser(description='Compare log data between baseline and test') | ||
| 122 | - parser.add_argument('base_path_prefix', help='The baseline path prefix') | ||
| 123 | - parser.add_argument('test_path_prefix', help='The test path prefix') | ||
| 124 | - parser.add_argument('--loss_error_rate', type=float, default=0.02, help='The loss error rate') | ||
| 125 | - parser.add_argument('--perf_error_rate', type=float, default=0.03, help='The perf error rate') | ||
| 126 | - parser.add_argument('--warm_up', type=int, default=1, help='The perf test start from warm_up step') | ||
| 127 | - parser.add_argument('--mem_error_rate', type=float, default=0.003, help='The memory error rate') | ||
| 128 | - parser.add_argument('--compute_steps', type=int, default=2000, help='The compute steps') | ||
| 129 | - args = parser.parse_args() | ||
| 130 | - main(args) | ||
| @@ -1,157 +0,0 @@ | |||
| 1 | -from abc import ABC, abstractmethod | ||
| 2 | -import json | ||
| 3 | -import argparse | ||
| 4 | - | ||
| 5 | -class BaseLogExtractor(ABC): | ||
| 6 | - def __init__(self, input_path:str, out_path_prefix:str): | ||
| 7 | - self.input_path = input_path | ||
| 8 | - self.out_path_prefix = out_path_prefix | ||
| 9 | - self.losses = [] | ||
| 10 | - self.memories = [] | ||
| 11 | - self.parameters = {"global_batch_size": 0, | ||
| 12 | - "seq_length": 0, | ||
| 13 | - "world_size": 0} | ||
| 14 | - | ||
| 15 | - @abstractmethod | ||
| 16 | - def _extract_parameter(self, line: str) -> None: | ||
| 17 | - pass | ||
| 18 | - | ||
| 19 | - @abstractmethod | ||
| 20 | - def _extract_iterline(self, line: str) -> None: | ||
| 21 | - pass | ||
| 22 | - | ||
| 23 | - @abstractmethod | ||
| 24 | - def _extract_memory(self, line) -> None: | ||
| 25 | - pass | ||
| 26 | - | ||
| 27 | - def _extract(self) -> None: | ||
| 28 | - with open(self.input_path, encoding='utf-8') as f: | ||
| 29 | - for line in f: | ||
| 30 | - self._extract_parameter(line) | ||
| 31 | - self._extract_iterline(line) | ||
| 32 | - self._extract_memory(line) | ||
| 33 | - | ||
| 34 | - | ||
| 35 | - def _save(self) -> None: | ||
| 36 | - loss_path = f"{self.out_path_prefix}_loss.tsv" | ||
| 37 | - with open(loss_path, 'w') as f: | ||
| 38 | - f.write("step\tloss\titer_time\n") | ||
| 39 | - for step, loss, iter_time in self.losses: | ||
| 40 | - f.write(f"{step}\t{loss}\t{iter_time}\n") | ||
| 41 | - | ||
| 42 | - memory_path = f"{self.out_path_prefix}_memory.tsv" | ||
| 43 | - with open(memory_path, 'w') as f: | ||
| 44 | - f.write("rank_id\tmemory\n") | ||
| 45 | - for rank_id, memory in sorted(self.memories): | ||
| 46 | - f.write(f"{rank_id}\t{memory}\n") | ||
| 47 | - | ||
| 48 | - parameters_path = f"{self.out_path_prefix}_parameters.json" | ||
| 49 | - with open(parameters_path, 'w') as f: | ||
| 50 | - json.dump(self.parameters, f, indent=4) | ||
| 51 | - | ||
| 52 | - | ||
| 53 | - def __call__(self): | ||
| 54 | - self._extract() | ||
| 55 | - self._save() | ||
| 56 | - | ||
| 57 | - | ||
| 58 | -class MegatronLogExtractor(BaseLogExtractor): | ||
| 59 | - | ||
| 60 | - def _extract_parameter(self, line: str) -> None: | ||
| 61 | - for param in self.parameters.keys(): | ||
| 62 | - if line.startswith(f" {param}"): | ||
| 63 | - blank_pos = line.rfind(' ') | ||
| 64 | - self.parameters[param] = int(line[blank_pos:]) | ||
| 65 | - | ||
| 66 | - def _extract_iterline(self, line: str): | ||
| 67 | - | ||
| 68 | - if (len(line) < 23 or not line[22:].startswith(" iteration")) and (not line.startswith(" iteration")): | ||
| 69 | - return | ||
| 70 | - | ||
| 71 | - backslash_pos = line.find('/') | ||
| 72 | - blank_pos = line.rfind(' ', 0, backslash_pos) | ||
| 73 | - step = line[blank_pos:backslash_pos] | ||
| 74 | - ms_pos = line.find('(ms):') | ||
| 75 | - pipe_pos = line.find('|', ms_pos) | ||
| 76 | - iter_time = line[ms_pos+6: pipe_pos-1] | ||
| 77 | - loss_pos = line.find('lm loss:') | ||
| 78 | - if loss_pos > 0: | ||
| 79 | - bar_pos = line.find('|', loss_pos) | ||
| 80 | - loss = line[loss_pos+9:bar_pos-1] | ||
| 81 | - else: | ||
| 82 | - loss = 'inf' | ||
| 83 | - self.losses.append((int(step), float(loss), float(iter_time))) | ||
| 84 | - | ||
| 85 | - def _extract_memory(self, line) -> None: | ||
| 86 | - if not line.startswith("[Rank"): | ||
| 87 | - return | ||
| 88 | - | ||
| 89 | - start = 0 | ||
| 90 | - while start >= 0: | ||
| 91 | - rsb_pos = line.find(']', start) | ||
| 92 | - rankid = line[start+6:rsb_pos] | ||
| 93 | - mem_pos = line.find('allocated:', rsb_pos) | ||
| 94 | - pipe_pos = line.find('|', mem_pos) | ||
| 95 | - memory = line[mem_pos+11:pipe_pos-1] | ||
| 96 | - self.memories.append((int(rankid), float(memory))) | ||
| 97 | - start = line.find("[Rank", pipe_pos) | ||
| 98 | - | ||
| 99 | - | ||
| 100 | -class DeepSpeedLogExtractor(BaseLogExtractor): | ||
| 101 | - | ||
| 102 | - def __init__(self, input_path: str, out_path_prefix: str): | ||
| 103 | - super().__init__(input_path, out_path_prefix) | ||
| 104 | - self.parameters = { | ||
| 105 | - "train_batch_size": 0, | ||
| 106 | - "seq-length": 0 | ||
| 107 | - } | ||
| 108 | - | ||
| 109 | - def _extract_parameter(self, line: str) -> None: | ||
| 110 | - for param in self.parameters.keys(): | ||
| 111 | - param_pos = line.find(f" \"{param}\":") | ||
| 112 | - if f" \"{param}\":" in line: | ||
| 113 | - colon_pos = line.find(':', param_pos) | ||
| 114 | - comma_pos = line.find(',', colon_pos) | ||
| 115 | - self.parameters[param] = int(line[colon_pos + 1 : comma_pos]) | ||
| 116 | - | ||
| 117 | - def _extract_iterline(self, line: str): | ||
| 118 | - if not line.startswith("steps: "): | ||
| 119 | - return | ||
| 120 | - | ||
| 121 | - step_pos = 0 | ||
| 122 | - loss_pos = line.find(' loss:') | ||
| 123 | - iter_time_pos = line.find(' iter time (s):') | ||
| 124 | - iter_time_end = line.find(' samples/sec:') | ||
| 125 | - | ||
| 126 | - step = line[step_pos + 7 : loss_pos] | ||
| 127 | - iter_time = line[iter_time_pos + 15 : iter_time_end] | ||
| 128 | - if loss_pos > 0: | ||
| 129 | - loss = line[loss_pos + 6 : iter_time_pos] | ||
| 130 | - else: | ||
| 131 | - loss = 'inf' | ||
| 132 | - self.losses.append((int(step), float(loss), float(iter_time))) | ||
| 133 | - | ||
| 134 | - def _extract_memory(self, line) -> None: | ||
| 135 | - if not line.startswith("after 1 iterations memory (MB)"): | ||
| 136 | - return | ||
| 137 | - | ||
| 138 | - mem_pos = line.find('allocated: ') | ||
| 139 | - pipe_pos = line.find('|', mem_pos) | ||
| 140 | - memory = line[mem_pos + 11 : pipe_pos - 1] | ||
| 141 | - self.memories.append((0, float(memory))) | ||
| 142 | - | ||
| 143 | - | ||
| 144 | -def main(args): | ||
| 145 | - if args.frame_kind.lower() == 'megatron': | ||
| 146 | - MegatronLogExtractor(args.input_path, args.output_path_prefix)() | ||
| 147 | - if args.frame_kind.lower() == 'deepspeed': | ||
| 148 | - DeepSpeedLogExtractor(args.input_path, args.output_path_prefix)() | ||
| 149 | - | ||
| 150 | - | ||
| 151 | -if __name__ == '__main__': | ||
| 152 | - parser = argparse.ArgumentParser(description='extract loss, performance and memory data from training log') | ||
| 153 | - parser.add_argument('frame_kind', help='The training frame: Megatron, Deepspeed or Torch') | ||
| 154 | - parser.add_argument('input_path', help='The training log path') | ||
| 155 | - parser.add_argument('output_path_prefix', help='The output path prefix') | ||
| 156 | - args = parser.parse_args() | ||
| 157 | - main(args) | ||
| @@ -1,7 +0,0 @@ | |||
| 1 | -# Provide uniform access for piepline. | ||
| 2 | - | ||
| 3 | -python tests/pipeline/gemma-7B/test_process_pretrain_data.py | ||
| 4 | -python tests/pipeline/gemma-7B/test_convert_ckpt_from_huggingface.py | ||
| 5 | - | ||
| 6 | -pytest -s tests/pipeline/gemma-7B/test_generation.py | ||
| 7 | -pytest -s tests/pipeline/gemma-7B/test_evaluation.py | ||
| @@ -1,76 +0,0 @@ | |||
| 1 | -{ | ||
| 2 | - "NETWORK_SIZE": [ | ||
| 3 | - "--num-layers", "28", | ||
| 4 | - "--hidden-size", "3072", | ||
| 5 | - "--ffn-hidden-size", "24576", | ||
| 6 | - "--num-attention-heads", "16", | ||
| 7 | - "--max-position-embeddings", "8192", | ||
| 8 | - "--position-embedding-type", "rope", | ||
| 9 | - "--make-vocab-size-divisible-by", "1", | ||
| 10 | - "--normalization", "RMSNorm", | ||
| 11 | - "--add-rmsnorm-offset", | ||
| 12 | - "--geglu", | ||
| 13 | - "--kv-channels", "256", | ||
| 14 | - "--input-embeds-norm", | ||
| 15 | - "--vocab-size", "256000" | ||
| 16 | - ], | ||
| 17 | - | ||
| 18 | - "INFERENCE_AUX": [ | ||
| 19 | - "--tokenizer-type", "PretrainedFromHF", | ||
| 20 | - "--tokenizer-model", "/home/dataset/gemma-7B-hf/tokenizer.model", | ||
| 21 | - "--tokenizer-name-or-path", "/home/dataset/gemma-7B-hf", | ||
| 22 | - "--load", "/home/dataset/gemma-7B-tp8-pp1", | ||
| 23 | - "--seed", "42", | ||
| 24 | - "--tokenizer-not-use-fast", | ||
| 25 | - "--exit-on-missing-checkpoint" | ||
| 26 | - ], | ||
| 27 | - | ||
| 28 | - "INFERENCE_PARAM": [ | ||
| 29 | - "--max-new-tokens", "256" | ||
| 30 | - ], | ||
| 31 | - | ||
| 32 | - "EVALUATION_PARAM": [ | ||
| 33 | - "--task-data-path", "/home/dataset/eval_dataset/mmlu/test/", | ||
| 34 | - "--max-new-tokens", "1" | ||
| 35 | - ], | ||
| 36 | - | ||
| 37 | - | ||
| 38 | - "DISTRIBUTED_PARAM": [ | ||
| 39 | - "--tensor-model-parallel-size", "8", | ||
| 40 | - "--pipeline-model-parallel-size", "1" | ||
| 41 | - ], | ||
| 42 | - | ||
| 43 | - "AUXILIARY_PARAM": [ | ||
| 44 | - "--micro-batch-size", "1", | ||
| 45 | - "--global-batch-size", "16", | ||
| 46 | - "--no-masked-softmax-fusion", | ||
| 47 | - "--disable-bias-linear", | ||
| 48 | - "--no-gradient-accumulation-fusion", | ||
| 49 | - "--bf16", | ||
| 50 | - "--attention-softmax-in-fp32", | ||
| 51 | - "--no-load-optim", | ||
| 52 | - "--no-load-rng", | ||
| 53 | - "--seq-length", "8192" | ||
| 54 | - ], | ||
| 55 | - | ||
| 56 | - | ||
| 57 | - "PROCESS_PRETRAIN_DATA": [ | ||
| 58 | - "--input", "/home/dataset/train-00000-of-00001-a09b74b3ef9c3b56.parquet", | ||
| 59 | - "--tokenizer-type", "PretrainedFromHF", | ||
| 60 | - "--output-prefix", "/home/dataset/pretrain-dataset-gemma-7B/alpaca", | ||
| 61 | - "--tokenizer-name-or-path", "/home/dataset/gemma-7B-hf", | ||
| 62 | - "--workers", "4", | ||
| 63 | - "--log-interval", "1000" | ||
| 64 | - ], | ||
| 65 | - | ||
| 66 | - | ||
| 67 | - "CONVERT_CKPT_FROM_HF": [ | ||
| 68 | - "--model-type", "GPT", | ||
| 69 | - "--loader", "gemma_hf", | ||
| 70 | - "--saver", "megatron", | ||
| 71 | - "--target-tensor-parallel-size", "8", | ||
| 72 | - "--load-dir", "/home/dataset/gemma-7B-hf", | ||
| 73 | - "--save-dir", "/home/dataset/gemma-7B-tp8-pp1", | ||
| 74 | - "--tokenizer-model", "/home/dataset/gemma-7B-hf/tokenizer.model" | ||
| 75 | - ] | ||
| 76 | -} | ||
| @@ -1,56 +0,0 @@ | |||
| 1 | -import unittest | ||
| 2 | -import sys | ||
| 3 | -import os | ||
| 4 | -import subprocess | ||
| 5 | -import glob | ||
| 6 | -from pathlib import Path | ||
| 7 | -import torch | ||
| 8 | -from utils import ParamConfig | ||
| 9 | -import modellink | ||
| 10 | - | ||
| 11 | - | ||
| 12 | -class TestConvertCkptFromHuggingface(unittest.TestCase): | ||
| 13 | - def setUp(self, config=ParamConfig): | ||
| 14 | - # configure params, the index starts from 1 | ||
| 15 | - self.config = config | ||
| 16 | - sys.argv = [sys.argv[0]] + self.config.convert_ckpt_param | ||
| 17 | - | ||
| 18 | - def test_file_exsit(self): | ||
| 19 | - """ | ||
| 20 | - Test if the file in the `--load-dir` exsit, including `.bin`, `.json`... | ||
| 21 | - """ | ||
| 22 | - st_file = glob.glob(os.path.join(self.config.convert_ckpt_param[9], "*.safetensors")) | ||
| 23 | - self.assertEqual(len(st_file), 4) | ||
| 24 | - self.assertTrue(os.path.exists(os.path.join(self.config.convert_ckpt_param[9], "model.safetensors.index.json"))) | ||
| 25 | - | ||
| 26 | - def test_convert_weights_form_huggingface(self): | ||
| 27 | - """ | ||
| 28 | - Test whether the weight to be converted as we want in `--save-dir`. We will check the model layer name, | ||
| 29 | - including embedding, final_norm, output and encoder. In the encoder, there will be some different layers | ||
| 30 | - to compose the unique transformer layer and all these layer stack to compose the entity of the model. | ||
| 31 | - """ | ||
| 32 | - base_dir = Path(__file__).absolute().parent.parent.parent.parent | ||
| 33 | - file_path = os.path.join(base_dir, "convert_ckpt.py") | ||
| 34 | - arguments = sys.argv[1:] | ||
| 35 | - subprocess.run(["python", file_path] + arguments) | ||
| 36 | - output_dir = os.path.join(self.config.convert_ckpt_param[11], "iter_0000001") | ||
| 37 | - weight_content = torch.load(os.path.join(output_dir, "mp_rank_00/model_optim_rng.pt")) | ||
| 38 | - weight_common_content = weight_content['model']['language_model'] # extract commmon content | ||
| 39 | - | ||
| 40 | - # embedding, encoder, output_layer is three out layers. | ||
| 41 | - self.assertEqual(len(os.listdir(output_dir)), int(self.config.convert_ckpt_param[7])) | ||
| 42 | - self.assertEqual(weight_common_content['embedding']['word_embeddings']['weight'].size(), torch.Size([32000, 3072])) | ||
| 43 | - self.assertEqual(weight_common_content['encoder']['final_norm.weight'].size(), torch.Size([3072])) | ||
| 44 | - | ||
| 45 | - # encoder has a common final_norm and each one has folliowing six layers | ||
| 46 | - weight_common_content['encoder'].pop('final_norm.weight') | ||
| 47 | - self.assertEqual(weight_common_content['encoder']['layers.0.self_attention.query_key_value.weight'].size(), torch.Size([1536, 3072])) | ||
| 48 | - self.assertEqual(weight_common_content['encoder']['layers.0.self_attention.dense.weight'].size(), torch.Size([3072, 512])) | ||
| 49 | - self.assertEqual(weight_common_content['encoder']['layers.0.mlp.dense_h_to_4h.weight'].size(), torch.Size([6144, 3072])) | ||
| 50 | - self.assertEqual(weight_common_content['encoder']['layers.0.mlp.dense_4h_to_h.weight'].size(), torch.Size([3072, 3072])) | ||
| 51 | - self.assertEqual(weight_common_content['encoder']['layers.0.input_norm.weight'].size(), torch.Size([3072])) | ||
| 52 | - self.assertEqual(weight_common_content['encoder']['layers.0.post_attention_norm.weight'].size(), torch.Size([3072])) | ||
| 53 | - | ||
| 54 | - | ||
| 55 | -if __name__ == "__main__": | ||
| 56 | - unittest.main() | ||
| @@ -1,106 +0,0 @@ | |||
| 1 | -import sys | ||
| 2 | -import os | ||
| 3 | -import json | ||
| 4 | -from pathlib import Path | ||
| 5 | -import tqdm | ||
| 6 | -import pandas as pd | ||
| 7 | -import torch | ||
| 8 | -import torch_npu | ||
| 9 | -from transformers import AutoTokenizer | ||
| 10 | -from tests.common import DistributedTest | ||
| 11 | -from utils import ParamConfig, assert_judge | ||
| 12 | -import modellink | ||
| 13 | -from megatron.legacy.model import GPTModel | ||
| 14 | -from modellink.tasks.evaluation.utils import add_text_generate_args | ||
| 15 | - | ||
| 16 | - | ||
| 17 | -class TestEvaluation(DistributedTest): | ||
| 18 | - world_size = 8 | ||
| 19 | - | ||
| 20 | - def init(self, config=ParamConfig): | ||
| 21 | - sys.argv = [sys.argv[0]] + config.distributed_param + config.network_size + config.auxiliary_param + \ | ||
| 22 | - config.inference_aux + config.evaluation_param | ||
| 23 | - from megatron.training.initialize import initialize_megatron | ||
| 24 | - os.environ.update({"CUDA_DEVICE_MAX_CONNECTIONS": "1"}) | ||
| 25 | - initialize_megatron(extra_args_provider=add_text_generate_args, | ||
| 26 | - args_defaults={'no_load_rng': True, | ||
| 27 | - 'no_load_optim': True}) | ||
| 28 | - | ||
| 29 | - from megatron.training import get_args | ||
| 30 | - self.args = get_args() | ||
| 31 | - | ||
| 32 | - def get_result(self, tokenizer, result): | ||
| 33 | - if result: | ||
| 34 | - final_result = [result[0]] | ||
| 35 | - if result[1][0][tokenizer.encode("Yes")[-1]] >= result[1][0][tokenizer.encode("No")[-1]]: | ||
| 36 | - final_result.append('T') | ||
| 37 | - else: | ||
| 38 | - final_result.append('F') | ||
| 39 | - else: | ||
| 40 | - final_result = None | ||
| 41 | - return final_result | ||
| 42 | - | ||
| 43 | - def test_mmlu_evaluation(self): | ||
| 44 | - self.init(config=ParamConfig) | ||
| 45 | - from evaluation import model_provider | ||
| 46 | - from modellink.tasks.evaluation.eval_impl.template import MMLU_TEMPLATE_DIR | ||
| 47 | - model = GPTModel.from_pretrained( | ||
| 48 | - model_provider=model_provider, | ||
| 49 | - pretrained_name_or_path=self.args.load | ||
| 50 | - ) | ||
| 51 | - tokenizer = AutoTokenizer.from_pretrained(self.args.tokenizer_name_or_path) | ||
| 52 | - max_new_tokens = self.args.max_new_tokens | ||
| 53 | - | ||
| 54 | - instruction_template = "{few_shot_examples}\n\n{question}\nAnswer:" | ||
| 55 | - | ||
| 56 | - total_acc_n = 0 | ||
| 57 | - total_n = 0 | ||
| 58 | - | ||
| 59 | - test_dir = None | ||
| 60 | - for path in self.args.task_data_path: | ||
| 61 | - if "mmlu" in path: | ||
| 62 | - test_dir = path | ||
| 63 | - base_dir = Path(__file__).absolute().parent.parent.parent.parent | ||
| 64 | - template_dir = os.path.join(base_dir, MMLU_TEMPLATE_DIR) | ||
| 65 | - with open(template_dir, encoding='utf-8') as f: | ||
| 66 | - mmlu_few_shot_template = json.load(f) | ||
| 67 | - | ||
| 68 | - temp = [] | ||
| 69 | - for file in tqdm.tqdm(os.listdir(test_dir)): | ||
| 70 | - file_path = os.path.join(test_dir, file) | ||
| 71 | - data_df = pd.read_csv(file_path, names=['question', 'A', 'B', 'C', 'D', 'answer']) | ||
| 72 | - subject_name = file[0: -9] | ||
| 73 | - subject = subject_name.replace("_", " ") | ||
| 74 | - acc_n = 0 | ||
| 75 | - data_df_test = data_df[0:10] | ||
| 76 | - for index, row in data_df_test.iterrows(): | ||
| 77 | - test_question = f"{row['question']}\nA. {row['A']}\nB. {row['B']}\nC. {row['C']}\nD. {row['D']}" | ||
| 78 | - instruction = instruction_template.format(few_shot_examples=mmlu_few_shot_template[subject_name], | ||
| 79 | - subject=subject, | ||
| 80 | - question=test_question) | ||
| 81 | - chat_result = model.generate( | ||
| 82 | - instruction, | ||
| 83 | - do_sample=False, | ||
| 84 | - max_new_tokens=max_new_tokens, | ||
| 85 | - tokenizer=tokenizer, | ||
| 86 | - stream=False, | ||
| 87 | - return_output_log_probs=True | ||
| 88 | - ) | ||
| 89 | - assert_judge(isinstance(chat_result, tuple)) | ||
| 90 | - assert_judge(isinstance(chat_result[1], torch.Tensor)) | ||
| 91 | - answer = None | ||
| 92 | - if chat_result: | ||
| 93 | - answer = chat_result[0][1] | ||
| 94 | - temp.append(answer) | ||
| 95 | - if answer == row['answer']: | ||
| 96 | - acc_n += 1 | ||
| 97 | - if torch.distributed.get_rank() == 0: | ||
| 98 | - total_n += len(data_df_test) | ||
| 99 | - total_acc_n += acc_n | ||
| 100 | - if torch.distributed.get_rank() == 0: | ||
| 101 | - try: | ||
| 102 | - final_acc = total_acc_n / total_n | ||
| 103 | - except ZeroDivisionError as e: | ||
| 104 | - raise e | ||
| 105 | - print(final_acc) | ||
| 106 | - assert_judge(abs(final_acc - 0.572) < 0.1) | ||