已合并
门禁、流水整改四：完善 UT, ST 执行逻辑 #1678
AtomGit-Bot创建于 2024年9月13日
门禁、流水整改四：完善 UT, ST 执行逻辑 #1678
已合并
从refs/pull/1678/head合入到master
共 6 个文件变更+138-60
| @@ -8,7 +8,7 @@ def read_files_from_txt(txt_file): | |||
| 8 | 8 | ||
| 9 | 9 | ||
| 10 | def is_examples(file): | 10 | def is_examples(file): |
| 11 | - return file.startswith("example/") | 11 | + return file.startswith("examples/") |
| 12 | 12 | ||
| 13 | 13 | ||
| 14 | def is_pipecase(file): | 14 | def is_pipecase(file): |
| @@ -19,17 +19,22 @@ def is_markdown(file): | |||
| 19 | return file.endswith(".md") | 19 | return file.endswith(".md") |
| 20 | 20 | ||
| 21 | 21 | ||
| 22 | -def skip_ci_file(files, skip_cond): | 22 | +def is_image(file): |
| 23 | + return file.endswith(".jpg") or file.endswith(".png") | ||
| 24 | + | ||
| 25 | + | ||
| 26 | +def is_ut(file): | ||
| 27 | + return file.startswith("tests/ut") | ||
| 28 | + | ||
| 29 | + | ||
| 30 | +def skip_ci(files, skip_conds): | ||
| 23 | for file in files: | 31 | for file in files: |
| 24 | - if not any(condition(file) for condition in skip_cond): | 32 | + if not any(condition(file) for condition in skip_conds): |
| 25 | return False | 33 | return False |
| 26 | return True | 34 | return True |
| 27 | 35 | ||
| 28 | 36 | ||
| 29 | -def alter_skip_ci(): | 37 | +def choose_skip_ci(raw_txt_file): |
| 30 | - parent_dir = Path(__file__).absolute().parents[2] | ||
| 31 | - raw_txt_file = os.path.join(parent_dir, "modify.txt") | ||
| 32 | - | ||
| 33 | if not os.path.exists(raw_txt_file): | 38 | if not os.path.exists(raw_txt_file): |
| 34 | return False | 39 | return False |
| 35 | 40 | ||
| @@ -37,10 +42,22 @@ def alter_skip_ci(): | |||
| 37 | skip_conds = [ | 42 | skip_conds = [ |
| 38 | is_examples, | 43 | is_examples, |
| 39 | is_pipecase, | 44 | is_pipecase, |
| 40 | - is_markdown | 45 | + is_markdown, |
| 46 | + is_image | ||
| 41 | ] | 47 | ] |
| 42 | 48 | ||
| 43 | - return skip_ci_file(file_list, skip_conds) | 49 | + return skip_ci(file_list, skip_conds) |
| 50 | + | ||
| 51 | + | ||
| 52 | +def filter_exec_ut(raw_txt_file): | ||
| 53 | + file_list = read_files_from_txt(raw_txt_file) | ||
| 54 | + filter_conds = [ | ||
| 55 | + is_ut | ||
| 56 | + ] | ||
| 57 | + for file in file_list: | ||
| 58 | + if not any(condition(file) for condition in filter_conds): | ||
| 59 | + return False, None | ||
| 60 | + return True, file_list | ||
| 44 | 61 | ||
| 45 | 62 | ||
| 46 | def acquire_exitcode(command): | 63 | def acquire_exitcode(command): |
| @@ -53,16 +70,25 @@ def acquire_exitcode(command): | |||
| 53 | # UT test, run with pytest | 70 | # UT test, run with pytest |
| 54 | # ============================= | 71 | # ============================= |
| 55 | 72 | ||
| 56 | -class UT_Test: | 73 | +class UTTest: |
| 57 | - | ||
| 58 | def __init__(self): | 74 | def __init__(self): |
| 59 | - | 75 | + self.base_dir = Path(__file__).absolute().parents[1] |
| 60 | - base_dir = Path(__file__).absolute().parent.parent | 76 | + self.test_dir = os.path.join(self.base_dir, 'tests') |
| 61 | - test_dir = os.path.join(base_dir, 'tests') | 77 | + self.ut_files = os.path.join( |
| 62 | - self.ut_file = os.path.join(test_dir, "ut") | 78 | + self.base_dir, self.test_dir, "ut" |
| 79 | + ) | ||
| 63 | 80 | ||
| 64 | - def run_ut(self): | 81 | + def run_ut(self, raw_txt_file=None): |
| 65 | - command = f"pytest -x {self.ut_file}" | 82 | + if raw_txt_file is not None and os.path.exists(raw_txt_file): |
| 83 | + filtered_results = filter_exec_ut(raw_txt_file) | ||
| 84 | + | ||
| 85 | + if filtered_results[0]: | ||
| 86 | + filtered_files = filtered_results[1] | ||
| 87 | + full_path = [os.path.join(self.base_dir, file) for file in filtered_files] | ||
| 88 | + exsit_ut_files = [file for file in full_path if os.path.exists(file) and file.endswith(".py")] | ||
| 89 | + self.ut_files = " ".join(exsit_ut_files) | ||
| 90 | + | ||
| 91 | + command = f"pytest -x {self.ut_files}" | ||
| 66 | code = acquire_exitcode(command) | 92 | code = acquire_exitcode(command) |
| 67 | if code == 0: | 93 | if code == 0: |
| 68 | print("UT test success") | 94 | print("UT test success") |
| @@ -75,16 +101,14 @@ class UT_Test: | |||
| 75 | # ST test, run with sh. | 101 | # ST test, run with sh. |
| 76 | # =============================================== | 102 | # =============================================== |
| 77 | 103 | ||
| 78 | -class ST_Test: | 104 | +class STTest: |
| 79 | - | ||
| 80 | def __init__(self): | 105 | def __init__(self): |
| 106 | + self.base_dir = Path(__file__).absolute().parents[1] | ||
| 107 | + self.test_dir = os.path.join(self.base_dir, 'tests') | ||
| 81 | 108 | ||
| 82 | - base_dir = Path(__file__).absolute().parent.parent | 109 | + self.st_dir = "st" |
| 83 | - test_dir = os.path.join(base_dir, 'tests') | ||
| 84 | - | ||
| 85 | - st_dir = "st" | ||
| 86 | self.st_shell = os.path.join( | 110 | self.st_shell = os.path.join( |
| 87 | - test_dir, st_dir, "st_run.sh" | 111 | + self.test_dir, self.st_dir, "st_run.sh" |
| 88 | ) | 112 | ) |
| 89 | 113 | ||
| 90 | def run_st(self): | 114 | def run_st(self): |
| @@ -95,19 +119,25 @@ class ST_Test: | |||
| 95 | exit(1) | 119 | exit(1) |
| 96 | 120 | ||
| 97 | 121 | ||
| 98 | -def run_tests(): | 122 | +def run_tests(raw_txt_file): |
| 99 | - ut = UT_Test() | 123 | + ut = UTTest() |
| 100 | - st = ST_Test() | 124 | + st = STTest() |
| 101 | - | 125 | + if filter_exec_ut(raw_txt_file)[0]: |
| 102 | - ut.run_ut() | 126 | + ut.run_ut(raw_txt_file) |
| 103 | - st.run_st() | 127 | + else: |
| 128 | + ut.run_ut() | ||
| 129 | + st.run_st() | ||
| 104 | 130 | ||
| 105 | 131 | ||
| 106 | def main(): | 132 | def main(): |
| 107 | - if alter_skip_ci(): | 133 | + parent_dir = Path(__file__).absolute().parents[2] |
| 134 | + raw_txt_file = os.path.join(parent_dir, "modify.txt") | ||
| 135 | + | ||
| 136 | + skip_signal = choose_skip_ci(raw_txt_file) | ||
| 137 | + if skip_signal: | ||
| 108 | print("Skipping CI") | 138 | print("Skipping CI") |
| 109 | else: | 139 | else: |
| 110 | - run_tests() | 140 | + run_tests(raw_txt_file) |
| 111 | 141 | ||
| 112 | if __name__ == "__main__": | 142 | if __name__ == "__main__": |
| 113 | main() | 143 | main() |
| @@ -269,7 +269,7 @@ | |||
| 269 | 269 | ||
| 270 | ① 贡献脚本用例请放置于 `st/shell_scripts` 文件夹下，命名规则为 **{模型名}_{切分策略}** 或者 **{模型名}_{特性名称}**， 如 `llama2_tp2_pp4_vpp2_ptd.sh`，请贡献者严格对齐； | 270 | ① 贡献脚本用例请放置于 `st/shell_scripts` 文件夹下，命名规则为 **{模型名}_{切分策略}** 或者 **{模型名}_{特性名称}**， 如 `llama2_tp2_pp4_vpp2_ptd.sh`，请贡献者严格对齐； |
| 271 | 271 | ||
| 272 | -② 注意脚本用例中不需要单独重定向log，日志收集工作已在 `run.sh` 中统一管理； | 272 | +② 注意脚本用例中不需要单独重定向log，日志收集工作已在 `st_run.sh` 中统一管理； |
| 273 | 273 | ||
| 274 | ③ 标杆数据请放置于 `st/baseline_results` 文件夹下，**命名保证完全与 shell 脚本对齐**，否则自动化脚本执行将扫描不到； | 274 | ③ 标杆数据请放置于 `st/baseline_results` 文件夹下，**命名保证完全与 shell 脚本对齐**，否则自动化脚本执行将扫描不到； |
| 275 | 275 | ||
| @@ -43,28 +43,3 @@ def pytest_fixture_setup(fixturedef, request): | |||
| 43 | dist_fixture_class(request) | 43 | dist_fixture_class(request) |
| 44 | 44 | ||
| 45 | 45 | ||
| 46 | -# we still want to configure path argument by ourselves | ||
| 47 | -# for different prefix_name of different scripts so we use this method. | ||
| 48 | -# One more thing, as you can see, it has more scalibility. | ||
| 49 | -def pytest_addoption(parser: pytest.Parser): | ||
| 50 | - parser.addoption("--baseline-json", action="store", default=None, | ||
| 51 | - help="Path to the baseline JSON file") | ||
| 52 | - parser.addoption("--generate-log", action="store", default=None, | ||
| 53 | - help="Path to the generate log file") | ||
| 54 | - parser.addoption("--generate-json", action="store", default=None, | ||
| 55 | - help="Path to the generate JSON file") | ||
| 56 | - | ||
| 57 | - | ||
| 58 | - | ||
| 59 | -def baseline_json(request: pytest.FixtureRequest): | ||
| 60 | - return request.config.getoption("--baseline-json") | ||
| 61 | - | ||
| 62 | - | ||
| 63 | - | ||
| 64 | -def generate_log(request: pytest.FixtureRequest): | ||
| 65 | - return request.config.getoption("--generate-log") | ||
| 66 | - | ||
| 67 | - | ||
| 68 | - | ||
| 69 | -def generate_json(request: pytest.FixtureRequest): | ||
| 70 | - return request.config.getoption("--generate-json") | ||
| @@ -23,8 +23,8 @@ find "$BASE_DIR" -mindepth 1 -maxdepth 1 -type d | while read -r dir; do | |||
| 23 | fi | 23 | fi |
| 24 | 24 | ||
| 25 | # begin to execute the logic of compare | 25 | # begin to execute the logic of compare |
| 26 | - echo "$(dirname "$BASE_DIR")/st/st_utils/test_ci_st.py" | 26 | + echo "$(dirname "$BASE_DIR")/test_tools/test_ci_st.py" |
| 27 | - pytest -x $(dirname "$BASE_DIR")/st/st_utils/test_ci_st.py \ | 27 | + pytest -x $(dirname "$BASE_DIR")/test_tools/test_ci_st.py \ |
| 28 | --baseline-json $BASELINE_DIR/$name.json \ | 28 | --baseline-json $BASELINE_DIR/$name.json \ |
| 29 | --generate-log $GENERATE_LOG_DIR/$name.log \ | 29 | --generate-log $GENERATE_LOG_DIR/$name.log \ |
| 30 | --generate-json $GENERATE_JSON_DIR/$name.json | 30 | --generate-json $GENERATE_JSON_DIR/$name.json |
| @@ -0,0 +1,28 @@ | |||
| 1 | +import pytest | ||
| 2 | + | ||
| 3 | + | ||
| 4 | +# we still want to configure path argument by ourselves | ||
| 5 | +# for different prefix_name of different scripts so we use this method. | ||
| 6 | +# One more thing, as you can see, it has more scalibility. | ||
| 7 | +def pytest_addoption(parser: pytest.Parser): | ||
| 8 | + parser.addoption("--baseline-json", action="store", default=None, | ||
| 9 | + help="Path to the baseline JSON file") | ||
| 10 | + parser.addoption("--generate-log", action="store", default=None, | ||
| 11 | + help="Path to the generate log file") | ||
| 12 | + parser.addoption("--generate-json", action="store", default=None, | ||
| 13 | + help="Path to the generate JSON file") | ||
| 14 | + | ||
| 15 | + | ||
| 16 | + | ||
| 17 | +def baseline_json(request: pytest.FixtureRequest): | ||
| 18 | + return request.config.getoption("--baseline-json") | ||
| 19 | + | ||
| 20 | + | ||
| 21 | + | ||
| 22 | +def generate_log(request: pytest.FixtureRequest): | ||
| 23 | + return request.config.getoption("--generate-log") | ||
| 24 | + | ||
| 25 | + | ||
| 26 | + | ||
| 27 | +def generate_json(request: pytest.FixtureRequest): | ||
| 28 | + return request.config.getoption("--generate-json") | ||
| @@ -0,0 +1,45 @@ | |||
| 1 | +# Copyright (c) Microsoft Corporation. | ||
| 2 | +# | ||
| 3 | +# This source code is licensed under the Apache license found in the | ||
| 4 | +# LICENSE file in the root directory of this source tree. | ||
| 5 | + | ||
| 6 | +# copied from https://github.com/microsoft/DeepSpeed/blob/master/tests/conftest.py | ||
| 7 | +# reworked/refactored some parts to make it run. | ||
| 8 | +import pytest | ||
| 9 | + | ||
| 10 | + | ||
| 11 | +def pytest_configure(config): | ||
| 12 | + config.option.color = "yes" | ||
| 13 | + config.option.durations = 0 | ||
| 14 | + config.option.durations_min = 1 | ||
| 15 | + config.option.verbose = True | ||
| 16 | + | ||
| 17 | + | ||
| 18 | +# Override of pytest "runtest" for DistributedTest class | ||
| 19 | +# This hook is run before the default pytest_runtest_call | ||
| 20 | + | ||
| 21 | +def pytest_runtest_call(item): | ||
| 22 | + # We want to use our own launching function for distributed tests | ||
| 23 | + if getattr(item.cls, "is_dist_test", False): | ||
| 24 | + dist_test_class = item.cls() | ||
| 25 | + dist_test_class(item._request) | ||
| 26 | + item.runtest = lambda: True # Dummy function so test is not run twice | ||
| 27 | + | ||
| 28 | + | ||
| 29 | +# We allow DistributedTest to reuse distributed environments. When the last | ||
| 30 | +# test for a class is run, we want to make sure those distributed environments | ||
| 31 | +# are destroyed. | ||
| 32 | +def pytest_runtest_teardown(item, nextitem): | ||
| 33 | + if getattr(item.cls, "reuse_dist_env", False) and not nextitem: | ||
| 34 | + dist_test_class = item.cls() | ||
| 35 | + for num_procs, pool in dist_test_class._pool_cache.items(): | ||
| 36 | + dist_test_class._close_pool(pool, num_procs, force=True) | ||
| 37 | + | ||
| 38 | + | ||
| 39 | + | ||
| 40 | +def pytest_fixture_setup(fixturedef, request): | ||
| 41 | + if getattr(fixturedef.func, "is_dist_fixture", False): | ||
| 42 | + dist_fixture_class = fixturedef.func() | ||
| 43 | + dist_fixture_class(request) | ||
| 44 | + | ||
| 45 | + | ||