Closed
【Feature】精度比对工具支持自定义比对算法 #956
HowSir_XAdd on Aug 15Closed at Aug 18
【Feature】精度比对工具支持自定义比对算法 #956
Closed
Merged fromHowSir_X/msprobemscompare_refactorintoAscend/msprobemaster
Total: 25 files changed+1860-881
| @@ -15,12 +15,69 @@ | |||
| 15 | # ------------------------------------------------------------------------- | 15 | # ------------------------------------------------------------------------- |
| 16 | 16 | ||
| 17 | # pylint: disable=too-many-lines | 17 | # pylint: disable=too-many-lines |
| 18 | - | 18 | +from __future__ import annotations |
| 19 | 19 | ||
| 20 | import os | 20 | import os |
| 21 | import stat | 21 | import stat |
| 22 | 22 | ||
| 23 | import numpy as np | 23 | import numpy as np |
| 24 | +from typing import Callable, Generic, TypeVar, Any, List, Dict | ||
| 25 | +import threading | ||
| 26 | + | ||
| 27 | +_T = TypeVar("_T") | ||
| 28 | + | ||
| 29 | + | ||
| 30 | +class LazyConst(Generic[_T]): | ||
| 31 | + """懒求值的常量""" | ||
| 32 | + | ||
| 33 | + def __init__(self, factory: Callable[[], _T]) -> None: | ||
| 34 | + if isinstance(factory, staticmethod): | ||
| 35 | + factory = factory.__func__ | ||
| 36 | + self.factory: Callable[[], _T] = factory | ||
| 37 | + self._value: _T | None = None | ||
| 38 | + self._initialized: bool = False | ||
| 39 | + self._lock = threading.RLock() | ||
| 40 | + | ||
| 41 | + def __get__(self, instance: Any | None, owner: type[Any]) -> _T: | ||
| 42 | + if not self._initialized: | ||
| 43 | + with self._lock: | ||
| 44 | + if not self._initialized: | ||
| 45 | + self._value = self.factory() | ||
| 46 | + self._initialized = True | ||
| 47 | + return self._value | ||
| 48 | + | ||
| 49 | + | ||
| 50 | +_KT = TypeVar("_KT") | ||
| 51 | +_VT = TypeVar("_VT") | ||
| 52 | + | ||
| 53 | + | ||
| 54 | +class LazyDict(dict, Generic[_KT, _VT]): | ||
| 55 | + """按 key 懒求值的字典""" | ||
| 56 | + | ||
| 57 | + _factories: Dict[_KT, Callable[[], _VT]] | ||
| 58 | + | ||
| 59 | + def __init__(self, factories: Dict[_KT, Callable[[], _VT]]) -> None: | ||
| 60 | + self._factories = factories | ||
| 61 | + super().__init__() | ||
| 62 | + self._lock = threading.RLock() | ||
| 63 | + | ||
| 64 | + def __getitem__(self, key: _KT) -> _VT: | ||
| 65 | + if key in self: | ||
| 66 | + return super().__getitem__(key) | ||
| 67 | + | ||
| 68 | + with self._lock: | ||
| 69 | + if key not in self and key in self._factories: | ||
| 70 | + self[key] = self._factories[key]() | ||
| 71 | + return super().__getitem__(key) | ||
| 72 | + | ||
| 73 | + def get(self, key: _KT, default: _VT = None) -> _VT: | ||
| 74 | + if key in self: | ||
| 75 | + return super().get(key, default) | ||
| 76 | + | ||
| 77 | + with self._lock: | ||
| 78 | + if key not in self and key in self._factories: | ||
| 79 | + self[key] = self._factories[key]() | ||
| 80 | + return super().get(key, default) | ||
| 24 | 81 | ||
| 25 | 82 | ||
| 26 | class Const: # pylint: disable=too-many-lines | 83 | class Const: # pylint: disable=too-many-lines |
| @@ -695,7 +752,7 @@ class CompareConst: | |||
| 695 | VERL_HYPER_PARAM_COMPARE_COLUM = [HYPER_PARAM_NAME, NPU_EFFECTIVE_VALUE, BENCH_EFFECTIVE_VALUE, IS_CONSISTENT] | 752 | VERL_HYPER_PARAM_COMPARE_COLUM = [HYPER_PARAM_NAME, NPU_EFFECTIVE_VALUE, BENCH_EFFECTIVE_VALUE, IS_CONSISTENT] |
| 696 | VERL_HYPER_PARAM_VERIFY_COLUM = [KEY_PARAM_NAME, BENCH_EFFECTIVE_VALUE, TARGET_EFFECTIVE_VALUE, IS_CONSISTENT] | 753 | VERL_HYPER_PARAM_VERIFY_COLUM = [KEY_PARAM_NAME, BENCH_EFFECTIVE_VALUE, TARGET_EFFECTIVE_VALUE, IS_CONSISTENT] |
| 697 | 754 | ||
| 698 | - ALL_COMPARE_INDEX = [ | 755 | + BUILTIN_COMPARE_COLUMNS = [ |
| 699 | COSINE, | 756 | COSINE, |
| 700 | EUC_DIST, | 757 | EUC_DIST, |
| 701 | MAX_ABS_ERR, | 758 | MAX_ABS_ERR, |
| @@ -703,6 +760,17 @@ class CompareConst: | |||
| 703 | ONE_THOUSANDTH_ERR_RATIO, | 760 | ONE_THOUSANDTH_ERR_RATIO, |
| 704 | FIVE_THOUSANDTHS_ERR_RATIO, | 761 | FIVE_THOUSANDTHS_ERR_RATIO, |
| 705 | ] | 762 | ] |
| 763 | + | ||
| 764 | + @staticmethod | ||
| 765 | + def _get_custom_compare_index() -> List[str]: | ||
| 766 | + from msprobe.core.compare.algorithm.algorithm_scheduler import AlgorithmScheduler | ||
| 767 | + | ||
| 768 | + algorithm_columns = AlgorithmScheduler.get_instance().get_algorithm_column_names() | ||
| 769 | + exist_compare_cols = [c for c in CompareConst.BUILTIN_COMPARE_COLUMNS if c in algorithm_columns] | ||
| 770 | + rest_columns = [col for col in algorithm_columns if col not in exist_compare_cols] | ||
| 771 | + return exist_compare_cols + rest_columns | ||
| 772 | + | ||
| 773 | + ALL_COMPARE_INDEX = LazyConst(_get_custom_compare_index) | ||
| 706 | SUMMARY_COMPARE_INDEX = [ | 774 | SUMMARY_COMPARE_INDEX = [ |
| 707 | MAX_DIFF, | 775 | MAX_DIFF, |
| 708 | MIN_DIFF, | 776 | MIN_DIFF, |
| @@ -737,23 +805,32 @@ class CompareConst: | |||
| 737 | BENCH_NORM, | 805 | BENCH_NORM, |
| 738 | ] | 806 | ] |
| 739 | 807 | ||
| 740 | - COMPARE_RESULT_HEADER = BASIC_INFO + ALL_COMPARE_INDEX + SUMMARY_INFO + EXTRACT_INDEX | 808 | + COMPARE_RESULT_HEADER = LazyConst( |
| 809 | + lambda: ( | ||
| 810 | + CompareConst.BASIC_INFO | ||
| 811 | + + CompareConst.ALL_COMPARE_INDEX | ||
| 812 | + + CompareConst.SUMMARY_INFO | ||
| 813 | + + CompareConst.EXTRACT_INDEX | ||
| 814 | + ) | ||
| 815 | + ) | ||
| 741 | 816 | ||
| 742 | SUMMARY_COMPARE_RESULT_HEADER = BASIC_INFO + SUMMARY_COMPARE_INDEX + SUMMARY_INFO + EXTRACT_INDEX | 817 | SUMMARY_COMPARE_RESULT_HEADER = BASIC_INFO + SUMMARY_COMPARE_INDEX + SUMMARY_INFO + EXTRACT_INDEX |
| 743 | 818 | ||
| 744 | MD5_COMPARE_RESULT_HEADER = BASIC_INFO + [NPU_MD5, BENCH_MD5] + EXTRACT_INDEX | 819 | MD5_COMPARE_RESULT_HEADER = BASIC_INFO + [NPU_MD5, BENCH_MD5] + EXTRACT_INDEX |
| 745 | 820 | ||
| 746 | - COMPARE_RESULT_HEADER_STACK = COMPARE_RESULT_HEADER + [STACK] | 821 | + COMPARE_RESULT_HEADER_STACK = LazyConst(lambda: CompareConst.COMPARE_RESULT_HEADER + [CompareConst.STACK]) |
| 747 | 822 | ||
| 748 | SUMMARY_COMPARE_RESULT_HEADER_STACK = SUMMARY_COMPARE_RESULT_HEADER + [STACK] | 823 | SUMMARY_COMPARE_RESULT_HEADER_STACK = SUMMARY_COMPARE_RESULT_HEADER + [STACK] |
| 749 | 824 | ||
| 750 | MD5_COMPARE_RESULT_HEADER_STACK = MD5_COMPARE_RESULT_HEADER + [STACK] | 825 | MD5_COMPARE_RESULT_HEADER_STACK = MD5_COMPARE_RESULT_HEADER + [STACK] |
| 751 | 826 | ||
| 752 | - HEAD_OF_COMPARE_MODE = { | 827 | + HEAD_OF_COMPARE_MODE: LazyDict[str, list[str]] = LazyDict( |
| 753 | - Const.ALL: COMPARE_RESULT_HEADER, | 828 | + { |
| 754 | - Const.SUMMARY: SUMMARY_COMPARE_RESULT_HEADER, | 829 | + Const.ALL: lambda: CompareConst.COMPARE_RESULT_HEADER, |
| 755 | - Const.MD5: MD5_COMPARE_RESULT_HEADER, | 830 | + Const.SUMMARY: lambda: CompareConst.SUMMARY_COMPARE_RESULT_HEADER, |
| 756 | - } | 831 | + Const.MD5: lambda: CompareConst.MD5_COMPARE_RESULT_HEADER, |
| 832 | + } | ||
| 833 | + ) | ||
| 757 | 834 | ||
| 758 | # dtype match | 835 | # dtype match |
| 759 | 836 | ||
| @@ -1627,15 +1627,8 @@ class CalcStatsDiff: | |||
| 1627 | result_df.loc[~condition_no_bench, [CompareConst.RESULT, CompareConst.ERROR_MESSAGE]] = '' | 1627 | result_df.loc[~condition_no_bench, [CompareConst.RESULT, CompareConst.ERROR_MESSAGE]] = '' |
| 1628 | result_df.loc[~condition_req_grad_consist, CompareConst.ERROR_MESSAGE] += 'Requires_grad inconsistent. ' | 1628 | result_df.loc[~condition_req_grad_consist, CompareConst.ERROR_MESSAGE] += 'Requires_grad inconsistent. ' |
| 1629 | else: | 1629 | else: |
| 1630 | - fill_cols = [ | 1630 | + fill_cols = CompareConst.ALL_COMPARE_INDEX.copy() |
| 1631 | - CompareConst.COSINE, | 1631 | + fill_cols.append(CompareConst.ERROR_MESSAGE) |
| 1632 | - CompareConst.EUC_DIST, | ||
| 1633 | - CompareConst.MAX_ABS_ERR, | ||
| 1634 | - CompareConst.MAX_RELATIVE_ERR, | ||
| 1635 | - CompareConst.ONE_THOUSANDTH_ERR_RATIO, | ||
| 1636 | - CompareConst.FIVE_THOUSANDTHS_ERR_RATIO, | ||
| 1637 | - CompareConst.ERROR_MESSAGE, | ||
| 1638 | - ] | ||
| 1639 | result_df.loc[~condition_no_bench, fill_cols] = ( | 1632 | result_df.loc[~condition_no_bench, fill_cols] = ( |
| 1640 | '' # 默认填充'', df默认省缺值为nan,不便后续处理,容易出现意外情况 | 1633 | '' # 默认填充'', df默认省缺值为nan,不便后续处理,容易出现意外情况 |
| 1641 | ) | 1634 | ) |
The file is empty
| @@ -0,0 +1,247 @@ | |||
| 1 | +# ------------------------------------------------------------------------- | ||
| 2 | +# This file is part of the MindStudio project. | ||
| 3 | +# Copyright (c) 2025 Huawei Technologies Co.,Ltd. | ||
| 4 | +# | ||
| 5 | +# MindStudio is licensed under Mulan PSL v2. | ||
| 6 | +# You can use this software according to the terms and conditions of the Mulan PSL v2. | ||
| 7 | +# You may obtain a copy of Mulan PSL v2 at: | ||
| 8 | +# | ||
| 9 | +# http://license.coscl.org.cn/MulanPSL2 | ||
| 10 | +# | ||
| 11 | +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, | ||
| 12 | +# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, | ||
| 13 | +# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. | ||
| 14 | +# See the Mulan PSL v2 for more details. | ||
| 15 | +# ------------------------------------------------------------------------- | ||
| 16 | + | ||
| 17 | +""" | ||
| 18 | +Function: | ||
| 19 | +AlgorithmScheduler class. | ||
| 20 | +Manages builtin and custom compare algorithms. | ||
| 21 | +""" | ||
| 22 | + | ||
| 23 | +import importlib | ||
| 24 | +import inspect | ||
| 25 | +import os | ||
| 26 | +import re | ||
| 27 | +import sys | ||
| 28 | +import threading | ||
| 29 | +import numpy as np | ||
| 30 | +import torch | ||
| 31 | +from typing import Any, Dict, List | ||
| 32 | +from msprobe.core.common.log import logger | ||
| 33 | + | ||
| 34 | + | ||
| 35 | +class AlgorithmScheduler: | ||
| 36 | + """ | ||
L | |||
| 37 | + The class for algorithm manager | ||
| 38 | + Singleton: please use AlgorithmScheduler.get_instance() to get instance, do not call constructor directly | ||
| 39 | + """ | ||
| 40 | + | ||
| 41 | + ALGORITHM_PATH = "msprobe.core.compare.algorithm" | ||
| 42 | + BUILT_IN_ALGORITHM_DIR_NAME = "builtin_algorithm" | ||
| 43 | + CUSTOM_ALGORITHM_DIR_NAME = "custom_algorithm" | ||
| 44 | + ALGORITHM_FILE_NAME_PATTERN = r"^alg_([a-z0-9_]+)\.py$" | ||
| 45 | + COMPARE_FUNC_NAME = "compare" | ||
| 46 | + COLUMN_NAME_FUNC_NAME = "column_name" | ||
| 47 | + ARG_COUNT = 2 # compare方法入参的个数 | ||
| 48 | + RESULT_COUNT = 2 # compare方法返回值的个数 | ||
| 49 | + | ||
| 50 | + _instance = None | ||
| 51 | + _lock = threading.Lock() | ||
| 52 | + | ||
| 53 | + @classmethod | ||
| 54 | + def get_instance(cls): | ||
| 55 | + """Thread‑safe lazy singleton entry, only this way to get instance""" | ||
| 56 | + if cls._instance is None: | ||
| 57 | + with cls._lock: | ||
| 58 | + # double‑check lock | ||
| 59 | + if cls._instance is None: | ||
| 60 | + instance = super().__new__(cls) | ||
| 61 | + instance.__init__() # pylint: disable=unnecessary-dunder-call | ||
| 62 | + cls._instance = instance | ||
| 63 | + return cls._instance | ||
| 64 | + | ||
| 65 | + def __new__(cls): | ||
| 66 | + raise RuntimeError("Do not instantiate AlgorithmScheduler directly, use AlgorithmScheduler.get_instance()") | ||
| 67 | + | ||
| 68 | + def __init__(self): | ||
| 69 | + # only invoked inside get_instance | ||
| 70 | + if os.path.dirname(__file__) not in sys.path: | ||
| 71 | + sys.path.append(os.path.dirname(__file__)) | ||
| 72 | + self.build_in_support_algorithm: List[str] = [] | ||
L 【review】sys.path.append 污染全局模块搜索路径,存在模块名冲突风险
这样与项目其他模块的导入方式保持一致,且避免全局副作用 ![]() ![]() | |||
| 73 | + self.custom_support_algorithm: List[str] = [] | ||
| 74 | + # 为了校验column_name唯一性,初始化全量加载 | ||
| 75 | + self._module_cache: Dict[str, Any] = {} | ||
| 76 | + self._column_name_algorithm_mapping: Dict[str, str] = {} # 算法名 -> 列名 | ||
| 77 | + self._make_support_algorithm() | ||
| 78 | + self.algorithm_names = self.build_in_support_algorithm + self.custom_support_algorithm | ||
| 79 | + self._validate_column_names() | ||
| 80 | + self.algorithm_column_names = list(self._column_name_algorithm_mapping.keys()) | ||
L 【review】init 中 _validate_column_names 会触发全部算法模块的导入,与 _algorithm_cache 的懒加载设计意图矛盾
![]() ![]() | |||
| 81 | + | ||
| 82 | + def _make_support_algorithm(self) -> None: | ||
| 83 | + buildin_dir = os.path.join(os.path.dirname(__file__), self.BUILT_IN_ALGORITHM_DIR_NAME) | ||
| 84 | + self._make_support_algorithm_from_dir(buildin_dir, self.build_in_support_algorithm) | ||
| 85 | + custom_dir = os.path.join(os.path.dirname(__file__), self.CUSTOM_ALGORITHM_DIR_NAME) | ||
| 86 | + self._make_support_algorithm_from_dir(custom_dir, self.custom_support_algorithm) | ||
| 87 | + | ||
| 88 | + def _make_support_algorithm_from_dir(self, dir_path: str, algorithm_list: List[str]) -> None: | ||
| 89 | + if not os.path.exists(dir_path): | ||
| 90 | + logger.warning(f"The algorithm directory {dir_path} does not exist, skip.") | ||
| 91 | + return | ||
| 92 | + | ||
| 93 | + for file_name in os.listdir(dir_path): | ||
| 94 | + file_path = os.path.join(dir_path, file_name) | ||
| 95 | + self._add_algorithm_file_to_list(file_path, algorithm_list) | ||
| 96 | + | ||
| 97 | + def _validate_column_names(self) -> None: | ||
| 98 | + all_algorithms = self.get_algorithm_names() | ||
| 99 | + for algo_name in all_algorithms: | ||
| 100 | + module = self._get_module(algo_name) | ||
| 101 | + col_name = self._get_column_name(algo_name, module) | ||
| 102 | + if col_name in self._column_name_algorithm_mapping: | ||
| 103 | + raise ValueError( | ||
| 104 | + f"The column name {col_name} is duplicated for algorithm {algo_name} " | ||
| 105 | + f"and {self._column_name_algorithm_mapping[col_name]}." | ||
| 106 | + ) | ||
| 107 | + self._column_name_algorithm_mapping[col_name] = algo_name | ||
| 108 | + for col_name, algo_name in self._column_name_algorithm_mapping.items(): | ||
| 109 | + logger.info(f"Load algorithm [{algo_name}] success, column: {col_name}") | ||
| 110 | + | ||
| 111 | + @staticmethod | ||
| 112 | + def _add_algorithm_file_to_list(file_path: str, algorithm_list: List[str]) -> bool: | ||
| 113 | + if not os.path.isfile(file_path): | ||
| 114 | + return False | ||
| 115 | + file_name = os.path.basename(file_path) | ||
| 116 | + match_result = re.match(AlgorithmScheduler.ALGORITHM_FILE_NAME_PATTERN, file_name) | ||
| 117 | + if not match_result: | ||
| 118 | + logger.warning( | ||
| 119 | + f"The algorithm file {file_name} does not match the pattern " | ||
| 120 | + f"{AlgorithmScheduler.ALGORITHM_FILE_NAME_PATTERN}, skip." | ||
| 121 | + ) | ||
| 122 | + return False | ||
| 123 | + algorithm_list.append(match_result.group(1)) | ||
| 124 | + return True | ||
| 125 | + | ||
| 126 | + def _get_module(self, algorithm_name: str) -> Any: | ||
| 127 | + if algorithm_name in self._module_cache: | ||
| 128 | + return self._module_cache[algorithm_name] | ||
| 129 | + if algorithm_name in self.custom_support_algorithm: | ||
| 130 | + module_name = f"{self.ALGORITHM_PATH}.{self.CUSTOM_ALGORITHM_DIR_NAME}.alg_{algorithm_name}" | ||
| 131 | + algorithm_module = importlib.import_module(module_name) | ||
| 132 | + elif algorithm_name in self.build_in_support_algorithm: | ||
| 133 | + module_name = f"{self.ALGORITHM_PATH}.{self.BUILT_IN_ALGORITHM_DIR_NAME}.alg_{algorithm_name}" | ||
| 134 | + algorithm_module = importlib.import_module(module_name) | ||
| 135 | + else: | ||
| 136 | + raise ValueError(f"Algorithm {algorithm_name} is not supported.") | ||
| 137 | + | ||
| 138 | + if not hasattr(algorithm_module, self.COMPARE_FUNC_NAME): | ||
| 139 | + raise AttributeError(f"Algorithm {algorithm_name} must define a {self.COMPARE_FUNC_NAME} function.") | ||
| 140 | + | ||
| 141 | + compare_func = getattr(algorithm_module, self.COMPARE_FUNC_NAME) | ||
| 142 | + if not callable(compare_func): | ||
| 143 | + raise TypeError(f"{self.COMPARE_FUNC_NAME} in {algorithm_name} is not a callable function.") | ||
| 144 | + | ||
| 145 | + sig = inspect.signature(compare_func) | ||
| 146 | + required_params = [ | ||
| 147 | + p | ||
| 148 | + for p in sig.parameters.values() | ||
| 149 | + if p.default is inspect.Parameter.empty | ||
| 150 | + and p.kind in (inspect.Parameter.POSITIONAL_ONLY, inspect.Parameter.POSITIONAL_OR_KEYWORD) | ||
| 151 | + ] | ||
| 152 | + if len(required_params) != self.ARG_COUNT: | ||
| 153 | + raise ValueError( | ||
L 【review】使用 co_argcount 校验入参数量,无法正确处理带默认参数、*args 或装饰器的 compare 函数
![]() ![]() | |||
| 154 | + f"{self.COMPARE_FUNC_NAME} in {algorithm_name} must have {self.ARG_COUNT} positional arguments." | ||
| 155 | + ) | ||
| 156 | + | ||
| 157 | + self._module_cache[algorithm_name] = algorithm_module | ||
| 158 | + return algorithm_module | ||
| 159 | + | ||
| 160 | + def _get_column_name(self, algorithm_name: str, algorithm_module: Any) -> str: | ||
| 161 | + if not hasattr(algorithm_module, self.COLUMN_NAME_FUNC_NAME): | ||
| 162 | + raise AttributeError(f"Algorithm {algorithm_name} must define a {self.COLUMN_NAME_FUNC_NAME} function.") | ||
| 163 | + column_name_func = getattr(algorithm_module, self.COLUMN_NAME_FUNC_NAME) | ||
| 164 | + if not callable(column_name_func): | ||
| 165 | + raise TypeError(f"{self.COLUMN_NAME_FUNC_NAME} in {algorithm_name} is not a callable function.") | ||
| 166 | + try: | ||
| 167 | + column_name = column_name_func() | ||
| 168 | + except Exception as e: | ||
| 169 | + raise ValueError(f"Error occurred when calling {self.COLUMN_NAME_FUNC_NAME} in {algorithm_name}: {e}") | ||
| 170 | + if not isinstance(column_name, str) or not column_name.strip(): | ||
| 171 | + raise ValueError(f"{self.COLUMN_NAME_FUNC_NAME} in {algorithm_name} must return a non‑empty string.") | ||
| 172 | + return column_name | ||
| 173 | + | ||
| 174 | + def _validate_return_value(self, value: Any, algorithm_name: str) -> bool: | ||
| 175 | + """ | ||
| 176 | + 校验自定义比对算法返回值格式 | ||
| 177 | + 要求返回二元组: (result_value, message) | ||
| 178 | + result_value: int | float | str | ||
| 179 | + message: str | ||
| 180 | + """ | ||
| 181 | + if not isinstance(value, tuple) or len(value) != self.RESULT_COUNT: | ||
| 182 | + raise ValueError( | ||
| 183 | + f"Algorithm [{algorithm_name}] return value invalid. " | ||
| 184 | + f"Expected {self.RESULT_COUNT}‑element tuple, got type={type(value).__name__}." | ||
| 185 | + ) | ||
| 186 | + | ||
| 187 | + ret_val, ret_msg = value | ||
| 188 | + | ||
| 189 | + if not self._is_real_number(ret_val) and not isinstance(ret_val, str): | ||
| 190 | + raise ValueError( | ||
| 191 | + f"Algorithm [{algorithm_name}] first return value type invalid. " | ||
| 192 | + f"Expected int/float/str, got {type(ret_val).__name__}" | ||
| 193 | + ) | ||
| 194 | + | ||
| 195 | + if not isinstance(ret_msg, str): | ||
| 196 | + raise ValueError( | ||
| 197 | + f"Algorithm [{algorithm_name}] second return value type invalid. " | ||
| 198 | + f"Expected str(message), got {type(ret_msg).__name__}" | ||
| 199 | + ) | ||
| 200 | + | ||
| 201 | + return True | ||
| 202 | + | ||
| 203 | + def _is_real_number(self, value) -> bool: | ||
| 204 | + return isinstance(value, (int, float)) and not isinstance(value, bool) | ||
| 205 | + | ||
| 206 | + def _call_algorithm(self, algorithm_name, n_value, b_value): | ||
| 207 | + if algorithm_name not in self.build_in_support_algorithm: | ||
| 208 | + if isinstance(n_value, np.ndarray): | ||
| 209 | + n_value = torch.from_numpy(n_value) | ||
| 210 | + if isinstance(b_value, np.ndarray): | ||
| 211 | + b_value = torch.from_numpy(b_value) | ||
| 212 | + with torch.no_grad(): | ||
| 213 | + return self._call_compare(algorithm_name, n_value, b_value) | ||
| 214 | + return self._call_compare(algorithm_name, n_value, b_value) | ||
| 215 | + | ||
| 216 | + def _call_compare(self, algorithm_name, n_value, b_value): | ||
| 217 | + try: | ||
| 218 | + algorithm_module = self._get_module(algorithm_name) | ||
| 219 | + compare_func = getattr(algorithm_module, self.COMPARE_FUNC_NAME) | ||
| 220 | + result = compare_func(n_value, b_value) | ||
| 221 | + self._validate_return_value(result, algorithm_name) | ||
| 222 | + return result | ||
| 223 | + except Exception as e: | ||
| 224 | + logger.error(f"Call algorithm [{algorithm_name}] failed, error: {e}") | ||
| 225 | + return "unsupported", str(e) | ||
| 226 | + | ||
L 【review】_call_algorithm 捕获所有 Exception 并统一返回 "unsupported",会掩盖算法实现中的真实 bug
这样在保持容错的同时,让真正的实现 bug 能被及时发现 ![]() ![]() | |||
| 227 | + def compare(self, n_value, b_value, column_names=None): | ||
| 228 | + if not column_names: | ||
| 229 | + column_names = self.algorithm_column_names | ||
| 230 | + results = [] | ||
| 231 | + error_msgs = [] | ||
| 232 | + for column_name in column_names: | ||
| 233 | + if column_name in self._column_name_algorithm_mapping: | ||
| 234 | + result, error_msg = self._call_algorithm( | ||
| 235 | + self._column_name_algorithm_mapping[column_name], n_value, b_value | ||
| 236 | + ) | ||
| 237 | + else: | ||
| 238 | + result, error_msg = "unsupported", f"No available algorithm for {column_name}" | ||
| 239 | + results.append(result) | ||
| 240 | + error_msgs.append(error_msg if error_msg else "") | ||
| 241 | + return results, error_msgs | ||
| 242 | + | ||
| 243 | + def get_algorithm_names(self) -> List[str]: | ||
| 244 | + return self.algorithm_names | ||
| 245 | + | ||
| 246 | + def get_algorithm_column_names(self) -> List[str]: | ||
| 247 | + return self.algorithm_column_names | ||
The file is empty
| @@ -0,0 +1,50 @@ | |||
| 1 | +# ------------------------------------------------------------------------- | ||
| 2 | +# This file is part of the MindStudio project. | ||
| 3 | +# Copyright (c) 2025 Huawei Technologies Co.,Ltd. | ||
| 4 | +# | ||
| 5 | +# MindStudio is licensed under Mulan PSL v2. | ||
| 6 | +# You can use this software according to the terms and conditions of the Mulan PSL v2. | ||
| 7 | +# You may obtain a copy of Mulan PSL v2 at: | ||
| 8 | +# | ||
| 9 | +# http://license.coscl.org.cn/MulanPSL2 | ||
| 10 | +# | ||
| 11 | +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, | ||
| 12 | +# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, | ||
| 13 | +# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. | ||
| 14 | +# See the Mulan PSL v2 for more details. | ||
| 15 | +# ------------------------------------------------------------------------- | ||
| 16 | + | ||
| 17 | + | ||
| 18 | +import numpy as np | ||
| 19 | +from msprobe.core.common.const import Const, CompareConst | ||
| 20 | +from msprobe.core.common.utils import format_value | ||
| 21 | +from msprobe.core.compare.utils import flatten_compare_numpy | ||
| 22 | + | ||
| 23 | + | ||
| 24 | +def column_name() -> str: | ||
| 25 | + return CompareConst.COSINE | ||
| 26 | + | ||
| 27 | + | ||
| 28 | +def compare(n_value: np.ndarray, b_value: np.ndarray): | ||
| 29 | + n_value, b_value, result, err = flatten_compare_numpy(n_value, b_value) | ||
| 30 | + if err: | ||
| 31 | + return result, err | ||
| 32 | + | ||
| 33 | + num = np.dot(n_value, b_value) | ||
| 34 | + a_norm = np.linalg.norm(n_value) | ||
| 35 | + b_norm = np.linalg.norm(b_value) | ||
| 36 | + | ||
| 37 | + if a_norm <= Const.FLOAT_EPSILON and b_norm <= Const.FLOAT_EPSILON: | ||
| 38 | + return 1.0, "" | ||
| 39 | + if a_norm <= Const.FLOAT_EPSILON: | ||
| 40 | + return CompareConst.NAN, "Cannot compare by Cosine Similarity, All the data is Zero in npu dump data." | ||
| 41 | + if b_norm <= Const.FLOAT_EPSILON: | ||
| 42 | + return CompareConst.NAN, "Cannot compare by Cosine Similarity, All the data is Zero in Bench dump data." | ||
| 43 | + | ||
| 44 | + cos = num / (a_norm * b_norm) | ||
| 45 | + if np.isnan(cos): | ||
| 46 | + return CompareConst.NAN, "Cannot compare by Cosine Similarity, the dump data has NaN." | ||
| 47 | + result = format_value(cos) | ||
| 48 | + if result > CompareConst.COSINE_THRESHOLD: | ||
| 49 | + result = round(result, 6) | ||
| 50 | + return result, "" | ||
| @@ -0,0 +1,33 @@ | |||
| 1 | +# ------------------------------------------------------------------------- | ||
| 2 | +# This file is part of the MindStudio project. | ||
| 3 | +# Copyright (c) 2025 Huawei Technologies Co.,Ltd. | ||
| 4 | +# | ||
| 5 | +# MindStudio is licensed under Mulan PSL v2. | ||
| 6 | +# You can use this software according to the terms and conditions of the Mulan PSL v2. | ||
| 7 | +# You may obtain a copy of Mulan PSL v2 at: | ||
| 8 | +# | ||
| 9 | +# http://license.coscl.org.cn/MulanPSL2 | ||
| 10 | +# | ||
| 11 | +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, | ||
| 12 | +# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, | ||
| 13 | +# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. | ||
| 14 | +# See the Mulan PSL v2 for more details. | ||
| 15 | +# ------------------------------------------------------------------------- | ||
| 16 | + | ||
| 17 | + | ||
| 18 | +import numpy as np | ||
| 19 | +from msprobe.core.common.const import CompareConst | ||
| 20 | +from msprobe.core.compare.utils import flatten_compare_numpy | ||
| 21 | + | ||
| 22 | + | ||
| 23 | +def column_name() -> str: | ||
| 24 | + return CompareConst.EUC_DIST | ||
| 25 | + | ||
| 26 | + | ||
| 27 | +def compare(n_value: np.ndarray, b_value: np.ndarray): | ||
| 28 | + n_value, b_value, result, err = flatten_compare_numpy(n_value, b_value) | ||
| 29 | + if err: | ||
| 30 | + return result, err | ||
| 31 | + | ||
| 32 | + distance = np.linalg.norm(n_value - b_value, ord=2) | ||
| 33 | + return distance, "" | ||
| @@ -0,0 +1,31 @@ | |||
| 1 | +# ------------------------------------------------------------------------- | ||
| 2 | +# This file is part of the MindStudio project. | ||
| 3 | +# Copyright (c) 2025 Huawei Technologies Co.,Ltd. | ||
| 4 | +# | ||
| 5 | +# MindStudio is licensed under Mulan PSL v2. | ||
| 6 | +# You can use this software according to the terms and conditions of the Mulan PSL v2. | ||
| 7 | +# You may obtain a copy of Mulan PSL v2 at: | ||
| 8 | +# | ||
| 9 | +# http://license.coscl.org.cn/MulanPSL2 | ||
| 10 | +# | ||
| 11 | +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, | ||
| 12 | +# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, | ||
| 13 | +# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. | ||
| 14 | +# See the Mulan PSL v2 for more details. | ||
| 15 | +# ------------------------------------------------------------------------- | ||
| 16 | + | ||
| 17 | + | ||
| 18 | +import numpy as np | ||
| 19 | +from msprobe.core.common.const import CompareConst | ||
| 20 | +from msprobe.core.compare.utils import validate_compare_numpy, get_relative_err, calc_err_ratio | ||
| 21 | + | ||
| 22 | + | ||
| 23 | +def column_name() -> str: | ||
| 24 | + return CompareConst.FIVE_THOUSANDTHS_ERR_RATIO | ||
| 25 | + | ||
| 26 | + | ||
| 27 | +def compare(n_value: np.ndarray, b_value: np.ndarray): | ||
| 28 | + result, err = validate_compare_numpy(n_value, b_value) | ||
| 29 | + if err: | ||
| 30 | + return result, err | ||
| 31 | + return calc_err_ratio(get_relative_err(n_value, b_value), CompareConst.FIVE_THOUSAND_RATIO_THRESHOLD), "" | ||
| @@ -0,0 +1,34 @@ | |||
| 1 | +# ------------------------------------------------------------------------- | ||
| 2 | +# This file is part of the MindStudio project. | ||
| 3 | +# Copyright (c) 2025 Huawei Technologies Co.,Ltd. | ||
| 4 | +# | ||
| 5 | +# MindStudio is licensed under Mulan PSL v2. | ||
| 6 | +# You can use this software according to the terms and conditions of the Mulan PSL v2. | ||
| 7 | +# You may obtain a copy of Mulan PSL v2 at: | ||
| 8 | +# | ||
| 9 | +# http://license.coscl.org.cn/MulanPSL2 | ||
| 10 | +# | ||
| 11 | +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, | ||
| 12 | +# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, | ||
| 13 | +# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. | ||
| 14 | +# See the Mulan PSL v2 for more details. | ||
| 15 | +# ------------------------------------------------------------------------- | ||
| 16 | + | ||
| 17 | + | ||
| 18 | +import numpy as np | ||
| 19 | +from msprobe.core.common.const import CompareConst | ||
| 20 | +from msprobe.core.common.utils import format_value | ||
| 21 | + | ||
| 22 | + | ||
| 23 | +def column_name() -> str: | ||
| 24 | + return CompareConst.MAX_ABS_ERR | ||
| 25 | + | ||
| 26 | + | ||
| 27 | +def compare(n_value: np.ndarray, b_value: np.ndarray): | ||
| 28 | + n_value = n_value.reshape(-1).astype(float) | ||
| 29 | + b_value = b_value.reshape(-1).astype(float) | ||
| 30 | + | ||
| 31 | + max_value = np.max(np.abs(n_value - b_value)) | ||
| 32 | + if np.isnan(max_value): | ||
| 33 | + return CompareConst.NAN, "Cannot compare by MaxAbsError, the data contains nan/inf/-inf." | ||
| 34 | + return format_value(max_value), "" | ||
| @@ -0,0 +1,35 @@ | |||
| 1 | +# ------------------------------------------------------------------------- | ||
| 2 | +# This file is part of the MindStudio project. | ||
| 3 | +# Copyright (c) 2025 Huawei Technologies Co.,Ltd. | ||
| 4 | +# | ||
| 5 | +# MindStudio is licensed under Mulan PSL v2. | ||
| 6 | +# You can use this software according to the terms and conditions of the Mulan PSL v2. | ||
| 7 | +# You may obtain a copy of Mulan PSL v2 at: | ||
| 8 | +# | ||
| 9 | +# http://license.coscl.org.cn/MulanPSL2 | ||
| 10 | +# | ||
| 11 | +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, | ||
| 12 | +# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, | ||
| 13 | +# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. | ||
| 14 | +# See the Mulan PSL v2 for more details. | ||
| 15 | +# ------------------------------------------------------------------------- | ||
| 16 | + | ||
| 17 | + | ||
| 18 | +import numpy as np | ||
| 19 | +from msprobe.core.common.const import CompareConst | ||
| 20 | +from msprobe.core.compare.utils import get_relative_err | ||
| 21 | +from msprobe.core.common.utils import format_value | ||
| 22 | + | ||
| 23 | + | ||
| 24 | +def column_name() -> str: | ||
| 25 | + return CompareConst.MAX_RELATIVE_ERR | ||
| 26 | + | ||
| 27 | + | ||
| 28 | +def compare(n_value: np.ndarray, b_value: np.ndarray): | ||
| 29 | + max_relative_err = np.max(np.abs(get_relative_err(n_value, b_value))) | ||
| 30 | + if np.isnan(max_relative_err): | ||
| 31 | + return ( | ||
| 32 | + CompareConst.NAN, | ||
| 33 | + "Cannot compare by MaxRelativeError, the data contains nan/inf/-inf.", | ||
| 34 | + ) | ||
| 35 | + return format_value(max_relative_err), "" | ||
| @@ -0,0 +1,31 @@ | |||
| 1 | +# ------------------------------------------------------------------------- | ||
| 2 | +# This file is part of the MindStudio project. | ||
| 3 | +# Copyright (c) 2025 Huawei Technologies Co.,Ltd. | ||
| 4 | +# | ||
| 5 | +# MindStudio is licensed under Mulan PSL v2. | ||
| 6 | +# You can use this software according to the terms and conditions of the Mulan PSL v2. | ||
| 7 | +# You may obtain a copy of Mulan PSL v2 at: | ||
| 8 | +# | ||
| 9 | +# http://license.coscl.org.cn/MulanPSL2 | ||
| 10 | +# | ||
| 11 | +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, | ||
| 12 | +# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, | ||
| 13 | +# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. | ||
| 14 | +# See the Mulan PSL v2 for more details. | ||
| 15 | +# ------------------------------------------------------------------------- | ||
| 16 | + | ||
| 17 | + | ||
| 18 | +import numpy as np | ||
| 19 | +from msprobe.core.common.const import CompareConst | ||
| 20 | +from msprobe.core.compare.utils import validate_compare_numpy, get_relative_err, calc_err_ratio | ||
| 21 | + | ||
| 22 | + | ||
| 23 | +def column_name() -> str: | ||
| 24 | + return CompareConst.ONE_THOUSANDTH_ERR_RATIO | ||
| 25 | + | ||
| 26 | + | ||
| 27 | +def compare(n_value: np.ndarray, b_value: np.ndarray): | ||
| 28 | + result, err = validate_compare_numpy(n_value, b_value) | ||
| 29 | + if err: | ||
| 30 | + return result, err | ||
| 31 | + return calc_err_ratio(get_relative_err(n_value, b_value), CompareConst.THOUSAND_RATIO_THRESHOLD), "" | ||
The file is empty
| @@ -17,7 +17,7 @@ | |||
| 17 | import multiprocessing | 17 | import multiprocessing |
| 18 | from dataclasses import dataclass | 18 | from dataclasses import dataclass |
| 19 | from functools import partial | 19 | from functools import partial |
| 20 | -from typing import Any | 20 | +from typing import Any, Dict, List |
| 21 | 21 | ||
| 22 | import pandas as pd | 22 | import pandas as pd |
| 23 | from tqdm import tqdm | 23 | from tqdm import tqdm |
| @@ -34,13 +34,13 @@ from msprobe.core.compare.tensor_postprocess.processor import TensorPostprocessM | |||
| 34 | 34 | ||
| 35 | @dataclass | 35 | @dataclass |
| 36 | class ComparisonResult: | 36 | class ComparisonResult: |
| 37 | - cos_result: list | 37 | + results: Dict[str, List[str]] # 列名 -> 结果列表 |
| 38 | - euc_dist_result: list | 38 | + err_msgs: List[str] # 错误信息列表 |
| 39 | - max_err_result: list | 39 | + |
| 40 | - max_relative_err_result: list | 40 | + @classmethod |
| 41 | - one_thousand_err_ratio_result: list | 41 | + def create_empty(cls, column_names: List[str], size: int) -> "ComparisonResult": |
| 42 | - five_thousand_err_ratio_result: list | 42 | + results = {name: [""] * size for name in column_names} |
| 43 | - err_msgs: list | 43 | + return cls(results=results, err_msgs=[""] * size) |
| 44 | 44 | ||
| 45 | 45 | ||
| 46 | @dataclass | 46 | @dataclass |
| @@ -151,19 +151,19 @@ class CompareRealData: | |||
| 151 | 151 | ||
| 152 | lock.acquire() | 152 | lock.acquire() |
| 153 | try: | 153 | try: |
| 154 | - for i, cos_item in enumerate(result.cos_result): | 154 | + slice_len = len(next(iter(result.results.values()), [])) |
| 155 | - process_index = i + offset | 155 | + slice_loc = slice(offset, offset + slice_len) |
| 156 | - result_df.loc[process_index, CompareConst.COSINE] = cos_item | 156 | + |
| 157 | - result_df.loc[process_index, CompareConst.EUC_DIST] = result.euc_dist_result[i] | 157 | + # 批量赋值算法列 |
| 158 | - result_df.loc[process_index, CompareConst.MAX_ABS_ERR] = result.max_err_result[i] | 158 | + for col_name, algo_results in result.results.items(): |
| 159 | - result_df.loc[process_index, CompareConst.MAX_RELATIVE_ERR] = result.max_relative_err_result[i] | 159 | + result_df.loc[slice_loc, col_name] = algo_results |
| 160 | - result_df.loc[process_index, CompareConst.ONE_THOUSANDTH_ERR_RATIO] = ( | 160 | + |
| 161 | - result.one_thousand_err_ratio_result | 161 | + # 错误消息处理 |
| 162 | - )[i] | 162 | + err_len = len(result.err_msgs) |
| 163 | - result_df.loc[process_index, CompareConst.FIVE_THOUSANDTHS_ERR_RATIO] = ( | 163 | + err_slice = slice(offset, offset + err_len) |
| 164 | - result.five_thousand_err_ratio_result | 164 | + |
| 165 | - )[i] | 165 | + target_index = result_df.loc[err_slice].index |
| 166 | - result_df.loc[process_index, CompareConst.ERROR_MESSAGE] += result.err_msgs[i] | 166 | + result_df.loc[err_slice, CompareConst.ERROR_MESSAGE] += pd.Series(result.err_msgs, index=target_index) |
| 167 | return result_df | 167 | return result_df |
| 168 | except ValueError as e: | 168 | except ValueError as e: |
| 169 | logger.error('result dataframe is not found.') | 169 | logger.error('result dataframe is not found.') |
| @@ -288,7 +288,13 @@ class CompareRealData: | |||
| 288 | 构建最终的对比结果 | 288 | 构建最终的对比结果 |
| 289 | 负责调用指标计算函数,并补充模糊匹配提示信息。 | 289 | 负责调用指标计算函数,并补充模糊匹配提示信息。 |
| 290 | """ | 290 | """ |
| 291 | - result_list, err_msg = compare_ops_apply(result.n_value, result.b_value, result.error_flag, result.err_msg) | 291 | + result_list, err_msg = compare_ops_apply( |
| 292 | + result.n_value, | ||
| 293 | + result.b_value, | ||
| 294 | + result.error_flag, | ||
| 295 | + result.err_msg, | ||
| 296 | + column_names=CompareConst.ALL_COMPARE_INDEX, | ||
| 297 | + ) | ||
| 292 | npu_op_name = npu_info.op_name | 298 | npu_op_name = npu_info.op_name |
| 293 | bench_op_name = bench_info.op_name | 299 | bench_op_name = bench_info.op_name |
| 294 | if self.mode_config.fuzzy_match and npu_op_name != bench_op_name and bench_op_name != CompareConst.N_A: | 300 | if self.mode_config.fuzzy_match and npu_op_name != bench_op_name and bench_op_name != CompareConst.N_A: |
| @@ -297,13 +303,9 @@ class CompareRealData: | |||
| 297 | return result_list | 303 | return result_list |
| 298 | 304 | ||
| 299 | def compare_ops(self, idx, dump_path_dict, result_df, lock, input_param): | 305 | def compare_ops(self, idx, dump_path_dict, result_df, lock, input_param): |
| 300 | - cos_result = [] | 306 | + columns_names = CompareConst.ALL_COMPARE_INDEX |
| 301 | - euc_dist_result = [] | 307 | + size = len(result_df) |
| 302 | - max_err_result = [] | 308 | + comparison_result = ComparisonResult.create_empty(columns_names, size) |
| 303 | - max_relative_err_result = [] | ||
| 304 | - one_thousand_err_ratio_result = [] | ||
| 305 | - five_thousand_err_ratio_result = [] | ||
| 306 | - err_mess = [] | ||
| 307 | 309 | ||
| 308 | is_print_compare_log = input_param.get("is_print_compare_log") | 310 | is_print_compare_log = input_param.get("is_print_compare_log") |
| 309 | 311 | ||
| @@ -326,47 +328,25 @@ class CompareRealData: | |||
| 326 | if is_print_compare_log: | 328 | if is_print_compare_log: |
| 327 | logger.info("start compare: {}".format(npu_op_name)) | 329 | logger.info("start compare: {}".format(npu_op_name)) |
| 328 | 330 | ||
| 329 | - ( | 331 | + compare_result = self.compare_by_op(npu_info, bench_info, input_param) |
| 330 | - cos_sim, | 332 | + err_msg = compare_result[-1] |
| 331 | - euc_dist, | ||
| 332 | - max_abs_err, | ||
| 333 | - max_relative_err, | ||
| 334 | - one_thousand_err_ratio, | ||
| 335 | - five_thousand_err_ratio, | ||
| 336 | - err_msg, | ||
| 337 | - ) = self.compare_by_op(npu_info, bench_info, input_param) | ||
| 338 | - | ||
| 339 | if is_print_compare_log: | 333 | if is_print_compare_log: |
| 340 | if "does not have data file" in err_msg: | 334 | if "does not have data file" in err_msg: |
| 341 | logger.info(f"[{npu_op_name}] Compare result: {err_msg} ") | 335 | logger.info(f"[{npu_op_name}] Compare result: {err_msg} ") |
| 342 | elif "Bench api/module unmatched" in err_msg: | 336 | elif "Bench api/module unmatched" in err_msg: |
| 343 | logger.info(f"[{npu_op_name}] Compare result: {err_msg} ") | 337 | logger.info(f"[{npu_op_name}] Compare result: {err_msg} ") |
| 344 | else: | 338 | else: |
| 345 | - logger.info( | 339 | + log_parts = [] |
| 346 | - f"[{npu_op_name}] Compare result: cosine {cos_sim}, euc_dist {euc_dist}, " | 340 | + for column_idx, column_name in enumerate(columns_names): |
| 347 | - f"max_abs_err {max_abs_err}, max_relative_err {max_relative_err}, " | 341 | + log_parts.append(f"{column_name} {compare_result[column_idx]}") |
| 348 | - f"one_thousand_err_ratio {one_thousand_err_ratio}, " | 342 | + log_parts.append(err_msg) |
| 349 | - f"five_thousand_err_ratio {five_thousand_err_ratio}, {err_msg}" | 343 | + logger.info(f"[{npu_op_name}] Compare result: {', '.join(log_parts)}") |
| 350 | - ) | ||
| 351 | - cos_result.append(cos_sim) | ||
| 352 | - euc_dist_result.append(euc_dist) | ||
| 353 | - max_err_result.append(max_abs_err) | ||
| 354 | - max_relative_err_result.append(max_relative_err) | ||
| 355 | - one_thousand_err_ratio_result.append(one_thousand_err_ratio) | ||
| 356 | - five_thousand_err_ratio_result.append(five_thousand_err_ratio) | ||
| 357 | - err_mess.append(err_msg) | ||
| 358 | 344 | ||
| 359 | - cr = ComparisonResult( | 345 | + for column_idx, column_name in enumerate(columns_names): |
| 360 | - cos_result=cos_result, | 346 | + comparison_result.results[column_name][i] = compare_result[column_idx] |
| 361 | - euc_dist_result=euc_dist_result, | 347 | + comparison_result.err_msgs[i] = err_msg |
| 362 | - max_err_result=max_err_result, | ||
| 363 | - max_relative_err_result=max_relative_err_result, | ||
| 364 | - one_thousand_err_ratio_result=one_thousand_err_ratio_result, | ||
| 365 | - five_thousand_err_ratio_result=five_thousand_err_ratio_result, | ||
| 366 | - err_msgs=err_mess, | ||
| 367 | - ) | ||
| 368 | 348 | ||
| 369 | - return self._save_cmp_result(idx, cr, result_df, lock) | 349 | + return self._save_cmp_result(idx, comparison_result, result_df, lock) |
| 370 | 350 | ||
| 371 | def do_multi_process(self, input_param, result_df): | 351 | def do_multi_process(self, input_param, result_df): |
| 372 | try: | 352 | try: |
| @@ -15,15 +15,15 @@ | |||
| 15 | # ------------------------------------------------------------------------- | 15 | # ------------------------------------------------------------------------- |
| 16 | 16 | ||
| 17 | 17 | ||
| 18 | -import abc | ||
| 19 | from dataclasses import dataclass | 18 | from dataclasses import dataclass |
| 20 | from typing import Any | 19 | from typing import Any |
| 21 | 20 | ||
| 22 | import numpy as np | 21 | import numpy as np |
| 23 | 22 | ||
| 24 | -from msprobe.core.common.const import Const, CompareConst | 23 | +from msprobe.core.common.const import CompareConst |
| 25 | from msprobe.core.common.log import logger | 24 | from msprobe.core.common.log import logger |
| 26 | -from msprobe.core.common.utils import CompareException, format_value | 25 | +from msprobe.core.common.utils import CompareException |
| 26 | +from msprobe.core.compare.algorithm.algorithm_scheduler import AlgorithmScheduler | ||
| 27 | 27 | ||
| 28 | 28 | ||
| 29 | @dataclass | 29 | @dataclass |
| @@ -71,19 +71,6 @@ def handle_inf_nan(n_value, b_value): | |||
| 71 | return n_value, b_value | 71 | return n_value, b_value |
| 72 | 72 | ||
| 73 | 73 | ||
| 74 | -def reshape_value(n_value, b_value): | ||
| 75 | - """返回reshape后的数据""" | ||
| 76 | - if not n_value.shape: # 判断数据是否为0维tensor, 如果0维tensor,不会转成1维tensor,直接返回 | ||
| 77 | - if n_value.dtype == bool: | ||
| 78 | - n_value = n_value.astype(float) | ||
| 79 | - b_value = b_value.astype(float) | ||
| 80 | - return n_value, b_value | ||
| 81 | - | ||
| 82 | - n_value = n_value.reshape(-1).astype(float) # 32转64为了防止某些数转dataframe时出现误差 | ||
| 83 | - b_value = b_value.reshape(-1).astype(float) | ||
| 84 | - return n_value, b_value | ||
| 85 | - | ||
| 86 | - | ||
| 87 | def npy_data_check(n_value, b_value): | 74 | def npy_data_check(n_value, b_value): |
| 88 | error_message = "" | 75 | error_message = "" |
| 89 | if not isinstance(n_value, np.ndarray) or not isinstance(b_value, np.ndarray): | 76 | if not isinstance(n_value, np.ndarray) or not isinstance(b_value, np.ndarray): |
| @@ -138,137 +125,6 @@ def statistics_data_check(result_dict): | |||
| 138 | return error_flag, error_message | 125 | return error_flag, error_message |
| 139 | 126 | ||
| 140 | 127 | ||
| 141 | -class TensorComparisonBasic(abc.ABC): | ||
| 142 | - """NPU和bench中npy数据的比较模板""" | ||
| 143 | - | ||
| 144 | - @abc.abstractmethod | ||
| 145 | - def apply(self, n_value, b_value, relative_err, err_msg): | ||
| 146 | - raise NotImplementedError | ||
| 147 | - | ||
| 148 | - | ||
| 149 | -def get_relative_err(n_value, b_value): | ||
| 150 | - """计算相对误差""" | ||
| 151 | - with np.errstate(divide='ignore', invalid='ignore'): | ||
| 152 | - if n_value.dtype not in CompareConst.FLOAT_TYPE: | ||
| 153 | - n_value = n_value.astype(float) | ||
| 154 | - if b_value.dtype not in CompareConst.FLOAT_TYPE: | ||
| 155 | - b_value = b_value.astype(float) | ||
| 156 | - | ||
| 157 | - n_value_copy = n_value.copy() | ||
| 158 | - b_value_copy = b_value.copy() | ||
| 159 | - zero_mask = b_value_copy == 0 | ||
| 160 | - | ||
| 161 | - result_type = np.result_type(n_value_copy, b_value_copy, np.float32) | ||
| 162 | - n = n_value_copy.astype(result_type) | ||
| 163 | - b = b_value_copy.astype(result_type) | ||
| 164 | - epsilon = np.finfo(result_type).eps | ||
| 165 | - n[zero_mask] += epsilon | ||
| 166 | - b[zero_mask] += epsilon | ||
| 167 | - relative_err = np.abs(np.divide((n - b), b)) | ||
| 168 | - | ||
| 169 | - return relative_err | ||
| 170 | - | ||
| 171 | - | ||
| 172 | -class GetCosineSimilarity(TensorComparisonBasic): | ||
| 173 | - """计算cosine相似度""" | ||
| 174 | - | ||
| 175 | - @staticmethod | ||
| 176 | - def correct_data(result): | ||
| 177 | - if result == CompareConst.NAN: | ||
| 178 | - return result | ||
| 179 | - if float(result) > CompareConst.COSINE_THRESHOLD: | ||
| 180 | - return round(float(result), 6) | ||
| 181 | - return result | ||
| 182 | - | ||
| 183 | - def apply(self, n_value, b_value, relative_err, err_msg): | ||
| 184 | - if "This is type of 0-d tensor" in err_msg: | ||
| 185 | - return CompareConst.UNSUPPORTED, err_msg | ||
| 186 | - | ||
| 187 | - with np.errstate(divide="ignore", invalid="ignore"): | ||
| 188 | - if len(n_value) == 1: | ||
| 189 | - return CompareConst.UNSUPPORTED, "This is a 1-d tensor of length 1." | ||
| 190 | - num = n_value.dot(b_value) | ||
| 191 | - a_norm = np.linalg.norm(n_value) | ||
| 192 | - b_norm = np.linalg.norm(b_value) | ||
| 193 | - | ||
| 194 | - if a_norm <= Const.FLOAT_EPSILON and b_norm <= Const.FLOAT_EPSILON: | ||
| 195 | - return 1.0, "" | ||
| 196 | - if a_norm <= Const.FLOAT_EPSILON: | ||
| 197 | - return CompareConst.NAN, "Cannot compare by Cosine Similarity, All the data is Zero in npu dump data." | ||
| 198 | - if b_norm <= Const.FLOAT_EPSILON: | ||
| 199 | - return CompareConst.NAN, "Cannot compare by Cosine Similarity, All the data is Zero in Bench dump data." | ||
| 200 | - | ||
| 201 | - cos = num / (a_norm * b_norm) | ||
| 202 | - if np.isnan(cos): | ||
| 203 | - return CompareConst.NAN, "Cannot compare by Cosine Similarity, the dump data has NaN." | ||
| 204 | - result = format_value(cos) | ||
| 205 | - result = self.correct_data(result) | ||
| 206 | - return result, "" | ||
| 207 | - | ||
| 208 | - | ||
| 209 | -class GetEuclideanDistance(TensorComparisonBasic): | ||
| 210 | - """计算欧氏距离""" | ||
| 211 | - | ||
| 212 | - def apply(self, n_value, b_value, relative_err, err_msg): | ||
| 213 | - if "This is type of 0-d tensor" in err_msg: | ||
| 214 | - return CompareConst.UNSUPPORTED, err_msg | ||
| 215 | - | ||
| 216 | - distance = np.linalg.norm(n_value - b_value, ord=2) | ||
| 217 | - | ||
| 218 | - return distance, "" | ||
| 219 | - | ||
| 220 | - | ||
| 221 | -class GetMaxAbsErr(TensorComparisonBasic): | ||
| 222 | - """计算最大绝对误差""" | ||
| 223 | - | ||
| 224 | - def apply(self, n_value, b_value, relative_err, err_msg): | ||
| 225 | - temp_res = n_value - b_value | ||
| 226 | - max_value = np.max(np.abs(temp_res)) | ||
| 227 | - if np.isnan(max_value): | ||
| 228 | - msg = "Cannot compare by MaxAbsError, the data contains nan/inf/-inf in dump data." | ||
| 229 | - return CompareConst.NAN, msg | ||
| 230 | - return format_value(max_value), "" | ||
| 231 | - | ||
| 232 | - | ||
| 233 | -class GetMaxRelativeErr(TensorComparisonBasic): | ||
| 234 | - """计算最大相对误差""" | ||
| 235 | - | ||
| 236 | - def apply(self, n_value, b_value, relative_err, err_msg): | ||
| 237 | - max_relative_err = np.max(np.abs(relative_err)) | ||
| 238 | - if np.isnan(max_relative_err): | ||
| 239 | - msg = "Cannot compare by MaxRelativeError, the data contains nan/inf/-inf in dump data." | ||
| 240 | - return CompareConst.NAN, msg | ||
| 241 | - return format_value(max_relative_err), "" | ||
| 242 | - | ||
| 243 | - | ||
| 244 | -class GetErrRatio(TensorComparisonBasic): | ||
| 245 | - """计算相对误差小于指定阈值(千分之一、千分之五)的比例""" | ||
| 246 | - | ||
| 247 | - def __init__(self, threshold): | ||
| 248 | - self.threshold = threshold | ||
| 249 | - | ||
| 250 | - def apply(self, n_value, b_value, relative_err, err_msg): | ||
| 251 | - if "This is type of 0-d tensor" in err_msg: | ||
| 252 | - return CompareConst.UNSUPPORTED, err_msg | ||
| 253 | - | ||
| 254 | - if not np.size(relative_err): | ||
| 255 | - return CompareConst.NAN, "" | ||
| 256 | - | ||
| 257 | - ratio = np.sum(relative_err < self.threshold) / np.size(relative_err) | ||
| 258 | - return format_value(ratio), "" | ||
| 259 | - | ||
| 260 | - | ||
| 261 | -class CompareOps: | ||
| 262 | - compare_ops = { | ||
| 263 | - "cosine_similarity": GetCosineSimilarity(), | ||
| 264 | - "euclidean_distance": GetEuclideanDistance(), | ||
| 265 | - "max_abs_error": GetMaxAbsErr(), | ||
| 266 | - "max_relative_error": GetMaxRelativeErr(), | ||
| 267 | - "one_thousand_err_ratio": GetErrRatio(CompareConst.THOUSAND_RATIO_THRESHOLD), | ||
| 268 | - "five_thousand_err_ratio": GetErrRatio(CompareConst.FIVE_THOUSAND_RATIO_THRESHOLD), | ||
| 269 | - } | ||
| 270 | - | ||
| 271 | - | ||
| 272 | def error_value_process(n_value): | 128 | def error_value_process(n_value): |
| 273 | if n_value in [ | 129 | if n_value in [ |
| 274 | CompareConst.READ_NONE, | 130 | CompareConst.READ_NONE, |
| @@ -285,22 +141,16 @@ def error_value_process(n_value): | |||
| 285 | return CompareConst.N_A, "" | 141 | return CompareConst.N_A, "" |
| 286 | 142 | ||
| 287 | 143 | ||
| 288 | -def compare_ops_apply(n_value, b_value, error_flag, err_msg): | 144 | +def compare_ops_apply(n_value, b_value, error_flag, err_msg, column_names=None): |
| 289 | - result_list = [] | 145 | + if not column_names: |
| 146 | + column_names = CompareConst.BUILTIN_COMPARE_COLUMNS | ||
| 290 | if error_flag: | 147 | if error_flag: |
| 291 | result, msg = error_value_process(n_value) | 148 | result, msg = error_value_process(n_value) |
| 292 | - result_list = [result] * len(CompareOps.compare_ops) | 149 | + result_list = [result] * len(column_names) |
| 293 | err_msg += msg | 150 | err_msg += msg |
| 294 | return result_list, err_msg | 151 | return result_list, err_msg |
| 295 | - | 152 | + results, err_msgs = AlgorithmScheduler.get_instance().compare(n_value, b_value, column_names=column_names) |
| 296 | - relative_err = get_relative_err(n_value, b_value) | 153 | + return results, err_msg + "".join(err_msgs) |
| 297 | - n_value, b_value = reshape_value(n_value, b_value) | ||
| 298 | - | ||
| 299 | - for op in CompareOps.compare_ops.values(): | ||
| 300 | - result, msg = op.apply(n_value, b_value, relative_err, err_msg) | ||
| 301 | - result_list.append(result) | ||
| 302 | - err_msg += msg | ||
| 303 | - return result_list, err_msg | ||
| 304 | 154 | ||
| 305 | 155 | ||
| 306 | class ValidateTensor: | 156 | class ValidateTensor: |
| @@ -17,8 +17,6 @@ | |||
| 17 | import numpy as np | 17 | import numpy as np |
| 18 | 18 | ||
| 19 | from msprobe.core.common.log import logger | 19 | from msprobe.core.common.log import logger |
| 20 | -from msprobe.core.compare.npy_compare import CompareOps | ||
| 21 | - | ||
| 22 | 20 | ||
| 23 | 21 | ||
| 24 | def in_different_shape(a, b): | 22 | def in_different_shape(a, b): |
| @@ -51,9 +49,9 @@ def cos_sim(a, b): | |||
| 51 | b_norm = np.linalg.norm(b) | 49 | b_norm = np.linalg.norm(b) |
| 52 | 50 | ||
| 53 | if a_norm == 0 and b_norm == 0: | 51 | if a_norm == 0 and b_norm == 0: |
| 54 | - return 1. | 52 | + return 1.0 |
| 55 | if a_norm == 0 or b_norm == 0: | 53 | if a_norm == 0 or b_norm == 0: |
| 56 | - logger.warning(f'One tensor norm is zero.') | 54 | + logger.warning('One tensor norm is zero.') |
| 57 | return None | 55 | return None |
| 58 | 56 | ||
| 59 | sim = num / (a_norm * b_norm) | 57 | sim = num / (a_norm * b_norm) |
| @@ -76,9 +74,4 @@ def shape(a, b): | |||
| 76 | return list(a.shape) | 74 | return list(a.shape) |
| 77 | 75 | ||
| 78 | 76 | ||
| 79 | -METRIC_FUNC = { | 77 | +METRIC_FUNC = {'l2': l2_distance, 'cos': cos_sim, 'numel': numel, 'shape': shape} |
| 80 | - 'l2': l2_distance, | ||
| 81 | - 'cos': cos_sim, | ||
| 82 | - 'numel': numel, | ||
| 83 | - 'shape': shape | ||
| 84 | - } | ||
| @@ -88,7 +88,7 @@ def pt_compare(input_param, output_path, **kwargs): | |||
| 88 | 'first_diff_analyze': config.first_diff_analyze, | 88 | 'first_diff_analyze': config.first_diff_analyze, |
| 89 | 'compared_file_type': config.compared_file_type, | 89 | 'compared_file_type': config.compared_file_type, |
| 90 | 'consistent_check': config.consistent_check, | 90 | 'consistent_check': config.consistent_check, |
| 91 | - 'backend': config.backend | 91 | + 'backend': config.backend, |
| 92 | } | 92 | } |
| 93 | mode_config = ModeConfig(**config_dict) | 93 | mode_config = ModeConfig(**config_dict) |
| 94 | mapping_config = MappingConfig(cell_mapping=config.cell_mapping, data_mapping=config.data_mapping) | 94 | mapping_config = MappingConfig(cell_mapping=config.cell_mapping, data_mapping=config.data_mapping) |
| @@ -21,8 +21,7 @@ import torch | |||
| 21 | 21 | ||
| 22 | from msprobe.core.common.utils import logger, CompareException | 22 | from msprobe.core.common.utils import logger, CompareException |
| 23 | from msprobe.core.common.file_utils import FileChecker, FileCheckConst | 23 | from msprobe.core.common.file_utils import FileChecker, FileCheckConst |
| 24 | -from msprobe.pytorch.common.utils import load_pt | 24 | +from msprobe.pytorch.common.utils import load_pt, is_float8_tensor |
| 25 | -from msprobe.pytorch.common.utils import is_float8_tensor, load_pt | ||
| 26 | 25 | ||
| 27 | 26 | ||
| 28 | def _prepare_tensor_for_numpy(tensor): | 27 | def _prepare_tensor_for_numpy(tensor): |
| @@ -31,6 +30,7 @@ def _prepare_tensor_for_numpy(tensor): | |||
| 31 | return tensor.to(torch.float32) | 30 | return tensor.to(torch.float32) |
| 32 | return tensor | 31 | return tensor |
| 33 | 32 | ||
| 33 | + | ||
| 34 | def read_pt_data(dir_path, file_name): | 34 | def read_pt_data(dir_path, file_name): |
| 35 | if not file_name: | 35 | if not file_name: |
| 36 | return None | 36 | return None |
| @@ -48,7 +48,7 @@ def read_pt_data(dir_path, file_name): | |||
| 48 | raise CompareException(CompareException.INVALID_FILE_ERROR) from e | 48 | raise CompareException(CompareException.INVALID_FILE_ERROR) from e |
| 49 | except AttributeError as e: | 49 | except AttributeError as e: |
| 50 | # 这里捕获 detach 方法抛出的异常 | 50 | # 这里捕获 detach 方法抛出的异常 |
| 51 | - logger.error(f"Failed to detach the loaded tensor.") | 51 | + logger.error("Failed to detach the loaded tensor.") |
| 52 | raise CompareException(CompareException.DETACH_ERROR) from e | 52 | raise CompareException(CompareException.DETACH_ERROR) from e |
| 53 | data_value = _prepare_tensor_for_numpy(data_value) | 53 | data_value = _prepare_tensor_for_numpy(data_value) |
| 54 | data_value = data_value.numpy() | 54 | data_value = data_value.numpy() |
| @@ -29,7 +29,7 @@ from msprobe.core.compare.utils import ( | |||
| 29 | ) | 29 | ) |
| 30 | from msprobe.core.common.utils import set_dump_path, get_dump_mode | 30 | from msprobe.core.common.utils import set_dump_path, get_dump_mode |
| 31 | from msprobe.visualization.utils import GraphConst | 31 | from msprobe.visualization.utils import GraphConst |
| 32 | -from msprobe.core.common.const import Const, CompareConst | 32 | +from msprobe.core.common.const import LazyConst, Const, CompareConst |
| 33 | from msprobe.core.compare.indicator_analysis.calculator import calculate_result | 33 | from msprobe.core.compare.indicator_analysis.calculator import calculate_result |
| 34 | from msprobe.core.common.file_utils import FileChecker, FileCheckConst | 34 | from msprobe.core.common.file_utils import FileChecker, FileCheckConst |
| 35 | from msprobe.core.common.log import BaseLogger, logger | 35 | from msprobe.core.common.log import BaseLogger, logger |
| @@ -240,7 +240,7 @@ class MatchedNodeCalculator: | |||
| 240 | 功能:用户在前端选择NPU和Bench节点匹配,前端会查询db数据打包发送到此类,此类进行精度指标的计算,最终返回结果给前端 | 240 | 功能:用户在前端选择NPU和Bench节点匹配,前端会查询db数据打包发送到此类,此类进行精度指标的计算,最终返回结果给前端 |
| 241 | """ | 241 | """ |
| 242 | 242 | ||
| 243 | - TENSOR_COMPARE_INDEX = CompareConst.ALL_COMPARE_INDEX + CompareConst.EXTRACT_INDEX | 243 | + TENSOR_COMPARE_INDEX = LazyConst(lambda: CompareConst.ALL_COMPARE_INDEX + CompareConst.EXTRACT_INDEX) |
| 244 | STATISTICS_COMPARE_INDEX = CompareConst.SUMMARY_COMPARE_INDEX + CompareConst.EXTRACT_INDEX | 244 | STATISTICS_COMPARE_INDEX = CompareConst.SUMMARY_COMPARE_INDEX + CompareConst.EXTRACT_INDEX |
| 245 | 245 | ||
| 246 | _RESP_SUCCESS = "success" | 246 | _RESP_SUCCESS = "success" |
| @@ -272,17 +272,10 @@ class MatchedNodeCalculator: | |||
| 272 | Const.REQ_GRAD: (CompareConst.NPU_REQ_GRAD, CompareConst.BENCH_REQ_GRAD), | 272 | Const.REQ_GRAD: (CompareConst.NPU_REQ_GRAD, CompareConst.BENCH_REQ_GRAD), |
| 273 | Const.MD5: (CompareConst.NPU_MD5, CompareConst.BENCH_MD5), | 273 | Const.MD5: (CompareConst.NPU_MD5, CompareConst.BENCH_MD5), |
| 274 | } | 274 | } |
| 275 | + algorithm_columns = CompareConst.ALL_COMPARE_INDEX | ||
| 276 | + compare_result_header = CompareConst.COMPARE_RESULT_HEADER | ||
| 275 | self.tensor_indicators_index = { | 277 | self.tensor_indicators_index = { |
| 276 | - CompareConst.COSINE: CompareConst.COMPARE_RESULT_HEADER.index(CompareConst.COSINE), | 278 | + col_name: compare_result_header.index(col_name) for col_name in algorithm_columns |
| 277 | - CompareConst.EUC_DIST: CompareConst.COMPARE_RESULT_HEADER.index(CompareConst.EUC_DIST), | ||
| 278 | - CompareConst.MAX_ABS_ERR: CompareConst.COMPARE_RESULT_HEADER.index(CompareConst.MAX_ABS_ERR), | ||
| 279 | - CompareConst.MAX_RELATIVE_ERR: CompareConst.COMPARE_RESULT_HEADER.index(CompareConst.MAX_RELATIVE_ERR), | ||
| 280 | - CompareConst.ONE_THOUSANDTH_ERR_RATIO: CompareConst.COMPARE_RESULT_HEADER.index( | ||
| 281 | - CompareConst.ONE_THOUSANDTH_ERR_RATIO | ||
| 282 | - ), | ||
| 283 | - CompareConst.FIVE_THOUSANDTHS_ERR_RATIO: CompareConst.COMPARE_RESULT_HEADER.index( | ||
| 284 | - CompareConst.FIVE_THOUSANDTHS_ERR_RATIO | ||
| 285 | - ), | ||
| 286 | } | 279 | } |
| 287 | self.extra_tensor_indicators_index = { | 280 | self.extra_tensor_indicators_index = { |
| 288 | CompareConst.REQ_GRAD_CONSIST: CompareConst.COMPARE_RESULT_HEADER.index(CompareConst.REQ_GRAD_CONSIST), | 281 | CompareConst.REQ_GRAD_CONSIST: CompareConst.COMPARE_RESULT_HEADER.index(CompareConst.REQ_GRAD_CONSIST), |
| @@ -17,7 +17,7 @@ | |||
| 17 | import math | 17 | import math |
| 18 | import json | 18 | import json |
| 19 | from msprobe.core.common.const import CompareConst, Const | 19 | from msprobe.core.common.const import CompareConst, Const |
| 20 | -from msprobe.visualization.utils import ToolTip, GraphConst, str2float | 20 | +from msprobe.visualization.utils import ToolTip, GraphConst |
| 21 | 21 | ||
| 22 | 22 | ||
| 23 | class ModeAdapter: | 23 | class ModeAdapter: |
| @@ -74,8 +74,7 @@ class ModeAdapter: | |||
| 74 | headers = CompareConst.COMPARE_RESULT_HEADER | 74 | headers = CompareConst.COMPARE_RESULT_HEADER |
| 75 | ModeAdapter._add_compare_data(node.input_data, compare_data_dict, key_list, headers) | 75 | ModeAdapter._add_compare_data(node.input_data, compare_data_dict, key_list, headers) |
| 76 | ModeAdapter._add_compare_data(node.output_data, compare_data_dict, key_list, headers) | 76 | ModeAdapter._add_compare_data(node.output_data, compare_data_dict, key_list, headers) |
| 77 | - precision_index = GraphConst.COMPARE_INDICATOR_TO_PRECISION_INDEX_MAPPING.get(api_indicator, | 77 | + precision_index = GraphConst.COMPARE_INDICATOR_TO_PRECISION_INDEX_MAPPING.get(api_indicator, CompareConst.PASS) |
| 78 | - CompareConst.PASS) | ||
| 79 | precision_index = self._ignore_precision_index(node.id, precision_index) | 78 | precision_index = self._ignore_precision_index(node.id, precision_index) |
| 80 | return precision_index | 79 | return precision_index |
| 81 | 80 | ||
| @@ -102,16 +101,14 @@ class ModeAdapter: | |||
| 102 | CompareConst.MAX_DIFF: ToolTip.MAX_DIFF, | 101 | CompareConst.MAX_DIFF: ToolTip.MAX_DIFF, |
| 103 | CompareConst.MIN_DIFF: ToolTip.MIN_DIFF, | 102 | CompareConst.MIN_DIFF: ToolTip.MIN_DIFF, |
| 104 | CompareConst.MEAN_DIFF: ToolTip.MEAN_DIFF, | 103 | CompareConst.MEAN_DIFF: ToolTip.MEAN_DIFF, |
| 105 | - CompareConst.NORM_DIFF: ToolTip.NORM_DIFF} | 104 | + CompareConst.NORM_DIFF: ToolTip.NORM_DIFF, |
| 105 | + } | ||
| 106 | elif self.compare_mode == GraphConst.MD5_COMPARE: | 106 | elif self.compare_mode == GraphConst.MD5_COMPARE: |
| 107 | tips = {Const.MD5: ToolTip.MD5} | 107 | tips = {Const.MD5: ToolTip.MD5} |
| 108 | else: | 108 | else: |
| 109 | - tips = { | 109 | + # tips 前端已经不展示,全部设置为"" |
| 110 | - CompareConst.ONE_THOUSANDTH_ERR_RATIO: ToolTip.ONE_THOUSANDTH_ERR_RATIO, | 110 | + algorithm_columns = CompareConst.ALL_COMPARE_INDEX |
| 111 | - CompareConst.FIVE_THOUSANDTHS_ERR_RATIO: ToolTip.FIVE_THOUSANDTHS_ERR_RATIO, | 111 | + tips = {col: "" for col in algorithm_columns} |
| 112 | - CompareConst.COSINE: ToolTip.COSINE, | ||
| 113 | - CompareConst.MAX_ABS_ERR: ToolTip.MAX_ABS_ERR, | ||
| 114 | - CompareConst.MAX_RELATIVE_ERR: ToolTip.MAX_RELATIVE_ERR} | ||
| 115 | return json.dumps(tips) | 112 | return json.dumps(tips) |
| 116 | 113 | ||
| 117 | def _ignore_precision_index(self, node_id, precision_index): | 114 | def _ignore_precision_index(self, node_id, precision_index): |
| @@ -17,10 +17,10 @@ | |||
| 17 | import os | 17 | import os |
| 18 | import re | 18 | import re |
| 19 | import json | 19 | import json |
| 20 | -import pickle | 20 | +import pickle # nosec B403 |
| 21 | import time | 21 | import time |
| 22 | from msprobe.core.common.file_utils import FileOpen, find_proc_dir | 22 | from msprobe.core.common.file_utils import FileOpen, find_proc_dir |
| 23 | -from msprobe.core.common.const import CompareConst, Const | 23 | +from msprobe.core.common.const import LazyConst, CompareConst, Const |
| 24 | from msprobe.core.common.log import logger | 24 | from msprobe.core.common.log import logger |
| 25 | from msprobe.core.common.exceptions import MsprobeException | 25 | from msprobe.core.common.exceptions import MsprobeException |
| 26 | from msprobe.core.compare.utils import check_and_return_dir_contents | 26 | from msprobe.core.compare.utils import check_and_return_dir_contents |
| @@ -117,9 +117,11 @@ def check_directory_content(input_path): | |||
| 117 | if step_all: | 117 | if step_all: |
| 118 | return GraphConst.STEPS | 118 | return GraphConst.STEPS |
| 119 | 119 | ||
| 120 | - raise ValueError("The input path content does not conform to the expected naming convention. " | 120 | + raise ValueError( |
| 121 | - "It is expected to be all step{number} named folders (such as step0), " | 121 | + "The input path content does not conform to the expected naming convention. " |
| 122 | - "all rank{number} named folders (such as rank0), or all files.") | 122 | + "It is expected to be all step{number} named folders (such as step0), " |
| 123 | + "all rank{number} named folders (such as rank0), or all files." | ||
| 124 | + ) | ||
| 123 | 125 | ||
| 124 | 126 | ||
| 125 | def extract_rank_number(rank_str): | 127 | def extract_rank_number(rank_str): |
| @@ -141,34 +143,52 @@ def load_parallel_param(args): | |||
| 141 | raise MsprobeException(MsprobeException.INVALID_PARAM_ERROR) | 143 | raise MsprobeException(MsprobeException.INVALID_PARAM_ERROR) |
| 142 | 144 | ||
| 143 | if not args.golden_path and len(args.rank_size) != 1: | 145 | if not args.golden_path and len(args.rank_size) != 1: |
| 144 | - logger.error('In the graph merge build task, ' | 146 | + logger.error('In the graph merge build task, the number of parameters "rank_size" to be filled in is either 1!') |
| 145 | - 'the number of parameters "rank_size" to be filled in is either 1!') | ||
| 146 | raise MsprobeException(MsprobeException.INVALID_PARAM_ERROR) | 147 | raise MsprobeException(MsprobeException.INVALID_PARAM_ERROR) |
| 147 | 148 | ||
| 148 | if args.golden_path and len(args.rank_size) != 2: | 149 | if args.golden_path and len(args.rank_size) != 2: |
| 149 | - logger.error('In the graph merge compare task, ' | 150 | + logger.error( |
| 150 | - 'the number of parameters "rank_size" to be filled in is either 2!') | 151 | + 'In the graph merge compare task, the number of parameters "rank_size" to be filled in is either 2!' |
| 152 | + ) | ||
| 151 | raise MsprobeException(MsprobeException.INVALID_PARAM_ERROR) | 153 | raise MsprobeException(MsprobeException.INVALID_PARAM_ERROR) |
| 152 | 154 | ||
| 153 | if len(args.rank_size) == 1: | 155 | if len(args.rank_size) == 1: |
| 154 | if any(len(x) != 1 for x in param_t): | 156 | if any(len(x) != 1 for x in param_t): |
| 155 | - logger.error('In the graph merge build task, ' | 157 | + logger.error( |
| 156 | - 'the number of parameters "tp/pp/rank_size" to be filled in is either 1!') | 158 | + 'In the graph merge build task, the number of parameters "tp/pp/rank_size" to be filled in is either 1!' |
| 159 | + ) | ||
| 157 | raise MsprobeException(MsprobeException.INVALID_PARAM_ERROR) | 160 | raise MsprobeException(MsprobeException.INVALID_PARAM_ERROR) |
| 158 | 161 | ||
| 159 | - param_n = ParallelParam(args.rank_size[0], args.tp[0], args.pp[0], args.vpp[0] if args.vpp else 1, | 162 | + param_n = ParallelParam( |
| 160 | - args.order[0] if args.order else 'tp-cp-ep-dp-pp') | 163 | + args.rank_size[0], |
| 164 | + args.tp[0], | ||
| 165 | + args.pp[0], | ||
| 166 | + args.vpp[0] if args.vpp else 1, | ||
| 167 | + args.order[0] if args.order else 'tp-cp-ep-dp-pp', | ||
| 168 | + ) | ||
| 161 | return (param_n,) | 169 | return (param_n,) |
| 162 | else: | 170 | else: |
| 163 | if any(len(x) != 2 for x in param_t): | 171 | if any(len(x) != 2 for x in param_t): |
| 164 | - logger.error('In the graph merge compare task, ' | 172 | + logger.error( |
| 165 | - 'the number of parameters "tp/pp/rank_size" to be filled in is either 2!') | 173 | + 'In the graph merge compare task, ' |
| 174 | + 'the number of parameters "tp/pp/rank_size" to be filled in is either 2!' | ||
| 175 | + ) | ||
| 166 | raise MsprobeException(MsprobeException.INVALID_PARAM_ERROR) | 176 | raise MsprobeException(MsprobeException.INVALID_PARAM_ERROR) |
| 167 | 177 | ||
| 168 | - param_n = ParallelParam(args.rank_size[0], args.tp[0], args.pp[0], args.vpp[0] if len(args.vpp) == 2 else 1, | 178 | + param_n = ParallelParam( |
| 169 | - args.order[0] if len(args.order) == 2 else 'tp-cp-ep-dp-pp') | 179 | + args.rank_size[0], |
| 170 | - param_b = ParallelParam(args.rank_size[1], args.tp[1], args.pp[1], args.vpp[1] if len(args.vpp) == 2 else 1, | 180 | + args.tp[0], |
| 171 | - args.order[1] if len(args.order) == 2 else 'tp-cp-ep-dp-pp') | 181 | + args.pp[0], |
| 182 | + args.vpp[0] if len(args.vpp) == 2 else 1, | ||
| 183 | + args.order[0] if len(args.order) == 2 else 'tp-cp-ep-dp-pp', | ||
| 184 | + ) | ||
| 185 | + param_b = ParallelParam( | ||
| 186 | + args.rank_size[1], | ||
| 187 | + args.tp[1], | ||
| 188 | + args.pp[1], | ||
| 189 | + args.vpp[1] if len(args.vpp) == 2 else 1, | ||
| 190 | + args.order[1] if len(args.order) == 2 else 'tp-cp-ep-dp-pp', | ||
| 191 | + ) | ||
| 172 | return param_n, param_b | 192 | return param_n, param_b |
| 173 | 193 | ||
| 174 | 194 | ||
| @@ -177,8 +197,10 @@ def validate_parallel_param(parallel_param, dump_path, log_prefix='[NPU]'): | |||
| 177 | params = [parallel_param.tp, parallel_param.pp, parallel_param.rank_size, parallel_param.vpp] | 197 | params = [parallel_param.tp, parallel_param.pp, parallel_param.rank_size, parallel_param.vpp] |
| 178 | ranks = check_and_return_dir_contents(dump_path, Const.RANK) | 198 | ranks = check_and_return_dir_contents(dump_path, Const.RANK) |
| 179 | if len(ranks) != parallel_param.rank_size: | 199 | if len(ranks) != parallel_param.rank_size: |
| 180 | - logger.error(f'{log_prefix} The parallel param "rank_size" error, ' | 200 | + logger.error( |
| 181 | - f'you set {parallel_param.rank_size} but expected {len(ranks)}.') | 201 | + f'{log_prefix} The parallel param "rank_size" error, ' |
| 202 | + f'you set {parallel_param.rank_size} but expected {len(ranks)}.' | ||
| 203 | + ) | ||
| 182 | raise MsprobeException(MsprobeException.INVALID_PARAM_ERROR) | 204 | raise MsprobeException(MsprobeException.INVALID_PARAM_ERROR) |
| 183 | if any(x is None for x in params): | 205 | if any(x is None for x in params): |
| 184 | logger.error(f'{log_prefix} The parallel params "tp/pp/rank_size" must not be null!') | 206 | logger.error(f'{log_prefix} The parallel params "tp/pp/rank_size" must not be null!') |
| @@ -233,8 +255,12 @@ class ToolTip: | |||
| 233 | MEAN_DIFF = 'NPU与标杆API统计信息比对,平均值的差值' | 255 | MEAN_DIFF = 'NPU与标杆API统计信息比对,平均值的差值' |
| 234 | NORM_DIFF = 'NPU与标杆API统计信息比对,2范数(平方根)的差值' | 256 | NORM_DIFF = 'NPU与标杆API统计信息比对,2范数(平方根)的差值' |
| 235 | MD5 = '数据MD5信息,用于比较两个数据信息是否完全一致' | 257 | MD5 = '数据MD5信息,用于比较两个数据信息是否完全一致' |
| 236 | - ONE_THOUSANDTH_ERR_RATIO = 'Tensor中的元素逐个与对应的标杆数据对比,相对误差小于千分之一的比例占总元素个数的比例,比例越接近1越好' | 258 | + ONE_THOUSANDTH_ERR_RATIO = ( |
| 237 | - FIVE_THOUSANDTHS_ERR_RATIO = 'Tensor中的元素逐个与对应的标杆数据对比,相对误差小于千分之五的比例占总元素个数的比例,比例越接近1越好' | 259 | + 'Tensor中的元素逐个与对应的标杆数据对比,相对误差小于千分之一的比例占总元素个数的比例,比例越接近1越好' |
| 260 | + ) | ||
| 261 | + FIVE_THOUSANDTHS_ERR_RATIO = ( | ||
| 262 | + 'Tensor中的元素逐个与对应的标杆数据对比,相对误差小于千分之五的比例占总元素个数的比例,比例越接近1越好' | ||
| 263 | + ) | ||
| 238 | COSINE = ( | 264 | COSINE = ( |
| 239 | '通过计算两个向量的余弦值来判断其相似度,数值越接近于1说明计算出的两个张量越相似,实际可接受阈值为大于0.99。' | 265 | '通过计算两个向量的余弦值来判断其相似度,数值越接近于1说明计算出的两个张量越相似,实际可接受阈值为大于0.99。' |
| 240 | '在计算中可能会存在nan,主要由于可能会出现其中一个向量为0' | 266 | '在计算中可能会存在nan,主要由于可能会出现其中一个向量为0' |
| @@ -272,12 +298,20 @@ class GraphConst: | |||
| 272 | INPUT = '.input.' | 298 | INPUT = '.input.' |
| 273 | OUTPUT = '.output.' | 299 | OUTPUT = '.output.' |
| 274 | STR_MAX_LEN = 50 | 300 | STR_MAX_LEN = 50 |
| 275 | - MD5_INDEX_LIST = CompareConst.MD5_COMPARE_INDEX + [CompareConst.REQ_GRAD_CONSIST, CompareConst.RESULT, | 301 | + MD5_INDEX_LIST = CompareConst.MD5_COMPARE_INDEX + [ |
| 276 | - CompareConst.ERROR_MESSAGE] | 302 | + CompareConst.REQ_GRAD_CONSIST, |
| 277 | - REAL_DATA_INDEX_LIST = CompareConst.ALL_COMPARE_INDEX + [CompareConst.REQ_GRAD_CONSIST, CompareConst.RESULT, | 303 | + CompareConst.RESULT, |
| 278 | - CompareConst.ERROR_MESSAGE] | 304 | + CompareConst.ERROR_MESSAGE, |
| 279 | - SUMMARY_INDEX_LIST = CompareConst.SUMMARY_COMPARE_INDEX + [CompareConst.REQ_GRAD_CONSIST, CompareConst.RESULT, | 305 | + ] |
| 280 | - CompareConst.ERROR_MESSAGE] | 306 | + REAL_DATA_INDEX_LIST = LazyConst( |
| 307 | + lambda: CompareConst.ALL_COMPARE_INDEX | ||
| 308 | + + [CompareConst.REQ_GRAD_CONSIST, CompareConst.RESULT, CompareConst.ERROR_MESSAGE] | ||
| 309 | + ) | ||
| 310 | + SUMMARY_INDEX_LIST = CompareConst.SUMMARY_COMPARE_INDEX + [ | ||
| 311 | + CompareConst.REQ_GRAD_CONSIST, | ||
| 312 | + CompareConst.RESULT, | ||
| 313 | + CompareConst.ERROR_MESSAGE, | ||
| 314 | + ] | ||
| 281 | APIS_BETWEEN_MODULES = 'Apis_Between_Modules' | 315 | APIS_BETWEEN_MODULES = 'Apis_Between_Modules' |
| 282 | APIS_BETWEEN_MODULES_ALL_RANKS = 'Apis_Between_Modules_All_Ranks' | 316 | APIS_BETWEEN_MODULES_ALL_RANKS = 'Apis_Between_Modules_All_Ranks' |
| 283 | NULL = 'null' | 317 | NULL = 'null' |
| @@ -294,20 +328,20 @@ class GraphConst: | |||
| 294 | Const.ALL: REAL_DATA_COMPARE, | 328 | Const.ALL: REAL_DATA_COMPARE, |
| 295 | Const.SUMMARY: SUMMARY_COMPARE, | 329 | Const.SUMMARY: SUMMARY_COMPARE, |
| 296 | Const.MD5: MD5_COMPARE, | 330 | Const.MD5: MD5_COMPARE, |
| 297 | - Const.STRUCTURE: STRUCTURE_COMPARE | 331 | + Const.STRUCTURE: STRUCTURE_COMPARE, |
| 298 | } | 332 | } |
| 299 | 333 | ||
| 300 | GRAPHCOMPARE_MODE_TO_DUMP_MODE_TO_MAPPING = { | 334 | GRAPHCOMPARE_MODE_TO_DUMP_MODE_TO_MAPPING = { |
| 301 | REAL_DATA_COMPARE: Const.ALL, | 335 | REAL_DATA_COMPARE: Const.ALL, |
| 302 | SUMMARY_COMPARE: Const.SUMMARY, | 336 | SUMMARY_COMPARE: Const.SUMMARY, |
| 303 | MD5_COMPARE: Const.MD5, | 337 | MD5_COMPARE: Const.MD5, |
| 304 | - STRUCTURE_COMPARE: Const.STRUCTURE | 338 | + STRUCTURE_COMPARE: Const.STRUCTURE, |
| 305 | } | 339 | } |
| 306 | 340 | ||
| 307 | COMPARE_INDICATOR_TO_PRECISION_INDEX_MAPPING = { | 341 | COMPARE_INDICATOR_TO_PRECISION_INDEX_MAPPING = { |
| 308 | CompareConst.PASS: 0, | 342 | CompareConst.PASS: 0, |
| 309 | CompareConst.WARNING: 0.5, | 343 | CompareConst.WARNING: 0.5, |
| 310 | - CompareConst.ERROR: 1 | 344 | + CompareConst.ERROR: 1, |
| 311 | } | 345 | } |
| 312 | 346 | ||
| 313 | RANKS = 'ranks' | 347 | RANKS = 'ranks' |
| @@ -325,8 +359,14 @@ class GraphConst: | |||
| 325 | UNCERTAINTY_THRESHOLD = 1e-6 | 359 | UNCERTAINTY_THRESHOLD = 1e-6 |
| 326 | REDUCE_OPERATIONS = ['reduce_scatter', 'all_reduce'] | 360 | REDUCE_OPERATIONS = ['reduce_scatter', 'all_reduce'] |
| 327 | 361 | ||
| 328 | - IGNORE_PRECISION_INDEX = {'empty', 'empty_like', 'empty_with_format', 'new_empty_strided', 'new_empty', | 362 | + IGNORE_PRECISION_INDEX = { |
| 329 | - 'empty_strided'} | 363 | + 'empty', |
| 364 | + 'empty_like', | ||
| 365 | + 'empty_with_format', | ||
| 366 | + 'new_empty_strided', | ||
| 367 | + 'new_empty', | ||
| 368 | + 'empty_strided', | ||
| 369 | + } | ||
| 330 | VPP_CHUNK_0 = '0' | 370 | VPP_CHUNK_0 = '0' |
| 331 | 371 | ||
| 332 | PBAR_TOTAL = 100 | 372 | PBAR_TOTAL = 100 |
| @@ -365,12 +405,15 @@ def calculate_list(path_n, path_b, prefix_str=Const.RANK, mode=GraphConst.INTERS | |||
| 365 | 计算列表,可取交集、并集 | 405 | 计算列表,可取交集、并集 |
| 366 | """ | 406 | """ |
| 367 | if prefix_str not in [Const.RANK, Const.STEP]: | 407 | if prefix_str not in [Const.RANK, Const.STEP]: |
| 368 | - logger.error(f'Parameter "prefix_str" is expected to be {Const.RANK} or {Const.STEP}, ' | 408 | + logger.error( |
| 369 | - f'but in reality it is {prefix_str}.') | 409 | + f'Parameter "prefix_str" is expected to be {Const.RANK} or {Const.STEP}, but in reality it is {prefix_str}.' |
| 410 | + ) | ||
| 370 | raise MsprobeException(MsprobeException.INVALID_PARAM_ERROR) | 411 | raise MsprobeException(MsprobeException.INVALID_PARAM_ERROR) |
| 371 | if mode not in [GraphConst.INTERSECTION, GraphConst.UNION]: | 412 | if mode not in [GraphConst.INTERSECTION, GraphConst.UNION]: |
| 372 | - logger.error(f'Parameter "mode" is expected to be {GraphConst.INTERSECTION} or {GraphConst.UNION}, ' | 413 | + logger.error( |
| 373 | - f'but in reality it is {mode}.') | 414 | + f'Parameter "mode" is expected to be {GraphConst.INTERSECTION} or {GraphConst.UNION}, ' |
| 415 | + f'but in reality it is {mode}.' | ||
| 416 | + ) | ||
| 374 | raise MsprobeException(MsprobeException.INVALID_PARAM_ERROR) | 417 | raise MsprobeException(MsprobeException.INVALID_PARAM_ERROR) |
| 375 | 418 | ||
| 376 | npu_list = check_and_return_dir_contents(path_n, prefix_str, skip_wrong_dir=True) | 419 | npu_list = check_and_return_dir_contents(path_n, prefix_str, skip_wrong_dir=True) |
| @@ -404,6 +447,7 @@ class ProgressInfo: | |||
| 404 | """ | 447 | """ |
| 405 | 前端监测此类属性获取进度信息 | 448 | 前端监测此类属性获取进度信息 |
| 406 | """ | 449 | """ |
| 450 | + | ||
| 407 | current_progress = 0 | 451 | current_progress = 0 |
| 408 | process_running = True | 452 | process_running = True |
| 409 | error_msg = [] | 453 | error_msg = [] |
| @@ -458,8 +502,9 @@ def monitor_progress(pbar_info, pbar, all_task_ids, is_parallel_merge=False): | |||
| 458 | task_progress[task_id] = 0 | 502 | task_progress[task_id] = 0 |
| 459 | # 图合并模式,总进度卡99%,剩余的1%给db后处理函数; 否则总进度卡98%,剩余的1%给db后处理函数,1%给分布式节点分析关联 | 503 | # 图合并模式,总进度卡99%,剩余的1%给db后处理函数; 否则总进度卡98%,剩余的1%给db后处理函数,1%给分布式节点分析关联 |
| 460 | global POST_DB_PROCESS | 504 | global POST_DB_PROCESS |
| 461 | - POST_DB_PROCESS = pbar_info.total - pbar_info.step_total if is_parallel_merge \ | 505 | + POST_DB_PROCESS = ( |
| 462 | - else pbar_info.total - 2 * pbar_info.step_total | 506 | + pbar_info.total - pbar_info.step_total if is_parallel_merge else pbar_info.total - 2 * pbar_info.step_total |
| 507 | + ) | ||
| 463 | all_complete = False | 508 | all_complete = False |
| 464 | 509 | ||
| 465 | while not all_complete: | 510 | while not all_complete: |
| @@ -513,7 +558,7 @@ def update_pbar_info(pbar_info, current_data, total_data, step_ratio=0.01, updat | |||
| 513 | return | 558 | return |
| 514 | 559 | ||
| 515 | progress = (current_data * pbar_info.total // total_data) // pbar_info.stage_total | 560 | progress = (current_data * pbar_info.total // total_data) // pbar_info.stage_total |
| 516 | - progress += (pbar_info.stage_progress * (pbar_info.current_stage_dict.get(pbar_info.task_id) - 1)) | 561 | + progress += pbar_info.stage_progress * (pbar_info.current_stage_dict.get(pbar_info.task_id) - 1) |
| 517 | if pbar_info.pbar is not None: | 562 | if pbar_info.pbar is not None: |
| 518 | if progress == pbar_info.total: | 563 | if progress == pbar_info.total: |
| 519 | # 卡99%,剩余的1%给db后处理函数 | 564 | # 卡99%,剩余的1%给db后处理函数 |
| @@ -0,0 +1,456 @@ | |||
| 1 | +# coding=utf-8 | ||
| 2 | +# ------------------------------------------------------------------------- | ||
| 3 | +# This file is part of the MindStudio project. | ||
| 4 | +# Copyright (c) 2025 Huawei Technologies Co.,Ltd. | ||
| 5 | +# | ||
| 6 | +# MindStudio is licensed under Mulan PSL v2. | ||
| 7 | +# You can use this software according to the terms and conditions of the Mulan PSL v2. | ||
| 8 | +# You may obtain a copy of Mulan PSL v2 at: | ||
| 9 | +# | ||
| 10 | +# http://license.coscl.org.cn/MulanPSL2 | ||
| 11 | +# | ||
| 12 | +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, | ||
| 13 | +# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, | ||
| 14 | +# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. | ||
| 15 | +# See the Mulan PSL v2 for more details. | ||
| 16 | +# ------------------------------------------------------------------------- | ||
| 17 | + | ||
| 18 | +import unittest | ||
| 19 | +from unittest import mock | ||
| 20 | + | ||
| 21 | +from msprobe.core.compare.algorithm.algorithm_scheduler import AlgorithmScheduler | ||
| 22 | + | ||
| 23 | +EXPECTED_BUILTIN_ALGORITHMS = { | ||
| 24 | + "cosine_similarity", | ||
| 25 | + "euclidean_distance", | ||
| 26 | + "five_thousand_error", | ||
| 27 | + "max_absolute_error", | ||
| 28 | + "max_relative_error", | ||
| 29 | + "one_thousand_error", | ||
| 30 | +} | ||
| 31 | +EXPECTED_BUILTIN_COLUMN_NAMES = { | ||
| 32 | + "Cosine", | ||
| 33 | + "EucDist", | ||
| 34 | + "Five Thousandths Err Ratio", | ||
| 35 | + "MaxAbsErr", | ||
| 36 | + "MaxRelativeErr", | ||
| 37 | + "One Thousandth Err Ratio", | ||
| 38 | +} | ||
| 39 | + | ||
| 40 | + | ||
| 41 | +def _two_param_func(n_value, b_value): | ||
| 42 | + return 0.0, "" | ||
| 43 | + | ||
| 44 | + | ||
| 45 | +def _three_param_func(n_value, b_value, args): | ||
| 46 | + return 0.0, "" | ||
| 47 | + | ||
| 48 | + | ||
| 49 | +def _one_param_func(n_value): | ||
| 50 | + return 0.0, "" | ||
| 51 | + | ||
| 52 | + | ||
| 53 | +class TestAlgorithmScheduler(unittest.TestCase): | ||
| 54 | + """AlgorithmScheduler 单例/发现/校验/调度行为单测。""" | ||
| 55 | + | ||
| 56 | + def setUp(self): | ||
| 57 | + # 每个用例隔离单例状态,避免相互污染 | ||
| 58 | + AlgorithmScheduler._instance = None | ||
| 59 | + | ||
| 60 | + def tearDown(self): | ||
| 61 | + AlgorithmScheduler._instance = None | ||
| 62 | + | ||
| 63 | + # ------------------------------------------------------------------ | ||
| 64 | + # 辅助:构造一个未经 __init__ 的纯净实例,用于测试纯方法/受控状态 | ||
| 65 | + # ------------------------------------------------------------------ | ||
| 66 | + @staticmethod | ||
| 67 | + def _bare_instance(): | ||
| 68 | + inst = object.__new__(AlgorithmScheduler) | ||
| 69 | + inst._module_cache = {} | ||
| 70 | + inst.build_in_support_algorithm = [] | ||
| 71 | + inst.custom_support_algorithm = [] | ||
| 72 | + inst.algorithm_names = [] | ||
| 73 | + inst._column_name_algorithm_mapping = {} | ||
| 74 | + inst.algorithm_column_names = [] | ||
| 75 | + return inst | ||
| 76 | + | ||
| 77 | + # ------------------------------------------------------------------ | ||
| 78 | + # 单例行为 | ||
| 79 | + # ------------------------------------------------------------------ | ||
| 80 | + def test_get_instance_given_multiple_calls_when_any_then_same_instance(self): | ||
| 81 | + inst1 = AlgorithmScheduler.get_instance() | ||
| 82 | + inst2 = AlgorithmScheduler.get_instance() | ||
| 83 | + inst3 = AlgorithmScheduler.get_instance() | ||
| 84 | + self.assertIs(inst1, inst2) | ||
| 85 | + self.assertIs(inst2, inst3) | ||
| 86 | + | ||
| 87 | + def test_direct_new_given_any_when_invoke_then_runtime_error(self): | ||
| 88 | + with self.assertRaises(RuntimeError): | ||
| 89 | + AlgorithmScheduler() | ||
| 90 | + | ||
| 91 | + # ------------------------------------------------------------------ | ||
| 92 | + # builtin 算法自动发现 | ||
| 93 | + # ------------------------------------------------------------------ | ||
| 94 | + def test_make_support_algorithm_given_builtin_dir_when_load_then_all_discovered(self): | ||
| 95 | + inst = AlgorithmScheduler.get_instance() | ||
| 96 | + self.assertEqual(set(inst.build_in_support_algorithm), EXPECTED_BUILTIN_ALGORITHMS) | ||
| 97 | + self.assertEqual(inst.custom_support_algorithm, []) | ||
| 98 | + | ||
| 99 | + def test_validate_column_names_given_builtin_when_load_then_no_duplicate(self): | ||
| 100 | + # __init__ 已执行 _validate_column_names,未抛异常即说明列名唯一 | ||
| 101 | + inst = AlgorithmScheduler.get_instance() | ||
| 102 | + self.assertEqual(set(inst.algorithm_column_names), EXPECTED_BUILTIN_COLUMN_NAMES) | ||
| 103 | + self.assertEqual(len(inst.algorithm_column_names), len(set(inst.algorithm_column_names))) | ||
| 104 | + | ||
| 105 | + def test_get_algorithm_names_given_loaded_when_call_then_returns_all(self): | ||
| 106 | + inst = AlgorithmScheduler.get_instance() | ||
| 107 | + self.assertEqual(set(inst.get_algorithm_names()), EXPECTED_BUILTIN_ALGORITHMS) | ||
| 108 | + | ||
| 109 | + def test_get_algorithm_column_names_given_loaded_when_call_then_returns_all(self): | ||
| 110 | + inst = AlgorithmScheduler.get_instance() | ||
| 111 | + self.assertEqual(set(inst.get_algorithm_column_names()), EXPECTED_BUILTIN_COLUMN_NAMES) | ||
| 112 | + | ||
| 113 | + # ------------------------------------------------------------------ | ||
| 114 | + # column_name 去重校验 | ||
| 115 | + # ------------------------------------------------------------------ | ||
| 116 | + def test_validate_column_names_given_duplicate_when_validate_then_value_error(self): | ||
| 117 | + # 跳过真实加载,构造受控状态后直接测试 _validate_column_names | ||
| 118 | + with mock.patch.object(AlgorithmScheduler, "_make_support_algorithm", lambda self: None): | ||
| 119 | + inst = AlgorithmScheduler.get_instance() | ||
| 120 | + inst.build_in_support_algorithm = ["alg_a", "alg_b"] | ||
| 121 | + inst.custom_support_algorithm = [] | ||
| 122 | + inst.algorithm_names = ["alg_a", "alg_b"] | ||
| 123 | + inst._column_name_algorithm_mapping = {} | ||
| 124 | + dummy_module = mock.Mock() | ||
| 125 | + with mock.patch.object(inst, "_get_module", return_value=dummy_module), \ | ||
| 126 | + mock.patch.object(inst, "_get_column_name", side_effect=lambda name, mod: "dup_col"): | ||
| 127 | + with self.assertRaisesRegex(ValueError, "duplicated"): | ||
| 128 | + inst._validate_column_names() | ||
| 129 | + | ||
| 130 | + def test_validate_column_names_given_unique_when_validate_then_mapping_built(self): | ||
| 131 | + with mock.patch.object(AlgorithmScheduler, "_make_support_algorithm", lambda self: None): | ||
| 132 | + inst = AlgorithmScheduler.get_instance() | ||
| 133 | + inst.build_in_support_algorithm = ["alg_a", "alg_b"] | ||
| 134 | + inst.custom_support_algorithm = [] | ||
| 135 | + inst.algorithm_names = ["alg_a", "alg_b"] | ||
| 136 | + inst._column_name_algorithm_mapping = {} | ||
| 137 | + dummy_module = mock.Mock() | ||
| 138 | + col_names = {"alg_a": "ColA", "alg_b": "ColB"} | ||
| 139 | + with mock.patch.object(inst, "_get_module", return_value=dummy_module), \ | ||
| 140 | + mock.patch.object(inst, "_get_column_name", side_effect=lambda name, mod: col_names[name]): | ||
| 141 | + inst._validate_column_names() | ||
| 142 | + self.assertEqual(inst._column_name_algorithm_mapping, {"ColA": "alg_a", "ColB": "alg_b"}) | ||
| 143 | + | ||
| 144 | + # ------------------------------------------------------------------ | ||
| 145 | + # _is_real_number | ||
| 146 | + # ------------------------------------------------------------------ | ||
| 147 | + def test_is_real_number_given_int_when_check_then_true(self): | ||
| 148 | + inst = self._bare_instance() | ||
| 149 | + self.assertTrue(inst._is_real_number(1)) | ||
| 150 | + | ||
| 151 | + def test_is_real_number_given_float_when_check_then_true(self): | ||
| 152 | + inst = self._bare_instance() | ||
| 153 | + self.assertTrue(inst._is_real_number(1.5)) | ||
| 154 | + | ||
| 155 | + def test_is_real_number_given_bool_when_check_then_false(self): | ||
| 156 | + inst = self._bare_instance() | ||
| 157 | + self.assertFalse(inst._is_real_number(True)) | ||
| 158 | + self.assertFalse(inst._is_real_number(False)) | ||
| 159 | + | ||
| 160 | + def test_is_real_number_given_non_number_when_check_then_false(self): | ||
| 161 | + inst = self._bare_instance() | ||
| 162 | + self.assertFalse(inst._is_real_number("1")) | ||
| 163 | + self.assertFalse(inst._is_real_number(None)) | ||
| 164 | + self.assertFalse(inst._is_real_number([1])) | ||
| 165 | + | ||
| 166 | + # ------------------------------------------------------------------ | ||
| 167 | + # _validate_return_value | ||
| 168 | + # ------------------------------------------------------------------ | ||
| 169 | + def test_validate_return_value_given_int_str_when_valid_then_true(self): | ||
| 170 | + inst = self._bare_instance() | ||
| 171 | + self.assertTrue(inst._validate_return_value((1, "msg"), "alg")) | ||
| 172 | + | ||
| 173 | + def test_validate_return_value_given_float_str_when_valid_then_true(self): | ||
| 174 | + inst = self._bare_instance() | ||
| 175 | + self.assertTrue(inst._validate_return_value((1.5, "msg"), "alg")) | ||
| 176 | + | ||
| 177 | + def test_validate_return_value_given_str_str_when_valid_then_true(self): | ||
| 178 | + inst = self._bare_instance() | ||
| 179 | + self.assertTrue(inst._validate_return_value(("err", "msg"), "alg")) | ||
| 180 | + | ||
| 181 | + def test_validate_return_value_given_non_tuple_when_invalid_then_value_error(self): | ||
| 182 | + inst = self._bare_instance() | ||
| 183 | + for bad in (5, "abc", [1, "a"], None): | ||
| 184 | + with self.assertRaises(ValueError): | ||
| 185 | + inst._validate_return_value(bad, "alg") | ||
| 186 | + | ||
| 187 | + def test_validate_return_value_given_wrong_length_when_invalid_then_value_error(self): | ||
| 188 | + inst = self._bare_instance() | ||
| 189 | + for bad in ((1,), (1, "a", "b"), tuple()): | ||
| 190 | + with self.assertRaises(ValueError): | ||
| 191 | + inst._validate_return_value(bad, "alg") | ||
| 192 | + | ||
| 193 | + def test_validate_return_value_given_bool_first_when_invalid_then_value_error(self): | ||
| 194 | + # bool 被排除在实数之外,且不属于 str,应当拦截 | ||
| 195 | + inst = self._bare_instance() | ||
| 196 | + with self.assertRaises(ValueError): | ||
| 197 | + inst._validate_return_value((True, "msg"), "alg") | ||
| 198 | + | ||
| 199 | + def test_validate_return_value_given_invalid_first_type_when_invalid_then_value_error(self): | ||
| 200 | + inst = self._bare_instance() | ||
| 201 | + for bad_first in ([1], {"a": 1}, None, object()): | ||
| 202 | + with self.assertRaises(ValueError): | ||
| 203 | + inst._validate_return_value((bad_first, "msg"), "alg") | ||
| 204 | + | ||
| 205 | + def test_validate_return_value_given_invalid_second_type_when_invalid_then_value_error(self): | ||
| 206 | + inst = self._bare_instance() | ||
| 207 | + for bad_second in (1, 1.5, None, [1]): | ||
| 208 | + with self.assertRaises(ValueError): | ||
| 209 | + inst._validate_return_value((1, bad_second), "alg") | ||
| 210 | + | ||
| 211 | + # ------------------------------------------------------------------ | ||
| 212 | + # _get_module | ||
| 213 | + # ------------------------------------------------------------------ | ||
| 214 | + def test_get_module_given_unsupported_name_when_call_then_value_error(self): | ||
| 215 | + inst = self._bare_instance() | ||
| 216 | + with self.assertRaisesRegex(ValueError, "is not supported"): | ||
| 217 | + inst._get_module("not_an_algorithm") | ||
| 218 | + | ||
| 219 | + def test_get_module_given_loaded_when_call_twice_then_cached(self): | ||
| 220 | + inst = AlgorithmScheduler.get_instance() | ||
| 221 | + inst._module_cache.clear() | ||
| 222 | + m1 = inst._get_module("cosine_similarity") | ||
| 223 | + m2 = inst._get_module("cosine_similarity") | ||
| 224 | + self.assertIs(m1, m2) | ||
| 225 | + self.assertIn("cosine_similarity", inst._module_cache) | ||
| 226 | + | ||
| 227 | + def test_get_module_given_missing_compare_when_load_then_attribute_error(self): | ||
| 228 | + inst = self._bare_instance() | ||
| 229 | + inst.build_in_support_algorithm = ["fake_alg"] | ||
| 230 | + fake_module = mock.Mock(spec=[]) # 无 compare 属性 | ||
| 231 | + with mock.patch("importlib.import_module", return_value=fake_module): | ||
| 232 | + with self.assertRaisesRegex(AttributeError, "must define a compare"): | ||
| 233 | + inst._get_module("fake_alg") | ||
| 234 | + | ||
| 235 | + def test_get_module_given_compare_not_callable_when_load_then_type_error(self): | ||
| 236 | + inst = self._bare_instance() | ||
| 237 | + inst.build_in_support_algorithm = ["fake_alg"] | ||
| 238 | + fake_module = mock.Mock() | ||
| 239 | + fake_module.compare = "not_callable" | ||
| 240 | + with mock.patch("importlib.import_module", return_value=fake_module): | ||
| 241 | + with self.assertRaisesRegex(TypeError, "not a callable"): | ||
| 242 | + inst._get_module("fake_alg") | ||
| 243 | + | ||
| 244 | + def test_get_module_given_wrong_arg_count_when_load_then_value_error(self): | ||
| 245 | + inst = self._bare_instance() | ||
| 246 | + inst.build_in_support_algorithm = ["fake_alg"] | ||
| 247 | + for func in (_one_param_func, _three_param_func): | ||
| 248 | + fake_module = mock.Mock() | ||
| 249 | + fake_module.compare = func | ||
| 250 | + with mock.patch("importlib.import_module", return_value=fake_module): | ||
| 251 | + with self.assertRaisesRegex(ValueError, "positional arguments"): | ||
| 252 | + inst._get_module("fake_alg") | ||
| 253 | + inst._module_cache.clear() | ||
| 254 | + | ||
| 255 | + def test_get_module_given_valid_signature_when_load_then_cached(self): | ||
| 256 | + inst = self._bare_instance() | ||
| 257 | + inst.build_in_support_algorithm = ["fake_alg"] | ||
| 258 | + fake_module = mock.Mock() | ||
| 259 | + fake_module.compare = _two_param_func | ||
| 260 | + with mock.patch("importlib.import_module", return_value=fake_module) as mock_import: | ||
| 261 | + module = inst._get_module("fake_alg") | ||
| 262 | + self.assertIs(module, fake_module) | ||
| 263 | + self.assertEqual(mock_import.call_count, 1) | ||
| 264 | + # 第二次命中缓存,不应再次 import | ||
| 265 | + inst._get_module("fake_alg") | ||
| 266 | + self.assertEqual(mock_import.call_count, 1) | ||
| 267 | + | ||
| 268 | + def test_get_module_given_import_failure_when_load_then_propagates(self): | ||
| 269 | + inst = self._bare_instance() | ||
| 270 | + inst.custom_support_algorithm = ["fake_alg"] | ||
| 271 | + with mock.patch("importlib.import_module", side_effect=ModuleNotFoundError("No module")): | ||
| 272 | + with self.assertRaises(ModuleNotFoundError): | ||
| 273 | + inst._get_module("fake_alg") | ||
| 274 | + | ||
| 275 | + def test_get_module_given_custom_when_load_then_uses_custom_path(self): | ||
| 276 | + inst = self._bare_instance() | ||
| 277 | + inst.custom_support_algorithm = ["fake_alg"] | ||
| 278 | + fake_module = mock.Mock() | ||
| 279 | + fake_module.compare = _two_param_func | ||
| 280 | + with mock.patch("importlib.import_module", return_value=fake_module) as mock_import: | ||
| 281 | + inst._get_module("fake_alg") | ||
| 282 | + called_module_name = mock_import.call_args[0][0] | ||
| 283 | + self.assertIn(AlgorithmScheduler.CUSTOM_ALGORITHM_DIR_NAME, called_module_name) | ||
| 284 | + | ||
| 285 | + def test_get_module_given_builtin_when_load_then_uses_builtin_path(self): | ||
| 286 | + inst = self._bare_instance() | ||
| 287 | + inst.build_in_support_algorithm = ["fake_alg"] | ||
| 288 | + fake_module = mock.Mock() | ||
| 289 | + fake_module.compare = _two_param_func | ||
| 290 | + with mock.patch("importlib.import_module", return_value=fake_module) as mock_import: | ||
| 291 | + inst._get_module("fake_alg") | ||
| 292 | + called_module_name = mock_import.call_args[0][0] | ||
| 293 | + self.assertIn(AlgorithmScheduler.BUILT_IN_ALGORITHM_DIR_NAME, called_module_name) | ||
| 294 | + | ||
| 295 | + # ------------------------------------------------------------------ | ||
| 296 | + # _get_column_name | ||
| 297 | + # ------------------------------------------------------------------ | ||
| 298 | + def test_get_column_name_given_missing_func_when_call_then_attribute_error(self): | ||
| 299 | + inst = self._bare_instance() | ||
| 300 | + fake_module = mock.Mock(spec=[]) | ||
| 301 | + with self.assertRaisesRegex(AttributeError, "must define a column_name"): | ||
| 302 | + inst._get_column_name("alg", fake_module) | ||
| 303 | + | ||
| 304 | + def test_get_column_name_given_not_callable_when_call_then_type_error(self): | ||
| 305 | + inst = self._bare_instance() | ||
| 306 | + fake_module = mock.Mock() | ||
| 307 | + fake_module.column_name = "not_callable" | ||
| 308 | + with self.assertRaisesRegex(TypeError, "not a callable"): | ||
| 309 | + inst._get_column_name("alg", fake_module) | ||
| 310 | + | ||
| 311 | + def test_get_column_name_given_func_raises_when_call_then_value_error(self): | ||
| 312 | + inst = self._bare_instance() | ||
| 313 | + fake_module = mock.Mock() | ||
| 314 | + fake_module.column_name = mock.Mock(side_effect=RuntimeError("boom")) | ||
| 315 | + with self.assertRaisesRegex(ValueError, "Error occurred"): | ||
| 316 | + inst._get_column_name("alg", fake_module) | ||
| 317 | + | ||
| 318 | + def test_get_column_name_given_empty_return_when_call_then_value_error(self): | ||
| 319 | + inst = self._bare_instance() | ||
| 320 | + fake_module = mock.Mock() | ||
| 321 | + fake_module.column_name = lambda: "" | ||
| 322 | + with self.assertRaisesRegex(ValueError, "non"): | ||
| 323 | + inst._get_column_name("alg", fake_module) | ||
| 324 | + | ||
| 325 | + def test_get_column_name_given_non_string_return_when_call_then_value_error(self): | ||
| 326 | + inst = self._bare_instance() | ||
| 327 | + fake_module = mock.Mock() | ||
| 328 | + fake_module.column_name = lambda: 123 | ||
| 329 | + with self.assertRaises(ValueError): | ||
| 330 | + inst._get_column_name("alg", fake_module) | ||
| 331 | + | ||
| 332 | + def test_get_column_name_given_valid_when_call_then_returns_name(self): | ||
| 333 | + inst = self._bare_instance() | ||
| 334 | + fake_module = mock.Mock() | ||
| 335 | + fake_module.column_name = lambda: "ColA" | ||
| 336 | + self.assertEqual(inst._get_column_name("alg", fake_module), "ColA") | ||
| 337 | + | ||
| 338 | + # ------------------------------------------------------------------ | ||
| 339 | + # _call_algorithm | ||
| 340 | + # ------------------------------------------------------------------ | ||
| 341 | + | ||
| 342 | + def test_call_algorithm_given_valid_return_when_call_then_returns_result(self): | ||
| 343 | + inst = self._bare_instance() | ||
| 344 | + fake_module = mock.Mock() | ||
| 345 | + fake_module.compare = lambda n, b: (0.5, "") | ||
| 346 | + with mock.patch.object(inst, "_get_module", return_value=fake_module): | ||
| 347 | + result = inst._call_algorithm("alg", 1, 2) | ||
| 348 | + self.assertEqual(result, (0.5, "")) | ||
| 349 | + | ||
| 350 | + # ------------------------------------------------------------------ | ||
| 351 | + # compare 编排 | ||
| 352 | + # ------------------------------------------------------------------ | ||
| 353 | + def test_compare_given_unknown_column_when_call_then_unsupported(self): | ||
| 354 | + inst = self._bare_instance() | ||
| 355 | + inst._column_name_algorithm_mapping = {"ColA": "alg_a", "ColB": "alg_b"} | ||
| 356 | + inst.algorithm_column_names = ["ColA", "ColB"] | ||
| 357 | + results, errors = inst.compare(1, 2, column_names=["Unknown"]) | ||
| 358 | + self.assertEqual(results, ["unsupported"]) | ||
| 359 | + self.assertIn("No available algorithm", errors[0]) | ||
| 360 | + | ||
| 361 | + def test_compare_given_none_columns_when_call_then_uses_all(self): | ||
| 362 | + inst = self._bare_instance() | ||
| 363 | + inst._column_name_algorithm_mapping = {"ColA": "alg_a", "ColB": "alg_b"} | ||
| 364 | + inst.algorithm_column_names = ["ColA", "ColB"] | ||
| 365 | + with mock.patch.object(inst, "_call_algorithm", return_value=(0.0, "")) as mock_call: | ||
| 366 | + results, errors = inst.compare(1, 2, column_names=None) | ||
| 367 | + self.assertEqual(len(results), 2) | ||
| 368 | + self.assertEqual(mock_call.call_count, 2) | ||
| 369 | + | ||
| 370 | + def test_compare_given_empty_columns_when_call_then_uses_all(self): | ||
| 371 | + inst = self._bare_instance() | ||
| 372 | + inst._column_name_algorithm_mapping = {"ColA": "alg_a", "ColB": "alg_b"} | ||
| 373 | + inst.algorithm_column_names = ["ColA", "ColB"] | ||
| 374 | + with mock.patch.object(inst, "_call_algorithm", return_value=(0.0, "")) as mock_call: | ||
| 375 | + results, errors = inst.compare(1, 2, column_names=[]) | ||
| 376 | + self.assertEqual(len(results), 2) | ||
| 377 | + self.assertEqual(mock_call.call_count, 2) | ||
| 378 | + | ||
| 379 | + def test_compare_given_subset_columns_when_call_then_only_subset_invoked(self): | ||
| 380 | + inst = self._bare_instance() | ||
| 381 | + inst._column_name_algorithm_mapping = {"ColA": "alg_a", "ColB": "alg_b"} | ||
| 382 | + inst.algorithm_column_names = ["ColA", "ColB"] | ||
| 383 | + with mock.patch.object(inst, "_call_algorithm", return_value=(0.0, "")) as mock_call: | ||
| 384 | + results, errors = inst.compare(1, 2, column_names=["ColA"]) | ||
| 385 | + self.assertEqual(len(results), 1) | ||
| 386 | + self.assertEqual(mock_call.call_count, 1) | ||
| 387 | + | ||
| 388 | + def test_compare_given_error_msg_when_call_then_error_msgs_collected(self): | ||
| 389 | + inst = self._bare_instance() | ||
| 390 | + inst._column_name_algorithm_mapping = {"ColA": "alg_a"} | ||
| 391 | + inst.algorithm_column_names = ["ColA"] | ||
| 392 | + with mock.patch.object(inst, "_call_algorithm", return_value=(0.0, "some error")): | ||
| 393 | + results, errors = inst.compare(1, 2, column_names=["ColA"]) | ||
| 394 | + self.assertEqual(errors, ["some error"]) | ||
| 395 | + | ||
| 396 | + # ------------------------------------------------------------------ | ||
| 397 | + # 自定义/内置算法目录加载(含空目录、非法文件名跳过) | ||
| 398 | + # ------------------------------------------------------------------ | ||
| 399 | + def test_make_support_algorithm_from_dir_given_nonexistent_when_call_then_skip(self): | ||
| 400 | + inst = self._bare_instance() | ||
| 401 | + algos = [] | ||
| 402 | + with mock.patch("os.path.exists", return_value=False): | ||
| 403 | + inst._make_support_algorithm_from_dir("/no/such/dir", algos) | ||
| 404 | + self.assertEqual(algos, []) | ||
| 405 | + | ||
| 406 | + def test_make_support_algorithm_from_dir_given_empty_when_call_then_no_algorithm(self): | ||
| 407 | + inst = self._bare_instance() | ||
| 408 | + algos = [] | ||
| 409 | + with mock.patch("os.path.exists", return_value=True), \ | ||
| 410 | + mock.patch("os.listdir", return_value=[]), \ | ||
| 411 | + mock.patch("os.path.isfile", return_value=True): | ||
| 412 | + inst._make_support_algorithm_from_dir("/fake/dir", algos) | ||
| 413 | + self.assertEqual(algos, []) | ||
| 414 | + | ||
| 415 | + def test_make_support_algorithm_from_dir_given_mixed_names_when_call_then_only_valid_kept(self): | ||
| 416 | + inst = self._bare_instance() | ||
| 417 | + algos = [] | ||
| 418 | + # alg_OK.py 含大写,不匹配 [a-z0-9_];subdir 无 .py 后缀;not_match.py 不以 alg_ 开头 | ||
| 419 | + files = ["alg_valid.py", "not_match.py", "alg_OK.py", "alg_another.py", "subdir"] | ||
| 420 | + with mock.patch("os.path.exists", return_value=True), \ | ||
| 421 | + mock.patch("os.listdir", return_value=files), \ | ||
| 422 | + mock.patch("os.path.isfile", return_value=True): | ||
| 423 | + inst._make_support_algorithm_from_dir("/fake/dir", algos) | ||
| 424 | + self.assertEqual(algos, ["valid", "another"]) | ||
| 425 | + | ||
| 426 | + def test_add_algorithm_file_to_list_given_valid_name_when_call_then_appended(self): | ||
| 427 | + algos = [] | ||
| 428 | + with mock.patch("os.path.isfile", return_value=True): | ||
| 429 | + result = AlgorithmScheduler._add_algorithm_file_to_list("/x/alg_test.py", algos) | ||
| 430 | + self.assertTrue(result) | ||
| 431 | + self.assertEqual(algos, ["test"]) | ||
| 432 | + | ||
| 433 | + def test_add_algorithm_file_to_list_given_invalid_name_when_call_then_skipped(self): | ||
| 434 | + algos = [] | ||
| 435 | + with mock.patch("os.path.isfile", return_value=True): | ||
| 436 | + result = AlgorithmScheduler._add_algorithm_file_to_list("/x/not_match.py", algos) | ||
| 437 | + self.assertFalse(result) | ||
| 438 | + self.assertEqual(algos, []) | ||
| 439 | + | ||
| 440 | + def test_add_algorithm_file_to_list_given_non_file_when_call_then_skipped(self): | ||
| 441 | + algos = [] | ||
| 442 | + with mock.patch("os.path.isfile", return_value=False): | ||
| 443 | + result = AlgorithmScheduler._add_algorithm_file_to_list("/x/alg_test.py", algos) | ||
| 444 | + self.assertFalse(result) | ||
| 445 | + self.assertEqual(algos, []) | ||
| 446 | + | ||
| 447 | + def test_add_algorithm_file_to_list_given_uppercase_when_call_then_skipped(self): | ||
| 448 | + algos = [] | ||
| 449 | + with mock.patch("os.path.isfile", return_value=True): | ||
| 450 | + result = AlgorithmScheduler._add_algorithm_file_to_list("/x/alg_OK.py", algos) | ||
| 451 | + self.assertFalse(result) | ||
| 452 | + self.assertEqual(algos, []) | ||
| 453 | + | ||
| 454 | + | ||
| 455 | +if __name__ == "__main__": | ||
| 456 | + unittest.main() | ||
| @@ -113,15 +113,11 @@ class TestCompareRealData(unittest.TestCase): | |||
| 113 | cross_frame = False | 113 | cross_frame = False |
| 114 | compare_real_data = CompareRealData(file_reader, mode_config, cross_frame) | 114 | compare_real_data = CompareRealData(file_reader, mode_config, cross_frame) |
| 115 | 115 | ||
| 116 | - comparison_result = ComparisonResult( | 116 | + results = {CompareConst.COSINE: [0.99, 0.98], CompareConst.MAX_ABS_ERR: [0.01, 0.02], |
| 117 | - cos_result=[0.99, 0.98], | 117 | + CompareConst.MAX_RELATIVE_ERR: [0.001, 0.002], CompareConst.EUC_DIST: [0.5, 0.49], |
| 118 | - max_err_result=[0.01, 0.02], | 118 | + CompareConst.ONE_THOUSANDTH_ERR_RATIO: [0.1, 0.2], CompareConst.FIVE_THOUSANDTHS_ERR_RATIO: [0.05, 0.1]} |
| 119 | - max_relative_err_result=[0.001, 0.002], | 119 | + comparison_result = ComparisonResult(results, ['', 'Error in comparison']) |
| 120 | - euc_dist_result=[0.5, 0.49], | 120 | + |
| 121 | - one_thousand_err_ratio_result=[0.1, 0.2], | ||
| 122 | - five_thousand_err_ratio_result=[0.05, 0.1], | ||
| 123 | - err_msgs=['', 'Error in comparison'] | ||
| 124 | - ) | ||
| 125 | offset = 0 | 121 | offset = 0 |
| 126 | updated_df = compare_real_data._save_cmp_result(offset, comparison_result, self.result_df, self.lock) | 122 | updated_df = compare_real_data._save_cmp_result(offset, comparison_result, self.result_df, self.lock) |
| 127 | 123 | ||
| @@ -135,18 +131,13 @@ class TestCompareRealData(unittest.TestCase): | |||
| 135 | cross_frame = False | 131 | cross_frame = False |
| 136 | compare_real_data = CompareRealData(file_reader, mode_config, cross_frame) | 132 | compare_real_data = CompareRealData(file_reader, mode_config, cross_frame) |
| 137 | 133 | ||
| 138 | - comparison_result = ComparisonResult( | 134 | + results = {CompareConst.COSINE: [0.99], CompareConst.MAX_ABS_ERR: [0.98], |
| 139 | - cos_result=[0.99], | 135 | + CompareConst.MAX_RELATIVE_ERR: [0.001], CompareConst.EUC_DIST: [0.5], |
| 140 | - max_err_result=[], | 136 | + CompareConst.ONE_THOUSANDTH_ERR_RATIO: [0.1], CompareConst.FIVE_THOUSANDTHS_ERR_RATIO: [0.05]} |
| 141 | - max_relative_err_result=[0.001], | 137 | + comparison_result = ComparisonResult(results, [""]) |
| 142 | - euc_dist_result=[0.5], | ||
| 143 | - one_thousand_err_ratio_result=[0.1], | ||
| 144 | - five_thousand_err_ratio_result=[0.05], | ||
| 145 | - err_msgs=[''] | ||
| 146 | - ) | ||
| 147 | with self.assertRaises(CompareException) as context: | 138 | with self.assertRaises(CompareException) as context: |
| 148 | compare_real_data._save_cmp_result(0, comparison_result, self.result_df, self.lock) | 139 | compare_real_data._save_cmp_result(0, comparison_result, self.result_df, self.lock) |
| 149 | - self.assertEqual(context.exception.code, CompareException.INDEX_OUT_OF_BOUNDS_ERROR) | 140 | + self.assertEqual(context.exception.code, CompareException.INVALID_DATA_ERROR) |
| 150 | 141 | ||
| 151 | def test_compare_by_op_bench_normal(self): | 142 | def test_compare_by_op_bench_normal(self): |
| 152 | npu_op_name = 'Functional.linear.0.forward.input.0' | 143 | npu_op_name = 'Functional.linear.0.forward.input.0' |
| @@ -81,9 +81,6 @@ class TestModeAdapter(unittest.TestCase): | |||
| 81 | 81 | ||
| 82 | self.adapter.compare_mode = GraphConst.REAL_DATA_COMPARE | 82 | self.adapter.compare_mode = GraphConst.REAL_DATA_COMPARE |
| 83 | tips = self.adapter.get_tool_tip() | 83 | tips = self.adapter.get_tool_tip() |
| 84 | - self.assertEqual(tips, json.dumps({ | 84 | + algorithm_columns = CompareConst.ALL_COMPARE_INDEX |
| 85 | - CompareConst.ONE_THOUSANDTH_ERR_RATIO: ToolTip.ONE_THOUSANDTH_ERR_RATIO, | 85 | + expect_tips = {col: "" for col in algorithm_columns} |
| 86 | - CompareConst.FIVE_THOUSANDTHS_ERR_RATIO: ToolTip.FIVE_THOUSANDTHS_ERR_RATIO, | 86 | + self.assertEqual(tips, json.dumps(expect_tips)) |
| 87 | - CompareConst.COSINE: ToolTip.COSINE, | ||
| 88 | - CompareConst.MAX_ABS_ERR: ToolTip.MAX_ABS_ERR, | ||
| 89 | - CompareConst.MAX_RELATIVE_ERR: ToolTip.MAX_RELATIVE_ERR})) | ||


【review】AlgorithmScheduler 作为本 PR 的核心新增组件(225 行),缺少独立的单元测试