Closed
【Feature】精度比对工具支持自定义比对算法 #956
HowSir_XAdd on Aug 15Closed at Aug 18
【Feature】精度比对工具支持自定义比对算法 #956
Closed
HowSir_XAdd on Aug 15Closed at Aug 18
Total: 25 files changed+1860-881
@@ -15,12 +15,69 @@
15# -------------------------------------------------------------------------15# -------------------------------------------------------------------------
16 16 
17# pylint: disable=too-many-lines17# pylint: disable=too-many-lines
18- 18+from __future__ import annotations
19 19 
20import os20import os
21import stat21import stat
22 22 
23import numpy as np23import numpy as np
24+from typing import Callable, Generic, TypeVar, Any, List, Dict
25+import threading
26+ 
27+_T = TypeVar("_T")
28+ 
29+ 
30+class LazyConst(Generic[_T]):
31+ """懒求值的常量"""
32+ 
33+ def __init__(self, factory: Callable[[], _T]) -> None:
34+ if isinstance(factory, staticmethod):
35+ factory = factory.__func__
36+ self.factory: Callable[[], _T] = factory
37+ self._value: _T | None = None
38+ self._initialized: bool = False
39+ self._lock = threading.RLock()
40+ 
41+ def __get__(self, instance: Any | None, owner: type[Any]) -> _T:
42+ if not self._initialized:
43+ with self._lock:
44+ if not self._initialized:
45+ self._value = self.factory()
46+ self._initialized = True
47+ return self._value
48+ 
49+ 
50+_KT = TypeVar("_KT")
51+_VT = TypeVar("_VT")
52+ 
53+ 
54+class LazyDict(dict, Generic[_KT, _VT]):
55+ """按 key 懒求值的字典"""
56+ 
57+ _factories: Dict[_KT, Callable[[], _VT]]
58+ 
59+ def __init__(self, factories: Dict[_KT, Callable[[], _VT]]) -> None:
60+ self._factories = factories
61+ super().__init__()
62+ self._lock = threading.RLock()
63+ 
64+ def __getitem__(self, key: _KT) -> _VT:
65+ if key in self:
66+ return super().__getitem__(key)
67+ 
68+ with self._lock:
69+ if key not in self and key in self._factories:
70+ self[key] = self._factories[key]()
71+ return super().__getitem__(key)
72+ 
73+ def get(self, key: _KT, default: _VT = None) -> _VT:
74+ if key in self:
75+ return super().get(key, default)
76+ 
77+ with self._lock:
78+ if key not in self and key in self._factories:
79+ self[key] = self._factories[key]()
80+ return super().get(key, default)
24 81 
25 82 
26class Const: # pylint: disable=too-many-lines83class Const: # pylint: disable=too-many-lines
@@ -695,7 +752,7 @@ class CompareConst:
695 VERL_HYPER_PARAM_COMPARE_COLUM = [HYPER_PARAM_NAME, NPU_EFFECTIVE_VALUE, BENCH_EFFECTIVE_VALUE, IS_CONSISTENT]752 VERL_HYPER_PARAM_COMPARE_COLUM = [HYPER_PARAM_NAME, NPU_EFFECTIVE_VALUE, BENCH_EFFECTIVE_VALUE, IS_CONSISTENT]
696 VERL_HYPER_PARAM_VERIFY_COLUM = [KEY_PARAM_NAME, BENCH_EFFECTIVE_VALUE, TARGET_EFFECTIVE_VALUE, IS_CONSISTENT]753 VERL_HYPER_PARAM_VERIFY_COLUM = [KEY_PARAM_NAME, BENCH_EFFECTIVE_VALUE, TARGET_EFFECTIVE_VALUE, IS_CONSISTENT]
697 754 
698- ALL_COMPARE_INDEX = [755+ BUILTIN_COMPARE_COLUMNS = [
699 COSINE,756 COSINE,
700 EUC_DIST,757 EUC_DIST,
701 MAX_ABS_ERR,758 MAX_ABS_ERR,
@@ -703,6 +760,17 @@ class CompareConst:
703 ONE_THOUSANDTH_ERR_RATIO,760 ONE_THOUSANDTH_ERR_RATIO,
704 FIVE_THOUSANDTHS_ERR_RATIO,761 FIVE_THOUSANDTHS_ERR_RATIO,
705 ]762 ]
763+ 
764+ @staticmethod
765+ def _get_custom_compare_index() -> List[str]:
766+ from msprobe.core.compare.algorithm.algorithm_scheduler import AlgorithmScheduler
767+ 
768+ algorithm_columns = AlgorithmScheduler.get_instance().get_algorithm_column_names()
769+ exist_compare_cols = [c for c in CompareConst.BUILTIN_COMPARE_COLUMNS if c in algorithm_columns]
770+ rest_columns = [col for col in algorithm_columns if col not in exist_compare_cols]
771+ return exist_compare_cols + rest_columns
772+ 
773+ ALL_COMPARE_INDEX = LazyConst(_get_custom_compare_index)
706 SUMMARY_COMPARE_INDEX = [774 SUMMARY_COMPARE_INDEX = [
707 MAX_DIFF,775 MAX_DIFF,
708 MIN_DIFF,776 MIN_DIFF,
@@ -737,23 +805,32 @@ class CompareConst:
737 BENCH_NORM,805 BENCH_NORM,
738 ]806 ]
739 807 
740- COMPARE_RESULT_HEADER = BASIC_INFO + ALL_COMPARE_INDEX + SUMMARY_INFO + EXTRACT_INDEX808+ COMPARE_RESULT_HEADER = LazyConst(
809+ lambda: (
810+ CompareConst.BASIC_INFO
811+ + CompareConst.ALL_COMPARE_INDEX
812+ + CompareConst.SUMMARY_INFO
813+ + CompareConst.EXTRACT_INDEX
814+ )
815+ )
741 816 
742 SUMMARY_COMPARE_RESULT_HEADER = BASIC_INFO + SUMMARY_COMPARE_INDEX + SUMMARY_INFO + EXTRACT_INDEX817 SUMMARY_COMPARE_RESULT_HEADER = BASIC_INFO + SUMMARY_COMPARE_INDEX + SUMMARY_INFO + EXTRACT_INDEX
743 818 
744 MD5_COMPARE_RESULT_HEADER = BASIC_INFO + [NPU_MD5, BENCH_MD5] + EXTRACT_INDEX819 MD5_COMPARE_RESULT_HEADER = BASIC_INFO + [NPU_MD5, BENCH_MD5] + EXTRACT_INDEX
745 820 
746- COMPARE_RESULT_HEADER_STACK = COMPARE_RESULT_HEADER + [STACK]821+ COMPARE_RESULT_HEADER_STACK = LazyConst(lambda: CompareConst.COMPARE_RESULT_HEADER + [CompareConst.STACK])
747 822 
748 SUMMARY_COMPARE_RESULT_HEADER_STACK = SUMMARY_COMPARE_RESULT_HEADER + [STACK]823 SUMMARY_COMPARE_RESULT_HEADER_STACK = SUMMARY_COMPARE_RESULT_HEADER + [STACK]
749 824 
750 MD5_COMPARE_RESULT_HEADER_STACK = MD5_COMPARE_RESULT_HEADER + [STACK]825 MD5_COMPARE_RESULT_HEADER_STACK = MD5_COMPARE_RESULT_HEADER + [STACK]
751 826 
752- HEAD_OF_COMPARE_MODE = {827+ HEAD_OF_COMPARE_MODE: LazyDict[str, list[str]] = LazyDict(
753- Const.ALL: COMPARE_RESULT_HEADER,828+ {
754- Const.SUMMARY: SUMMARY_COMPARE_RESULT_HEADER,829+ Const.ALL: lambda: CompareConst.COMPARE_RESULT_HEADER,
755- Const.MD5: MD5_COMPARE_RESULT_HEADER,830+ Const.SUMMARY: lambda: CompareConst.SUMMARY_COMPARE_RESULT_HEADER,
756- }831+ Const.MD5: lambda: CompareConst.MD5_COMPARE_RESULT_HEADER,
832+ }
833+ )
757 834 
758 # dtype match835 # dtype match
759 836 
@@ -1627,15 +1627,8 @@ class CalcStatsDiff:
1627 result_df.loc[~condition_no_bench, [CompareConst.RESULT, CompareConst.ERROR_MESSAGE]] = ''1627 result_df.loc[~condition_no_bench, [CompareConst.RESULT, CompareConst.ERROR_MESSAGE]] = ''
1628 result_df.loc[~condition_req_grad_consist, CompareConst.ERROR_MESSAGE] += 'Requires_grad inconsistent. '1628 result_df.loc[~condition_req_grad_consist, CompareConst.ERROR_MESSAGE] += 'Requires_grad inconsistent. '
1629 else:1629 else:
1630- fill_cols = [1630+ fill_cols = CompareConst.ALL_COMPARE_INDEX.copy()
1631- CompareConst.COSINE,1631+ fill_cols.append(CompareConst.ERROR_MESSAGE)
1632- CompareConst.EUC_DIST,
1633- CompareConst.MAX_ABS_ERR,
1634- CompareConst.MAX_RELATIVE_ERR,
1635- CompareConst.ONE_THOUSANDTH_ERR_RATIO,
1636- CompareConst.FIVE_THOUSANDTHS_ERR_RATIO,
1637- CompareConst.ERROR_MESSAGE,
1638- ]
1639 result_df.loc[~condition_no_bench, fill_cols] = (1632 result_df.loc[~condition_no_bench, fill_cols] = (
1640 '' # 默认填充'', df默认省缺值为nan,不便后续处理,容易出现意外情况1633 '' # 默认填充'', df默认省缺值为nan,不便后续处理,容易出现意外情况
1641 )1634 )
@@ -0,0 +1,247 @@
1+# -------------------------------------------------------------------------
2+# This file is part of the MindStudio project.
3+# Copyright (c) 2025 Huawei Technologies Co.,Ltd.
4+#
5+# MindStudio is licensed under Mulan PSL v2.
6+# You can use this software according to the terms and conditions of the Mulan PSL v2.
7+# You may obtain a copy of Mulan PSL v2 at:
8+#
9+# http://license.coscl.org.cn/MulanPSL2
10+#
11+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
12+# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
13+# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
14+# See the Mulan PSL v2 for more details.
15+# -------------------------------------------------------------------------
16+ 
17+"""
18+Function:
19+AlgorithmScheduler class.
20+Manages builtin and custom compare algorithms.
21+"""
22+ 
23+import importlib
24+import inspect
25+import os
26+import re
27+import sys
28+import threading
29+import numpy as np
30+import torch
31+from typing import Any, Dict, List
32+from msprobe.core.common.log import logger
33+ 
34+ 
35+class AlgorithmScheduler:
36+ """
L

【review】AlgorithmScheduler 作为本 PR 的核心新增组件(225 行),缺少独立的单元测试

  • 问题:当前测试仅覆盖了 prepare_tensor、get_relative_err 等工具函数和各 builtin 算法的 compare 函数,但 AlgorithmScheduler 类本身(单例、模块加载、列名去重校验、返回值校验、自定义算法注册、异常回退路径)没有任何测试覆盖
  • 风险:_validate_column_names 重复列名检测、_validate_return_value 对非法返回值的拦截、_call_algorithm 异常时返回 "unsupported"、_get_module 导入失败等关键路径未经验证,回归风险高
  • 改进方案:新增 test_algorithm_scheduler.py,至少覆盖:
    1. get_instance 单例行为(多次调用返回同一实例)
    2. builtin 算法自动发现和 column_name 去重校验
    3. _validate_return_value 对非法返回值(非 tuple、元素数不对、类型不符)的拦截
    4. _call_algorithm 异常时返回 "unsupported" 的行为
    5. 自定义算法目录加载(含空目录、非法文件名跳过场景)
likedislike
37+ The class for algorithm manager
38+ Singleton: please use AlgorithmScheduler.get_instance() to get instance, do not call constructor directly
39+ """
40+ 
41+ ALGORITHM_PATH = "msprobe.core.compare.algorithm"
42+ BUILT_IN_ALGORITHM_DIR_NAME = "builtin_algorithm"
43+ CUSTOM_ALGORITHM_DIR_NAME = "custom_algorithm"
44+ ALGORITHM_FILE_NAME_PATTERN = r"^alg_([a-z0-9_]+)\.py$"
45+ COMPARE_FUNC_NAME = "compare"
46+ COLUMN_NAME_FUNC_NAME = "column_name"
47+ ARG_COUNT = 2 # compare方法入参的个数
48+ RESULT_COUNT = 2 # compare方法返回值的个数
49+ 
50+ _instance = None
51+ _lock = threading.Lock()
52+ 
53+ @classmethod
54+ def get_instance(cls):
55+ """Thread‑safe lazy singleton entry, only this way to get instance"""
56+ if cls._instance is None:
57+ with cls._lock:
58+ # double‑check lock
59+ if cls._instance is None:
60+ instance = super().__new__(cls)
61+ instance.__init__() # pylint: disable=unnecessary-dunder-call
62+ cls._instance = instance
63+ return cls._instance
64+ 
65+ def __new__(cls):
66+ raise RuntimeError("Do not instantiate AlgorithmScheduler directly, use AlgorithmScheduler.get_instance()")
67+ 
68+ def __init__(self):
69+ # only invoked inside get_instance
70+ if os.path.dirname(__file__) not in sys.path:
71+ sys.path.append(os.path.dirname(__file__))
72+ self.build_in_support_algorithm: List[str] = []
L

【review】sys.path.append 污染全局模块搜索路径,存在模块名冲突风险

  • 问题:将 os.path.dirname(__file__) 追加到 sys.path 后,importlib.import_module("builtin_algorithm.alg_xxx") 依赖此全局副作用。若运行环境中存在其他同名顶层包 builtin_algorithm 或 custom_algorithm,会导入到错误的模块
  • 改进方案:使用绝对导入路径,无需修改 sys.path:
# _get_module 中改为:
if algorithm_name in self.custom_support_algorithm:
    module_name = f"msprobe.core.compare.algorithm.custom_algorithm.alg_{algorithm_name}"
elif algorithm_name in self.build_in_support_algorithm:
    module_name = f"msprobe.core.compare.algorithm.builtin_algorithm.alg_{algorithm_name}"

这样与项目其他模块的导入方式保持一致,且避免全局副作用

likedislike
73+ self.custom_support_algorithm: List[str] = []
74+ # 为了校验column_name唯一性,初始化全量加载
75+ self._module_cache: Dict[str, Any] = {}
76+ self._column_name_algorithm_mapping: Dict[str, str] = {} # 算法名 -> 列名
77+ self._make_support_algorithm()
78+ self.algorithm_names = self.build_in_support_algorithm + self.custom_support_algorithm
79+ self._validate_column_names()
80+ self.algorithm_column_names = list(self._column_name_algorithm_mapping.keys())
L

【review】init 中 _validate_column_names 会触发全部算法模块的导入,与 _algorithm_cache 的懒加载设计意图矛盾

  • 问题:_validate_column_names 遍历所有算法并调用 _get_module,后者执行 importlib.import_module 并写入 _algorithm_cache。初始化完成后缓存已全部填充,后续 _call_algorithm 不会再触发懒加载
  • 影响:若 custom_algorithm 目录存在大量自定义算法,单例初始化耗时增加;且 _algorithm_cache 命名暗示"按需加载",实际为 eager 加载,语义不符
  • 改进方案(二选一):
    1. 若需保持懒加载:将 column_name 校验改为约定式(如 column_name == 文件名去 alg_ 前缀),延迟到 _call_algorithm 时才导入模块
    2. 若 eager 加载可接受:将 _algorithm_cache 重命名为 _module_cache 并补充注释说明初始化时全量加载,避免后续维护者误解
likedislike
81+ 
82+ def _make_support_algorithm(self) -> None:
83+ buildin_dir = os.path.join(os.path.dirname(__file__), self.BUILT_IN_ALGORITHM_DIR_NAME)
84+ self._make_support_algorithm_from_dir(buildin_dir, self.build_in_support_algorithm)
85+ custom_dir = os.path.join(os.path.dirname(__file__), self.CUSTOM_ALGORITHM_DIR_NAME)
86+ self._make_support_algorithm_from_dir(custom_dir, self.custom_support_algorithm)
87+ 
88+ def _make_support_algorithm_from_dir(self, dir_path: str, algorithm_list: List[str]) -> None:
89+ if not os.path.exists(dir_path):
90+ logger.warning(f"The algorithm directory {dir_path} does not exist, skip.")
91+ return
92+ 
93+ for file_name in os.listdir(dir_path):
94+ file_path = os.path.join(dir_path, file_name)
95+ self._add_algorithm_file_to_list(file_path, algorithm_list)
96+ 
97+ def _validate_column_names(self) -> None:
98+ all_algorithms = self.get_algorithm_names()
99+ for algo_name in all_algorithms:
100+ module = self._get_module(algo_name)
101+ col_name = self._get_column_name(algo_name, module)
102+ if col_name in self._column_name_algorithm_mapping:
103+ raise ValueError(
104+ f"The column name {col_name} is duplicated for algorithm {algo_name} "
105+ f"and {self._column_name_algorithm_mapping[col_name]}."
106+ )
107+ self._column_name_algorithm_mapping[col_name] = algo_name
108+ for col_name, algo_name in self._column_name_algorithm_mapping.items():
109+ logger.info(f"Load algorithm [{algo_name}] success, column: {col_name}")
110+ 
111+ @staticmethod
112+ def _add_algorithm_file_to_list(file_path: str, algorithm_list: List[str]) -> bool:
113+ if not os.path.isfile(file_path):
114+ return False
115+ file_name = os.path.basename(file_path)
116+ match_result = re.match(AlgorithmScheduler.ALGORITHM_FILE_NAME_PATTERN, file_name)
117+ if not match_result:
118+ logger.warning(
119+ f"The algorithm file {file_name} does not match the pattern "
120+ f"{AlgorithmScheduler.ALGORITHM_FILE_NAME_PATTERN}, skip."
121+ )
122+ return False
123+ algorithm_list.append(match_result.group(1))
124+ return True
125+ 
126+ def _get_module(self, algorithm_name: str) -> Any:
127+ if algorithm_name in self._module_cache:
128+ return self._module_cache[algorithm_name]
129+ if algorithm_name in self.custom_support_algorithm:
130+ module_name = f"{self.ALGORITHM_PATH}.{self.CUSTOM_ALGORITHM_DIR_NAME}.alg_{algorithm_name}"
131+ algorithm_module = importlib.import_module(module_name)
132+ elif algorithm_name in self.build_in_support_algorithm:
133+ module_name = f"{self.ALGORITHM_PATH}.{self.BUILT_IN_ALGORITHM_DIR_NAME}.alg_{algorithm_name}"
134+ algorithm_module = importlib.import_module(module_name)
135+ else:
136+ raise ValueError(f"Algorithm {algorithm_name} is not supported.")
137+ 
138+ if not hasattr(algorithm_module, self.COMPARE_FUNC_NAME):
139+ raise AttributeError(f"Algorithm {algorithm_name} must define a {self.COMPARE_FUNC_NAME} function.")
140+ 
141+ compare_func = getattr(algorithm_module, self.COMPARE_FUNC_NAME)
142+ if not callable(compare_func):
143+ raise TypeError(f"{self.COMPARE_FUNC_NAME} in {algorithm_name} is not a callable function.")
144+ 
145+ sig = inspect.signature(compare_func)
146+ required_params = [
147+ p
148+ for p in sig.parameters.values()
149+ if p.default is inspect.Parameter.empty
150+ and p.kind in (inspect.Parameter.POSITIONAL_ONLY, inspect.Parameter.POSITIONAL_OR_KEYWORD)
151+ ]
152+ if len(required_params) != self.ARG_COUNT:
153+ raise ValueError(
L

【review】使用 co_argcount 校验入参数量,无法正确处理带默认参数、*args 或装饰器的 compare 函数

  • 问题:compare_func.__code__.co_argcount 对 def compare(a, b, c=1) 返回 3(会被拦截),但对 def compare(a, b, **kwargs) 返回 2(会通过,可能不符合预期);对被装饰器包装的 compare 函数,co_argcount 反映的是 wrapper 的代码对象而非原始函数,可能导致误判
  • 改进方案:使用 inspect.signature 进行更健壮的校验:
import inspect
sig = inspect.signature(compare_func)
required_params = [
    p for p in sig.parameters.values()
    if p.default is inspect.Parameter.empty
    and p.kind in (inspect.Parameter.POSITIONAL_ONLY, inspect.Parameter.POSITIONAL_OR_KEYWORD)
]
if len(required_params) != self.ARG_COUNT:
    raise ValueError(f"{self.COMPARE_FUNC_NAME} in {algorithm_name} must have {self.ARG_COUNT} positional arguments.")
likedislike
154+ f"{self.COMPARE_FUNC_NAME} in {algorithm_name} must have {self.ARG_COUNT} positional arguments."
155+ )
156+ 
157+ self._module_cache[algorithm_name] = algorithm_module
158+ return algorithm_module
159+ 
160+ def _get_column_name(self, algorithm_name: str, algorithm_module: Any) -> str:
161+ if not hasattr(algorithm_module, self.COLUMN_NAME_FUNC_NAME):
162+ raise AttributeError(f"Algorithm {algorithm_name} must define a {self.COLUMN_NAME_FUNC_NAME} function.")
163+ column_name_func = getattr(algorithm_module, self.COLUMN_NAME_FUNC_NAME)
164+ if not callable(column_name_func):
165+ raise TypeError(f"{self.COLUMN_NAME_FUNC_NAME} in {algorithm_name} is not a callable function.")
166+ try:
167+ column_name = column_name_func()
168+ except Exception as e:
169+ raise ValueError(f"Error occurred when calling {self.COLUMN_NAME_FUNC_NAME} in {algorithm_name}: {e}")
170+ if not isinstance(column_name, str) or not column_name.strip():
171+ raise ValueError(f"{self.COLUMN_NAME_FUNC_NAME} in {algorithm_name} must return a non‑empty string.")
172+ return column_name
173+ 
174+ def _validate_return_value(self, value: Any, algorithm_name: str) -> bool:
175+ """
176+ 校验自定义比对算法返回值格式
177+ 要求返回二元组: (result_value, message)
178+ result_value: int | float | str
179+ message: str
180+ """
181+ if not isinstance(value, tuple) or len(value) != self.RESULT_COUNT:
182+ raise ValueError(
183+ f"Algorithm [{algorithm_name}] return value invalid. "
184+ f"Expected {self.RESULT_COUNT}‑element tuple, got type={type(value).__name__}."
185+ )
186+ 
187+ ret_val, ret_msg = value
188+ 
189+ if not self._is_real_number(ret_val) and not isinstance(ret_val, str):
190+ raise ValueError(
191+ f"Algorithm [{algorithm_name}] first return value type invalid. "
192+ f"Expected int/float/str, got {type(ret_val).__name__}"
193+ )
194+ 
195+ if not isinstance(ret_msg, str):
196+ raise ValueError(
197+ f"Algorithm [{algorithm_name}] second return value type invalid. "
198+ f"Expected str(message), got {type(ret_msg).__name__}"
199+ )
200+ 
201+ return True
202+ 
203+ def _is_real_number(self, value) -> bool:
204+ return isinstance(value, (int, float)) and not isinstance(value, bool)
205+ 
206+ def _call_algorithm(self, algorithm_name, n_value, b_value):
207+ if algorithm_name not in self.build_in_support_algorithm:
208+ if isinstance(n_value, np.ndarray):
209+ n_value = torch.from_numpy(n_value)
210+ if isinstance(b_value, np.ndarray):
211+ b_value = torch.from_numpy(b_value)
212+ with torch.no_grad():
213+ return self._call_compare(algorithm_name, n_value, b_value)
214+ return self._call_compare(algorithm_name, n_value, b_value)
215+ 
216+ def _call_compare(self, algorithm_name, n_value, b_value):
217+ try:
218+ algorithm_module = self._get_module(algorithm_name)
219+ compare_func = getattr(algorithm_module, self.COMPARE_FUNC_NAME)
220+ result = compare_func(n_value, b_value)
221+ self._validate_return_value(result, algorithm_name)
222+ return result
223+ except Exception as e:
224+ logger.error(f"Call algorithm [{algorithm_name}] failed, error: {e}")
225+ return "unsupported", str(e)
226+ 
L

【review】_call_algorithm 捕获所有 Exception 并统一返回 "unsupported",会掩盖算法实现中的真实 bug

  • 问题:算法内部任何异常(包括 AttributeError、TypeError、逻辑 bug)都会被吞掉,调用方只看到 "unsupported" 和错误消息字符串。虽然 logger.error 记录了原始异常,但在多进程场景下日志容易被淹没,排查困难
  • 改进方案:区分"算法不支持当前数据类型"(预期异常)和"算法执行错误"(非预期异常):
except (ValueError, TypeError) as e:
    # 数据类型不支持等预期异常
    logger.warning(f"Algorithm [{algorithm_name}] unsupported for this input: {e}")
    return "unsupported", str(e)
except Exception as e:
    # 非预期异常不应被吞掉,避免将代码 bug 误判为"不支持"
    logger.error(f"Algorithm [{algorithm_name}] internal error: {e}", exc_info=True)
    raise

这样在保持容错的同时,让真正的实现 bug 能被及时发现

likedislike
227+ def compare(self, n_value, b_value, column_names=None):
228+ if not column_names:
229+ column_names = self.algorithm_column_names
230+ results = []
231+ error_msgs = []
232+ for column_name in column_names:
233+ if column_name in self._column_name_algorithm_mapping:
234+ result, error_msg = self._call_algorithm(
235+ self._column_name_algorithm_mapping[column_name], n_value, b_value
236+ )
237+ else:
238+ result, error_msg = "unsupported", f"No available algorithm for {column_name}"
239+ results.append(result)
240+ error_msgs.append(error_msg if error_msg else "")
241+ return results, error_msgs
242+ 
243+ def get_algorithm_names(self) -> List[str]:
244+ return self.algorithm_names
245+ 
246+ def get_algorithm_column_names(self) -> List[str]:
247+ return self.algorithm_column_names
@@ -0,0 +1,50 @@
1+# -------------------------------------------------------------------------
2+# This file is part of the MindStudio project.
3+# Copyright (c) 2025 Huawei Technologies Co.,Ltd.
4+#
5+# MindStudio is licensed under Mulan PSL v2.
6+# You can use this software according to the terms and conditions of the Mulan PSL v2.
7+# You may obtain a copy of Mulan PSL v2 at:
8+#
9+# http://license.coscl.org.cn/MulanPSL2
10+#
11+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
12+# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
13+# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
14+# See the Mulan PSL v2 for more details.
15+# -------------------------------------------------------------------------
16+ 
17+ 
18+import numpy as np
19+from msprobe.core.common.const import Const, CompareConst
20+from msprobe.core.common.utils import format_value
21+from msprobe.core.compare.utils import flatten_compare_numpy
22+ 
23+ 
24+def column_name() -> str:
25+ return CompareConst.COSINE
26+ 
27+ 
28+def compare(n_value: np.ndarray, b_value: np.ndarray):
29+ n_value, b_value, result, err = flatten_compare_numpy(n_value, b_value)
30+ if err:
31+ return result, err
32+ 
33+ num = np.dot(n_value, b_value)
34+ a_norm = np.linalg.norm(n_value)
35+ b_norm = np.linalg.norm(b_value)
36+ 
37+ if a_norm <= Const.FLOAT_EPSILON and b_norm <= Const.FLOAT_EPSILON:
38+ return 1.0, ""
39+ if a_norm <= Const.FLOAT_EPSILON:
40+ return CompareConst.NAN, "Cannot compare by Cosine Similarity, All the data is Zero in npu dump data."
41+ if b_norm <= Const.FLOAT_EPSILON:
42+ return CompareConst.NAN, "Cannot compare by Cosine Similarity, All the data is Zero in Bench dump data."
43+ 
44+ cos = num / (a_norm * b_norm)
45+ if np.isnan(cos):
46+ return CompareConst.NAN, "Cannot compare by Cosine Similarity, the dump data has NaN."
47+ result = format_value(cos)
48+ if result > CompareConst.COSINE_THRESHOLD:
49+ result = round(result, 6)
50+ return result, ""
@@ -0,0 +1,33 @@
1+# -------------------------------------------------------------------------
2+# This file is part of the MindStudio project.
3+# Copyright (c) 2025 Huawei Technologies Co.,Ltd.
4+#
5+# MindStudio is licensed under Mulan PSL v2.
6+# You can use this software according to the terms and conditions of the Mulan PSL v2.
7+# You may obtain a copy of Mulan PSL v2 at:
8+#
9+# http://license.coscl.org.cn/MulanPSL2
10+#
11+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
12+# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
13+# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
14+# See the Mulan PSL v2 for more details.
15+# -------------------------------------------------------------------------
16+ 
17+ 
18+import numpy as np
19+from msprobe.core.common.const import CompareConst
20+from msprobe.core.compare.utils import flatten_compare_numpy
21+ 
22+ 
23+def column_name() -> str:
24+ return CompareConst.EUC_DIST
25+ 
26+ 
27+def compare(n_value: np.ndarray, b_value: np.ndarray):
28+ n_value, b_value, result, err = flatten_compare_numpy(n_value, b_value)
29+ if err:
30+ return result, err
31+ 
32+ distance = np.linalg.norm(n_value - b_value, ord=2)
33+ return distance, ""
@@ -0,0 +1,31 @@
1+# -------------------------------------------------------------------------
2+# This file is part of the MindStudio project.
3+# Copyright (c) 2025 Huawei Technologies Co.,Ltd.
4+#
5+# MindStudio is licensed under Mulan PSL v2.
6+# You can use this software according to the terms and conditions of the Mulan PSL v2.
7+# You may obtain a copy of Mulan PSL v2 at:
8+#
9+# http://license.coscl.org.cn/MulanPSL2
10+#
11+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
12+# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
13+# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
14+# See the Mulan PSL v2 for more details.
15+# -------------------------------------------------------------------------
16+ 
17+ 
18+import numpy as np
19+from msprobe.core.common.const import CompareConst
20+from msprobe.core.compare.utils import validate_compare_numpy, get_relative_err, calc_err_ratio
21+ 
22+ 
23+def column_name() -> str:
24+ return CompareConst.FIVE_THOUSANDTHS_ERR_RATIO
25+ 
26+ 
27+def compare(n_value: np.ndarray, b_value: np.ndarray):
28+ result, err = validate_compare_numpy(n_value, b_value)
29+ if err:
30+ return result, err
31+ return calc_err_ratio(get_relative_err(n_value, b_value), CompareConst.FIVE_THOUSAND_RATIO_THRESHOLD), ""
@@ -0,0 +1,34 @@
1+# -------------------------------------------------------------------------
2+# This file is part of the MindStudio project.
3+# Copyright (c) 2025 Huawei Technologies Co.,Ltd.
4+#
5+# MindStudio is licensed under Mulan PSL v2.
6+# You can use this software according to the terms and conditions of the Mulan PSL v2.
7+# You may obtain a copy of Mulan PSL v2 at:
8+#
9+# http://license.coscl.org.cn/MulanPSL2
10+#
11+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
12+# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
13+# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
14+# See the Mulan PSL v2 for more details.
15+# -------------------------------------------------------------------------
16+ 
17+ 
18+import numpy as np
19+from msprobe.core.common.const import CompareConst
20+from msprobe.core.common.utils import format_value
21+ 
22+ 
23+def column_name() -> str:
24+ return CompareConst.MAX_ABS_ERR
25+ 
26+ 
27+def compare(n_value: np.ndarray, b_value: np.ndarray):
28+ n_value = n_value.reshape(-1).astype(float)
29+ b_value = b_value.reshape(-1).astype(float)
30+ 
31+ max_value = np.max(np.abs(n_value - b_value))
32+ if np.isnan(max_value):
33+ return CompareConst.NAN, "Cannot compare by MaxAbsError, the data contains nan/inf/-inf."
34+ return format_value(max_value), ""
@@ -0,0 +1,35 @@
1+# -------------------------------------------------------------------------
2+# This file is part of the MindStudio project.
3+# Copyright (c) 2025 Huawei Technologies Co.,Ltd.
4+#
5+# MindStudio is licensed under Mulan PSL v2.
6+# You can use this software according to the terms and conditions of the Mulan PSL v2.
7+# You may obtain a copy of Mulan PSL v2 at:
8+#
9+# http://license.coscl.org.cn/MulanPSL2
10+#
11+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
12+# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
13+# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
14+# See the Mulan PSL v2 for more details.
15+# -------------------------------------------------------------------------
16+ 
17+ 
18+import numpy as np
19+from msprobe.core.common.const import CompareConst
20+from msprobe.core.compare.utils import get_relative_err
21+from msprobe.core.common.utils import format_value
22+ 
23+ 
24+def column_name() -> str:
25+ return CompareConst.MAX_RELATIVE_ERR
26+ 
27+ 
28+def compare(n_value: np.ndarray, b_value: np.ndarray):
29+ max_relative_err = np.max(np.abs(get_relative_err(n_value, b_value)))
30+ if np.isnan(max_relative_err):
31+ return (
32+ CompareConst.NAN,
33+ "Cannot compare by MaxRelativeError, the data contains nan/inf/-inf.",
34+ )
35+ return format_value(max_relative_err), ""
@@ -0,0 +1,31 @@
1+# -------------------------------------------------------------------------
2+# This file is part of the MindStudio project.
3+# Copyright (c) 2025 Huawei Technologies Co.,Ltd.
4+#
5+# MindStudio is licensed under Mulan PSL v2.
6+# You can use this software according to the terms and conditions of the Mulan PSL v2.
7+# You may obtain a copy of Mulan PSL v2 at:
8+#
9+# http://license.coscl.org.cn/MulanPSL2
10+#
11+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
12+# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
13+# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
14+# See the Mulan PSL v2 for more details.
15+# -------------------------------------------------------------------------
16+ 
17+ 
18+import numpy as np
19+from msprobe.core.common.const import CompareConst
20+from msprobe.core.compare.utils import validate_compare_numpy, get_relative_err, calc_err_ratio
21+ 
22+ 
23+def column_name() -> str:
24+ return CompareConst.ONE_THOUSANDTH_ERR_RATIO
25+ 
26+ 
27+def compare(n_value: np.ndarray, b_value: np.ndarray):
28+ result, err = validate_compare_numpy(n_value, b_value)
29+ if err:
30+ return result, err
31+ return calc_err_ratio(get_relative_err(n_value, b_value), CompareConst.THOUSAND_RATIO_THRESHOLD), ""
@@ -17,7 +17,7 @@
17import multiprocessing17import multiprocessing
18from dataclasses import dataclass18from dataclasses import dataclass
19from functools import partial19from functools import partial
20-from typing import Any20+from typing import Any, Dict, List
21 21 
22import pandas as pd22import pandas as pd
23from tqdm import tqdm23from tqdm import tqdm
@@ -34,13 +34,13 @@ from msprobe.core.compare.tensor_postprocess.processor import TensorPostprocessM
34 34 
35@dataclass35@dataclass
36class ComparisonResult:36class ComparisonResult:
37- cos_result: list37+ results: Dict[str, List[str]] # 列名 -> 结果列表
38- euc_dist_result: list38+ err_msgs: List[str] # 错误信息列表
39- max_err_result: list39+ 
40- max_relative_err_result: list40+ @classmethod
41- one_thousand_err_ratio_result: list41+ def create_empty(cls, column_names: List[str], size: int) -> "ComparisonResult":
42- five_thousand_err_ratio_result: list42+ results = {name: [""] * size for name in column_names}
43- err_msgs: list43+ return cls(results=results, err_msgs=[""] * size)
44 44 
45 45 
46@dataclass46@dataclass
@@ -151,19 +151,19 @@ class CompareRealData:
151 151 
152 lock.acquire()152 lock.acquire()
153 try:153 try:
154- for i, cos_item in enumerate(result.cos_result):154+ slice_len = len(next(iter(result.results.values()), []))
155- process_index = i + offset155+ slice_loc = slice(offset, offset + slice_len)
156- result_df.loc[process_index, CompareConst.COSINE] = cos_item156+ 
157- result_df.loc[process_index, CompareConst.EUC_DIST] = result.euc_dist_result[i]157+ # 批量赋值算法列
158- result_df.loc[process_index, CompareConst.MAX_ABS_ERR] = result.max_err_result[i]158+ for col_name, algo_results in result.results.items():
159- result_df.loc[process_index, CompareConst.MAX_RELATIVE_ERR] = result.max_relative_err_result[i]159+ result_df.loc[slice_loc, col_name] = algo_results
160- result_df.loc[process_index, CompareConst.ONE_THOUSANDTH_ERR_RATIO] = (160+ 
161- result.one_thousand_err_ratio_result161+ # 错误消息处理
162- )[i]162+ err_len = len(result.err_msgs)
163- result_df.loc[process_index, CompareConst.FIVE_THOUSANDTHS_ERR_RATIO] = (163+ err_slice = slice(offset, offset + err_len)
164- result.five_thousand_err_ratio_result164+ 
165- )[i]165+ target_index = result_df.loc[err_slice].index
166- result_df.loc[process_index, CompareConst.ERROR_MESSAGE] += result.err_msgs[i]166+ result_df.loc[err_slice, CompareConst.ERROR_MESSAGE] += pd.Series(result.err_msgs, index=target_index)
167 return result_df167 return result_df
168 except ValueError as e:168 except ValueError as e:
169 logger.error('result dataframe is not found.')169 logger.error('result dataframe is not found.')
@@ -288,7 +288,13 @@ class CompareRealData:
288 构建最终的对比结果288 构建最终的对比结果
289 负责调用指标计算函数,并补充模糊匹配提示信息。289 负责调用指标计算函数,并补充模糊匹配提示信息。
290 """290 """
291- result_list, err_msg = compare_ops_apply(result.n_value, result.b_value, result.error_flag, result.err_msg)291+ result_list, err_msg = compare_ops_apply(
292+ result.n_value,
293+ result.b_value,
294+ result.error_flag,
295+ result.err_msg,
296+ column_names=CompareConst.ALL_COMPARE_INDEX,
297+ )
292 npu_op_name = npu_info.op_name298 npu_op_name = npu_info.op_name
293 bench_op_name = bench_info.op_name299 bench_op_name = bench_info.op_name
294 if self.mode_config.fuzzy_match and npu_op_name != bench_op_name and bench_op_name != CompareConst.N_A:300 if self.mode_config.fuzzy_match and npu_op_name != bench_op_name and bench_op_name != CompareConst.N_A:
@@ -297,13 +303,9 @@ class CompareRealData:
297 return result_list303 return result_list
298 304 
299 def compare_ops(self, idx, dump_path_dict, result_df, lock, input_param):305 def compare_ops(self, idx, dump_path_dict, result_df, lock, input_param):
300- cos_result = []306+ columns_names = CompareConst.ALL_COMPARE_INDEX
301- euc_dist_result = []307+ size = len(result_df)
302- max_err_result = []308+ comparison_result = ComparisonResult.create_empty(columns_names, size)
303- max_relative_err_result = []
304- one_thousand_err_ratio_result = []
305- five_thousand_err_ratio_result = []
306- err_mess = []
307 309 
308 is_print_compare_log = input_param.get("is_print_compare_log")310 is_print_compare_log = input_param.get("is_print_compare_log")
309 311 
@@ -326,47 +328,25 @@ class CompareRealData:
326 if is_print_compare_log:328 if is_print_compare_log:
327 logger.info("start compare: {}".format(npu_op_name))329 logger.info("start compare: {}".format(npu_op_name))
328 330 
329- (331+ compare_result = self.compare_by_op(npu_info, bench_info, input_param)
330- cos_sim,332+ err_msg = compare_result[-1]
331- euc_dist,
332- max_abs_err,
333- max_relative_err,
334- one_thousand_err_ratio,
335- five_thousand_err_ratio,
336- err_msg,
337- ) = self.compare_by_op(npu_info, bench_info, input_param)
338- 
339 if is_print_compare_log:333 if is_print_compare_log:
340 if "does not have data file" in err_msg:334 if "does not have data file" in err_msg:
341 logger.info(f"[{npu_op_name}] Compare result: {err_msg} ")335 logger.info(f"[{npu_op_name}] Compare result: {err_msg} ")
342 elif "Bench api/module unmatched" in err_msg:336 elif "Bench api/module unmatched" in err_msg:
343 logger.info(f"[{npu_op_name}] Compare result: {err_msg} ")337 logger.info(f"[{npu_op_name}] Compare result: {err_msg} ")
344 else:338 else:
345- logger.info(339+ log_parts = []
346- f"[{npu_op_name}] Compare result: cosine {cos_sim}, euc_dist {euc_dist}, "340+ for column_idx, column_name in enumerate(columns_names):
347- f"max_abs_err {max_abs_err}, max_relative_err {max_relative_err}, "341+ log_parts.append(f"{column_name} {compare_result[column_idx]}")
348- f"one_thousand_err_ratio {one_thousand_err_ratio}, "342+ log_parts.append(err_msg)
349- f"five_thousand_err_ratio {five_thousand_err_ratio}, {err_msg}"343+ logger.info(f"[{npu_op_name}] Compare result: {', '.join(log_parts)}")
350- )
351- cos_result.append(cos_sim)
352- euc_dist_result.append(euc_dist)
353- max_err_result.append(max_abs_err)
354- max_relative_err_result.append(max_relative_err)
355- one_thousand_err_ratio_result.append(one_thousand_err_ratio)
356- five_thousand_err_ratio_result.append(five_thousand_err_ratio)
357- err_mess.append(err_msg)
358 344 
359- cr = ComparisonResult(345+ for column_idx, column_name in enumerate(columns_names):
360- cos_result=cos_result,346+ comparison_result.results[column_name][i] = compare_result[column_idx]
361- euc_dist_result=euc_dist_result,347+ comparison_result.err_msgs[i] = err_msg
362- max_err_result=max_err_result,
363- max_relative_err_result=max_relative_err_result,
364- one_thousand_err_ratio_result=one_thousand_err_ratio_result,
365- five_thousand_err_ratio_result=five_thousand_err_ratio_result,
366- err_msgs=err_mess,
367- )
368 348 
369- return self._save_cmp_result(idx, cr, result_df, lock)349+ return self._save_cmp_result(idx, comparison_result, result_df, lock)
370 350 
371 def do_multi_process(self, input_param, result_df):351 def do_multi_process(self, input_param, result_df):
372 try:352 try:
@@ -15,15 +15,15 @@
15# -------------------------------------------------------------------------15# -------------------------------------------------------------------------
16 16 
17 17 
18-import abc
19from dataclasses import dataclass18from dataclasses import dataclass
20from typing import Any19from typing import Any
21 20 
22import numpy as np21import numpy as np
23 22 
24-from msprobe.core.common.const import Const, CompareConst23+from msprobe.core.common.const import CompareConst
25from msprobe.core.common.log import logger24from msprobe.core.common.log import logger
26-from msprobe.core.common.utils import CompareException, format_value25+from msprobe.core.common.utils import CompareException
26+from msprobe.core.compare.algorithm.algorithm_scheduler import AlgorithmScheduler
27 27 
28 28 
29@dataclass29@dataclass
@@ -71,19 +71,6 @@ def handle_inf_nan(n_value, b_value):
71 return n_value, b_value71 return n_value, b_value
72 72 
73 73 
74-def reshape_value(n_value, b_value):
75- """返回reshape后的数据"""
76- if not n_value.shape: # 判断数据是否为0维tensor, 如果0维tensor,不会转成1维tensor,直接返回
77- if n_value.dtype == bool:
78- n_value = n_value.astype(float)
79- b_value = b_value.astype(float)
80- return n_value, b_value
81- 
82- n_value = n_value.reshape(-1).astype(float) # 32转64为了防止某些数转dataframe时出现误差
83- b_value = b_value.reshape(-1).astype(float)
84- return n_value, b_value
85- 
86- 
87def npy_data_check(n_value, b_value):74def npy_data_check(n_value, b_value):
88 error_message = ""75 error_message = ""
89 if not isinstance(n_value, np.ndarray) or not isinstance(b_value, np.ndarray):76 if not isinstance(n_value, np.ndarray) or not isinstance(b_value, np.ndarray):
@@ -138,137 +125,6 @@ def statistics_data_check(result_dict):
138 return error_flag, error_message125 return error_flag, error_message
139 126 
140 127 
141-class TensorComparisonBasic(abc.ABC):
142- """NPU和bench中npy数据的比较模板"""
143- 
144- @abc.abstractmethod
145- def apply(self, n_value, b_value, relative_err, err_msg):
146- raise NotImplementedError
147- 
148- 
149-def get_relative_err(n_value, b_value):
150- """计算相对误差"""
151- with np.errstate(divide='ignore', invalid='ignore'):
152- if n_value.dtype not in CompareConst.FLOAT_TYPE:
153- n_value = n_value.astype(float)
154- if b_value.dtype not in CompareConst.FLOAT_TYPE:
155- b_value = b_value.astype(float)
156- 
157- n_value_copy = n_value.copy()
158- b_value_copy = b_value.copy()
159- zero_mask = b_value_copy == 0
160- 
161- result_type = np.result_type(n_value_copy, b_value_copy, np.float32)
162- n = n_value_copy.astype(result_type)
163- b = b_value_copy.astype(result_type)
164- epsilon = np.finfo(result_type).eps
165- n[zero_mask] += epsilon
166- b[zero_mask] += epsilon
167- relative_err = np.abs(np.divide((n - b), b))
168- 
169- return relative_err
170- 
171- 
172-class GetCosineSimilarity(TensorComparisonBasic):
173- """计算cosine相似度"""
174- 
175- @staticmethod
176- def correct_data(result):
177- if result == CompareConst.NAN:
178- return result
179- if float(result) > CompareConst.COSINE_THRESHOLD:
180- return round(float(result), 6)
181- return result
182- 
183- def apply(self, n_value, b_value, relative_err, err_msg):
184- if "This is type of 0-d tensor" in err_msg:
185- return CompareConst.UNSUPPORTED, err_msg
186- 
187- with np.errstate(divide="ignore", invalid="ignore"):
188- if len(n_value) == 1:
189- return CompareConst.UNSUPPORTED, "This is a 1-d tensor of length 1."
190- num = n_value.dot(b_value)
191- a_norm = np.linalg.norm(n_value)
192- b_norm = np.linalg.norm(b_value)
193- 
194- if a_norm <= Const.FLOAT_EPSILON and b_norm <= Const.FLOAT_EPSILON:
195- return 1.0, ""
196- if a_norm <= Const.FLOAT_EPSILON:
197- return CompareConst.NAN, "Cannot compare by Cosine Similarity, All the data is Zero in npu dump data."
198- if b_norm <= Const.FLOAT_EPSILON:
199- return CompareConst.NAN, "Cannot compare by Cosine Similarity, All the data is Zero in Bench dump data."
200- 
201- cos = num / (a_norm * b_norm)
202- if np.isnan(cos):
203- return CompareConst.NAN, "Cannot compare by Cosine Similarity, the dump data has NaN."
204- result = format_value(cos)
205- result = self.correct_data(result)
206- return result, ""
207- 
208- 
209-class GetEuclideanDistance(TensorComparisonBasic):
210- """计算欧氏距离"""
211- 
212- def apply(self, n_value, b_value, relative_err, err_msg):
213- if "This is type of 0-d tensor" in err_msg:
214- return CompareConst.UNSUPPORTED, err_msg
215- 
216- distance = np.linalg.norm(n_value - b_value, ord=2)
217- 
218- return distance, ""
219- 
220- 
221-class GetMaxAbsErr(TensorComparisonBasic):
222- """计算最大绝对误差"""
223- 
224- def apply(self, n_value, b_value, relative_err, err_msg):
225- temp_res = n_value - b_value
226- max_value = np.max(np.abs(temp_res))
227- if np.isnan(max_value):
228- msg = "Cannot compare by MaxAbsError, the data contains nan/inf/-inf in dump data."
229- return CompareConst.NAN, msg
230- return format_value(max_value), ""
231- 
232- 
233-class GetMaxRelativeErr(TensorComparisonBasic):
234- """计算最大相对误差"""
235- 
236- def apply(self, n_value, b_value, relative_err, err_msg):
237- max_relative_err = np.max(np.abs(relative_err))
238- if np.isnan(max_relative_err):
239- msg = "Cannot compare by MaxRelativeError, the data contains nan/inf/-inf in dump data."
240- return CompareConst.NAN, msg
241- return format_value(max_relative_err), ""
242- 
243- 
244-class GetErrRatio(TensorComparisonBasic):
245- """计算相对误差小于指定阈值(千分之一、千分之五)的比例"""
246- 
247- def __init__(self, threshold):
248- self.threshold = threshold
249- 
250- def apply(self, n_value, b_value, relative_err, err_msg):
251- if "This is type of 0-d tensor" in err_msg:
252- return CompareConst.UNSUPPORTED, err_msg
253- 
254- if not np.size(relative_err):
255- return CompareConst.NAN, ""
256- 
257- ratio = np.sum(relative_err < self.threshold) / np.size(relative_err)
258- return format_value(ratio), ""
259- 
260- 
261-class CompareOps:
262- compare_ops = {
263- "cosine_similarity": GetCosineSimilarity(),
264- "euclidean_distance": GetEuclideanDistance(),
265- "max_abs_error": GetMaxAbsErr(),
266- "max_relative_error": GetMaxRelativeErr(),
267- "one_thousand_err_ratio": GetErrRatio(CompareConst.THOUSAND_RATIO_THRESHOLD),
268- "five_thousand_err_ratio": GetErrRatio(CompareConst.FIVE_THOUSAND_RATIO_THRESHOLD),
269- }
270- 
271- 
272def error_value_process(n_value):128def error_value_process(n_value):
273 if n_value in [129 if n_value in [
274 CompareConst.READ_NONE,130 CompareConst.READ_NONE,
@@ -285,22 +141,16 @@ def error_value_process(n_value):
285 return CompareConst.N_A, ""141 return CompareConst.N_A, ""
286 142 
287 143 
288-def compare_ops_apply(n_value, b_value, error_flag, err_msg):144+def compare_ops_apply(n_value, b_value, error_flag, err_msg, column_names=None):
289- result_list = []145+ if not column_names:
146+ column_names = CompareConst.BUILTIN_COMPARE_COLUMNS
290 if error_flag:147 if error_flag:
291 result, msg = error_value_process(n_value)148 result, msg = error_value_process(n_value)
292- result_list = [result] * len(CompareOps.compare_ops)149+ result_list = [result] * len(column_names)
293 err_msg += msg150 err_msg += msg
294 return result_list, err_msg151 return result_list, err_msg
295- 152+ results, err_msgs = AlgorithmScheduler.get_instance().compare(n_value, b_value, column_names=column_names)
296- relative_err = get_relative_err(n_value, b_value)153+ return results, err_msg + "".join(err_msgs)
297- n_value, b_value = reshape_value(n_value, b_value)
298- 
299- for op in CompareOps.compare_ops.values():
300- result, msg = op.apply(n_value, b_value, relative_err, err_msg)
301- result_list.append(result)
302- err_msg += msg
303- return result_list, err_msg
304 154 
305 155 
306class ValidateTensor:156class ValidateTensor:
@@ -17,8 +17,6 @@
17import numpy as np17import numpy as np
18 18 
19from msprobe.core.common.log import logger19from msprobe.core.common.log import logger
20-from msprobe.core.compare.npy_compare import CompareOps
21- 
22 20 
23 21 
24def in_different_shape(a, b):22def in_different_shape(a, b):
@@ -51,9 +49,9 @@ def cos_sim(a, b):
51 b_norm = np.linalg.norm(b)49 b_norm = np.linalg.norm(b)
52 50 
53 if a_norm == 0 and b_norm == 0:51 if a_norm == 0 and b_norm == 0:
54- return 1.52+ return 1.0
55 if a_norm == 0 or b_norm == 0:53 if a_norm == 0 or b_norm == 0:
56- logger.warning(f'One tensor norm is zero.')54+ logger.warning('One tensor norm is zero.')
57 return None55 return None
58 56 
59 sim = num / (a_norm * b_norm)57 sim = num / (a_norm * b_norm)
@@ -76,9 +74,4 @@ def shape(a, b):
76 return list(a.shape)74 return list(a.shape)
77 75 
78 76 
79-METRIC_FUNC = {77+METRIC_FUNC = {'l2': l2_distance, 'cos': cos_sim, 'numel': numel, 'shape': shape}
80- 'l2': l2_distance,
81- 'cos': cos_sim,
82- 'numel': numel,
83- 'shape': shape
84- }
@@ -88,7 +88,7 @@ def pt_compare(input_param, output_path, **kwargs):
88 'first_diff_analyze': config.first_diff_analyze,88 'first_diff_analyze': config.first_diff_analyze,
89 'compared_file_type': config.compared_file_type,89 'compared_file_type': config.compared_file_type,
90 'consistent_check': config.consistent_check,90 'consistent_check': config.consistent_check,
91- 'backend': config.backend91+ 'backend': config.backend,
92 }92 }
93 mode_config = ModeConfig(**config_dict)93 mode_config = ModeConfig(**config_dict)
94 mapping_config = MappingConfig(cell_mapping=config.cell_mapping, data_mapping=config.data_mapping)94 mapping_config = MappingConfig(cell_mapping=config.cell_mapping, data_mapping=config.data_mapping)
@@ -21,8 +21,7 @@ import torch
21 21 
22from msprobe.core.common.utils import logger, CompareException22from msprobe.core.common.utils import logger, CompareException
23from msprobe.core.common.file_utils import FileChecker, FileCheckConst23from msprobe.core.common.file_utils import FileChecker, FileCheckConst
24-from msprobe.pytorch.common.utils import load_pt24+from msprobe.pytorch.common.utils import load_pt, is_float8_tensor
25-from msprobe.pytorch.common.utils import is_float8_tensor, load_pt
26 25 
27 26 
28def _prepare_tensor_for_numpy(tensor):27def _prepare_tensor_for_numpy(tensor):
@@ -31,6 +30,7 @@ def _prepare_tensor_for_numpy(tensor):
31 return tensor.to(torch.float32)30 return tensor.to(torch.float32)
32 return tensor31 return tensor
33 32 
33+ 
34def read_pt_data(dir_path, file_name):34def read_pt_data(dir_path, file_name):
35 if not file_name:35 if not file_name:
36 return None36 return None
@@ -48,7 +48,7 @@ def read_pt_data(dir_path, file_name):
48 raise CompareException(CompareException.INVALID_FILE_ERROR) from e48 raise CompareException(CompareException.INVALID_FILE_ERROR) from e
49 except AttributeError as e:49 except AttributeError as e:
50 # 这里捕获 detach 方法抛出的异常50 # 这里捕获 detach 方法抛出的异常
51- logger.error(f"Failed to detach the loaded tensor.")51+ logger.error("Failed to detach the loaded tensor.")
52 raise CompareException(CompareException.DETACH_ERROR) from e52 raise CompareException(CompareException.DETACH_ERROR) from e
53 data_value = _prepare_tensor_for_numpy(data_value)53 data_value = _prepare_tensor_for_numpy(data_value)
54 data_value = data_value.numpy()54 data_value = data_value.numpy()
@@ -29,7 +29,7 @@ from msprobe.core.compare.utils import (
29)29)
30from msprobe.core.common.utils import set_dump_path, get_dump_mode30from msprobe.core.common.utils import set_dump_path, get_dump_mode
31from msprobe.visualization.utils import GraphConst31from msprobe.visualization.utils import GraphConst
32-from msprobe.core.common.const import Const, CompareConst32+from msprobe.core.common.const import LazyConst, Const, CompareConst
33from msprobe.core.compare.indicator_analysis.calculator import calculate_result33from msprobe.core.compare.indicator_analysis.calculator import calculate_result
34from msprobe.core.common.file_utils import FileChecker, FileCheckConst34from msprobe.core.common.file_utils import FileChecker, FileCheckConst
35from msprobe.core.common.log import BaseLogger, logger35from msprobe.core.common.log import BaseLogger, logger
@@ -240,7 +240,7 @@ class MatchedNodeCalculator:
240 功能:用户在前端选择NPU和Bench节点匹配,前端会查询db数据打包发送到此类,此类进行精度指标的计算,最终返回结果给前端240 功能:用户在前端选择NPU和Bench节点匹配,前端会查询db数据打包发送到此类,此类进行精度指标的计算,最终返回结果给前端
241 """241 """
242 242 
243- TENSOR_COMPARE_INDEX = CompareConst.ALL_COMPARE_INDEX + CompareConst.EXTRACT_INDEX243+ TENSOR_COMPARE_INDEX = LazyConst(lambda: CompareConst.ALL_COMPARE_INDEX + CompareConst.EXTRACT_INDEX)
244 STATISTICS_COMPARE_INDEX = CompareConst.SUMMARY_COMPARE_INDEX + CompareConst.EXTRACT_INDEX244 STATISTICS_COMPARE_INDEX = CompareConst.SUMMARY_COMPARE_INDEX + CompareConst.EXTRACT_INDEX
245 245 
246 _RESP_SUCCESS = "success"246 _RESP_SUCCESS = "success"
@@ -272,17 +272,10 @@ class MatchedNodeCalculator:
272 Const.REQ_GRAD: (CompareConst.NPU_REQ_GRAD, CompareConst.BENCH_REQ_GRAD),272 Const.REQ_GRAD: (CompareConst.NPU_REQ_GRAD, CompareConst.BENCH_REQ_GRAD),
273 Const.MD5: (CompareConst.NPU_MD5, CompareConst.BENCH_MD5),273 Const.MD5: (CompareConst.NPU_MD5, CompareConst.BENCH_MD5),
274 }274 }
275+ algorithm_columns = CompareConst.ALL_COMPARE_INDEX
276+ compare_result_header = CompareConst.COMPARE_RESULT_HEADER
275 self.tensor_indicators_index = {277 self.tensor_indicators_index = {
276- CompareConst.COSINE: CompareConst.COMPARE_RESULT_HEADER.index(CompareConst.COSINE),278+ col_name: compare_result_header.index(col_name) for col_name in algorithm_columns
277- CompareConst.EUC_DIST: CompareConst.COMPARE_RESULT_HEADER.index(CompareConst.EUC_DIST),
278- CompareConst.MAX_ABS_ERR: CompareConst.COMPARE_RESULT_HEADER.index(CompareConst.MAX_ABS_ERR),
279- CompareConst.MAX_RELATIVE_ERR: CompareConst.COMPARE_RESULT_HEADER.index(CompareConst.MAX_RELATIVE_ERR),
280- CompareConst.ONE_THOUSANDTH_ERR_RATIO: CompareConst.COMPARE_RESULT_HEADER.index(
281- CompareConst.ONE_THOUSANDTH_ERR_RATIO
282- ),
283- CompareConst.FIVE_THOUSANDTHS_ERR_RATIO: CompareConst.COMPARE_RESULT_HEADER.index(
284- CompareConst.FIVE_THOUSANDTHS_ERR_RATIO
285- ),
286 }279 }
287 self.extra_tensor_indicators_index = {280 self.extra_tensor_indicators_index = {
288 CompareConst.REQ_GRAD_CONSIST: CompareConst.COMPARE_RESULT_HEADER.index(CompareConst.REQ_GRAD_CONSIST),281 CompareConst.REQ_GRAD_CONSIST: CompareConst.COMPARE_RESULT_HEADER.index(CompareConst.REQ_GRAD_CONSIST),
@@ -17,7 +17,7 @@
17import math17import math
18import json18import json
19from msprobe.core.common.const import CompareConst, Const19from msprobe.core.common.const import CompareConst, Const
20-from msprobe.visualization.utils import ToolTip, GraphConst, str2float20+from msprobe.visualization.utils import ToolTip, GraphConst
21 21 
22 22 
23class ModeAdapter:23class ModeAdapter:
@@ -74,8 +74,7 @@ class ModeAdapter:
74 headers = CompareConst.COMPARE_RESULT_HEADER74 headers = CompareConst.COMPARE_RESULT_HEADER
75 ModeAdapter._add_compare_data(node.input_data, compare_data_dict, key_list, headers)75 ModeAdapter._add_compare_data(node.input_data, compare_data_dict, key_list, headers)
76 ModeAdapter._add_compare_data(node.output_data, compare_data_dict, key_list, headers)76 ModeAdapter._add_compare_data(node.output_data, compare_data_dict, key_list, headers)
77- precision_index = GraphConst.COMPARE_INDICATOR_TO_PRECISION_INDEX_MAPPING.get(api_indicator,77+ precision_index = GraphConst.COMPARE_INDICATOR_TO_PRECISION_INDEX_MAPPING.get(api_indicator, CompareConst.PASS)
78- CompareConst.PASS)
79 precision_index = self._ignore_precision_index(node.id, precision_index)78 precision_index = self._ignore_precision_index(node.id, precision_index)
80 return precision_index79 return precision_index
81 80 
@@ -102,16 +101,14 @@ class ModeAdapter:
102 CompareConst.MAX_DIFF: ToolTip.MAX_DIFF,101 CompareConst.MAX_DIFF: ToolTip.MAX_DIFF,
103 CompareConst.MIN_DIFF: ToolTip.MIN_DIFF,102 CompareConst.MIN_DIFF: ToolTip.MIN_DIFF,
104 CompareConst.MEAN_DIFF: ToolTip.MEAN_DIFF,103 CompareConst.MEAN_DIFF: ToolTip.MEAN_DIFF,
105- CompareConst.NORM_DIFF: ToolTip.NORM_DIFF}104+ CompareConst.NORM_DIFF: ToolTip.NORM_DIFF,
105+ }
106 elif self.compare_mode == GraphConst.MD5_COMPARE:106 elif self.compare_mode == GraphConst.MD5_COMPARE:
107 tips = {Const.MD5: ToolTip.MD5}107 tips = {Const.MD5: ToolTip.MD5}
108 else:108 else:
109- tips = {109+ # tips 前端已经不展示,全部设置为""
110- CompareConst.ONE_THOUSANDTH_ERR_RATIO: ToolTip.ONE_THOUSANDTH_ERR_RATIO,110+ algorithm_columns = CompareConst.ALL_COMPARE_INDEX
111- CompareConst.FIVE_THOUSANDTHS_ERR_RATIO: ToolTip.FIVE_THOUSANDTHS_ERR_RATIO,111+ tips = {col: "" for col in algorithm_columns}
112- CompareConst.COSINE: ToolTip.COSINE,
113- CompareConst.MAX_ABS_ERR: ToolTip.MAX_ABS_ERR,
114- CompareConst.MAX_RELATIVE_ERR: ToolTip.MAX_RELATIVE_ERR}
115 return json.dumps(tips)112 return json.dumps(tips)
116 113 
117 def _ignore_precision_index(self, node_id, precision_index):114 def _ignore_precision_index(self, node_id, precision_index):
@@ -17,10 +17,10 @@
17import os17import os
18import re18import re
19import json19import json
20-import pickle20+import pickle # nosec B403
21import time21import time
22from msprobe.core.common.file_utils import FileOpen, find_proc_dir22from msprobe.core.common.file_utils import FileOpen, find_proc_dir
23-from msprobe.core.common.const import CompareConst, Const23+from msprobe.core.common.const import LazyConst, CompareConst, Const
24from msprobe.core.common.log import logger24from msprobe.core.common.log import logger
25from msprobe.core.common.exceptions import MsprobeException25from msprobe.core.common.exceptions import MsprobeException
26from msprobe.core.compare.utils import check_and_return_dir_contents26from msprobe.core.compare.utils import check_and_return_dir_contents
@@ -117,9 +117,11 @@ def check_directory_content(input_path):
117 if step_all:117 if step_all:
118 return GraphConst.STEPS118 return GraphConst.STEPS
119 119 
120- raise ValueError("The input path content does not conform to the expected naming convention. "120+ raise ValueError(
121- "It is expected to be all step{number} named folders (such as step0), "121+ "The input path content does not conform to the expected naming convention. "
122- "all rank{number} named folders (such as rank0), or all files.")122+ "It is expected to be all step{number} named folders (such as step0), "
123+ "all rank{number} named folders (such as rank0), or all files."
124+ )
123 125 
124 126 
125def extract_rank_number(rank_str):127def extract_rank_number(rank_str):
@@ -141,34 +143,52 @@ def load_parallel_param(args):
141 raise MsprobeException(MsprobeException.INVALID_PARAM_ERROR)143 raise MsprobeException(MsprobeException.INVALID_PARAM_ERROR)
142 144 
143 if not args.golden_path and len(args.rank_size) != 1:145 if not args.golden_path and len(args.rank_size) != 1:
144- logger.error('In the graph merge build task, '146+ logger.error('In the graph merge build task, the number of parameters "rank_size" to be filled in is either 1!')
145- 'the number of parameters "rank_size" to be filled in is either 1!')
146 raise MsprobeException(MsprobeException.INVALID_PARAM_ERROR)147 raise MsprobeException(MsprobeException.INVALID_PARAM_ERROR)
147 148 
148 if args.golden_path and len(args.rank_size) != 2:149 if args.golden_path and len(args.rank_size) != 2:
149- logger.error('In the graph merge compare task, '150+ logger.error(
150- 'the number of parameters "rank_size" to be filled in is either 2!')151+ 'In the graph merge compare task, the number of parameters "rank_size" to be filled in is either 2!'
152+ )
151 raise MsprobeException(MsprobeException.INVALID_PARAM_ERROR)153 raise MsprobeException(MsprobeException.INVALID_PARAM_ERROR)
152 154 
153 if len(args.rank_size) == 1:155 if len(args.rank_size) == 1:
154 if any(len(x) != 1 for x in param_t):156 if any(len(x) != 1 for x in param_t):
155- logger.error('In the graph merge build task, '157+ logger.error(
156- 'the number of parameters "tp/pp/rank_size" to be filled in is either 1!')158+ 'In the graph merge build task, the number of parameters "tp/pp/rank_size" to be filled in is either 1!'
159+ )
157 raise MsprobeException(MsprobeException.INVALID_PARAM_ERROR)160 raise MsprobeException(MsprobeException.INVALID_PARAM_ERROR)
158 161 
159- param_n = ParallelParam(args.rank_size[0], args.tp[0], args.pp[0], args.vpp[0] if args.vpp else 1,162+ param_n = ParallelParam(
160- args.order[0] if args.order else 'tp-cp-ep-dp-pp')163+ args.rank_size[0],
164+ args.tp[0],
165+ args.pp[0],
166+ args.vpp[0] if args.vpp else 1,
167+ args.order[0] if args.order else 'tp-cp-ep-dp-pp',
168+ )
161 return (param_n,)169 return (param_n,)
162 else:170 else:
163 if any(len(x) != 2 for x in param_t):171 if any(len(x) != 2 for x in param_t):
164- logger.error('In the graph merge compare task, '172+ logger.error(
165- 'the number of parameters "tp/pp/rank_size" to be filled in is either 2!')173+ 'In the graph merge compare task, '
174+ 'the number of parameters "tp/pp/rank_size" to be filled in is either 2!'
175+ )
166 raise MsprobeException(MsprobeException.INVALID_PARAM_ERROR)176 raise MsprobeException(MsprobeException.INVALID_PARAM_ERROR)
167 177 
168- param_n = ParallelParam(args.rank_size[0], args.tp[0], args.pp[0], args.vpp[0] if len(args.vpp) == 2 else 1,178+ param_n = ParallelParam(
169- args.order[0] if len(args.order) == 2 else 'tp-cp-ep-dp-pp')179+ args.rank_size[0],
170- param_b = ParallelParam(args.rank_size[1], args.tp[1], args.pp[1], args.vpp[1] if len(args.vpp) == 2 else 1,180+ args.tp[0],
171- args.order[1] if len(args.order) == 2 else 'tp-cp-ep-dp-pp')181+ args.pp[0],
182+ args.vpp[0] if len(args.vpp) == 2 else 1,
183+ args.order[0] if len(args.order) == 2 else 'tp-cp-ep-dp-pp',
184+ )
185+ param_b = ParallelParam(
186+ args.rank_size[1],
187+ args.tp[1],
188+ args.pp[1],
189+ args.vpp[1] if len(args.vpp) == 2 else 1,
190+ args.order[1] if len(args.order) == 2 else 'tp-cp-ep-dp-pp',
191+ )
172 return param_n, param_b192 return param_n, param_b
173 193 
174 194 
@@ -177,8 +197,10 @@ def validate_parallel_param(parallel_param, dump_path, log_prefix='[NPU]'):
177 params = [parallel_param.tp, parallel_param.pp, parallel_param.rank_size, parallel_param.vpp]197 params = [parallel_param.tp, parallel_param.pp, parallel_param.rank_size, parallel_param.vpp]
178 ranks = check_and_return_dir_contents(dump_path, Const.RANK)198 ranks = check_and_return_dir_contents(dump_path, Const.RANK)
179 if len(ranks) != parallel_param.rank_size:199 if len(ranks) != parallel_param.rank_size:
180- logger.error(f'{log_prefix} The parallel param "rank_size" error, '200+ logger.error(
181- f'you set {parallel_param.rank_size} but expected {len(ranks)}.')201+ f'{log_prefix} The parallel param "rank_size" error, '
202+ f'you set {parallel_param.rank_size} but expected {len(ranks)}.'
203+ )
182 raise MsprobeException(MsprobeException.INVALID_PARAM_ERROR)204 raise MsprobeException(MsprobeException.INVALID_PARAM_ERROR)
183 if any(x is None for x in params):205 if any(x is None for x in params):
184 logger.error(f'{log_prefix} The parallel params "tp/pp/rank_size" must not be null!')206 logger.error(f'{log_prefix} The parallel params "tp/pp/rank_size" must not be null!')
@@ -233,8 +255,12 @@ class ToolTip:
233 MEAN_DIFF = 'NPU与标杆API统计信息比对,平均值的差值'255 MEAN_DIFF = 'NPU与标杆API统计信息比对,平均值的差值'
234 NORM_DIFF = 'NPU与标杆API统计信息比对,2范数(平方根)的差值'256 NORM_DIFF = 'NPU与标杆API统计信息比对,2范数(平方根)的差值'
235 MD5 = '数据MD5信息,用于比较两个数据信息是否完全一致'257 MD5 = '数据MD5信息,用于比较两个数据信息是否完全一致'
236- ONE_THOUSANDTH_ERR_RATIO = 'Tensor中的元素逐个与对应的标杆数据对比,相对误差小于千分之一的比例占总元素个数的比例,比例越接近1越好'258+ ONE_THOUSANDTH_ERR_RATIO = (
237- FIVE_THOUSANDTHS_ERR_RATIO = 'Tensor中的元素逐个与对应的标杆数据对比,相对误差小于千分之五的比例占总元素个数的比例,比例越接近1越好'259+ 'Tensor中的元素逐个与对应的标杆数据对比,相对误差小于千分之一的比例占总元素个数的比例,比例越接近1越好'
260+ )
261+ FIVE_THOUSANDTHS_ERR_RATIO = (
262+ 'Tensor中的元素逐个与对应的标杆数据对比,相对误差小于千分之五的比例占总元素个数的比例,比例越接近1越好'
263+ )
238 COSINE = (264 COSINE = (
239 '通过计算两个向量的余弦值来判断其相似度,数值越接近于1说明计算出的两个张量越相似,实际可接受阈值为大于0.99。'265 '通过计算两个向量的余弦值来判断其相似度,数值越接近于1说明计算出的两个张量越相似,实际可接受阈值为大于0.99。'
240 '在计算中可能会存在nan,主要由于可能会出现其中一个向量为0'266 '在计算中可能会存在nan,主要由于可能会出现其中一个向量为0'
@@ -272,12 +298,20 @@ class GraphConst:
272 INPUT = '.input.'298 INPUT = '.input.'
273 OUTPUT = '.output.'299 OUTPUT = '.output.'
274 STR_MAX_LEN = 50300 STR_MAX_LEN = 50
275- MD5_INDEX_LIST = CompareConst.MD5_COMPARE_INDEX + [CompareConst.REQ_GRAD_CONSIST, CompareConst.RESULT,301+ MD5_INDEX_LIST = CompareConst.MD5_COMPARE_INDEX + [
276- CompareConst.ERROR_MESSAGE]302+ CompareConst.REQ_GRAD_CONSIST,
277- REAL_DATA_INDEX_LIST = CompareConst.ALL_COMPARE_INDEX + [CompareConst.REQ_GRAD_CONSIST, CompareConst.RESULT,303+ CompareConst.RESULT,
278- CompareConst.ERROR_MESSAGE]304+ CompareConst.ERROR_MESSAGE,
279- SUMMARY_INDEX_LIST = CompareConst.SUMMARY_COMPARE_INDEX + [CompareConst.REQ_GRAD_CONSIST, CompareConst.RESULT,305+ ]
280- CompareConst.ERROR_MESSAGE]306+ REAL_DATA_INDEX_LIST = LazyConst(
307+ lambda: CompareConst.ALL_COMPARE_INDEX
308+ + [CompareConst.REQ_GRAD_CONSIST, CompareConst.RESULT, CompareConst.ERROR_MESSAGE]
309+ )
310+ SUMMARY_INDEX_LIST = CompareConst.SUMMARY_COMPARE_INDEX + [
311+ CompareConst.REQ_GRAD_CONSIST,
312+ CompareConst.RESULT,
313+ CompareConst.ERROR_MESSAGE,
314+ ]
281 APIS_BETWEEN_MODULES = 'Apis_Between_Modules'315 APIS_BETWEEN_MODULES = 'Apis_Between_Modules'
282 APIS_BETWEEN_MODULES_ALL_RANKS = 'Apis_Between_Modules_All_Ranks'316 APIS_BETWEEN_MODULES_ALL_RANKS = 'Apis_Between_Modules_All_Ranks'
283 NULL = 'null'317 NULL = 'null'
@@ -294,20 +328,20 @@ class GraphConst:
294 Const.ALL: REAL_DATA_COMPARE,328 Const.ALL: REAL_DATA_COMPARE,
295 Const.SUMMARY: SUMMARY_COMPARE,329 Const.SUMMARY: SUMMARY_COMPARE,
296 Const.MD5: MD5_COMPARE,330 Const.MD5: MD5_COMPARE,
297- Const.STRUCTURE: STRUCTURE_COMPARE331+ Const.STRUCTURE: STRUCTURE_COMPARE,
298 }332 }
299 333 
300 GRAPHCOMPARE_MODE_TO_DUMP_MODE_TO_MAPPING = {334 GRAPHCOMPARE_MODE_TO_DUMP_MODE_TO_MAPPING = {
301 REAL_DATA_COMPARE: Const.ALL,335 REAL_DATA_COMPARE: Const.ALL,
302 SUMMARY_COMPARE: Const.SUMMARY,336 SUMMARY_COMPARE: Const.SUMMARY,
303 MD5_COMPARE: Const.MD5,337 MD5_COMPARE: Const.MD5,
304- STRUCTURE_COMPARE: Const.STRUCTURE338+ STRUCTURE_COMPARE: Const.STRUCTURE,
305 }339 }
306 340 
307 COMPARE_INDICATOR_TO_PRECISION_INDEX_MAPPING = {341 COMPARE_INDICATOR_TO_PRECISION_INDEX_MAPPING = {
308 CompareConst.PASS: 0,342 CompareConst.PASS: 0,
309 CompareConst.WARNING: 0.5,343 CompareConst.WARNING: 0.5,
310- CompareConst.ERROR: 1344+ CompareConst.ERROR: 1,
311 }345 }
312 346 
313 RANKS = 'ranks'347 RANKS = 'ranks'
@@ -325,8 +359,14 @@ class GraphConst:
325 UNCERTAINTY_THRESHOLD = 1e-6359 UNCERTAINTY_THRESHOLD = 1e-6
326 REDUCE_OPERATIONS = ['reduce_scatter', 'all_reduce']360 REDUCE_OPERATIONS = ['reduce_scatter', 'all_reduce']
327 361 
328- IGNORE_PRECISION_INDEX = {'empty', 'empty_like', 'empty_with_format', 'new_empty_strided', 'new_empty',362+ IGNORE_PRECISION_INDEX = {
329- 'empty_strided'}363+ 'empty',
364+ 'empty_like',
365+ 'empty_with_format',
366+ 'new_empty_strided',
367+ 'new_empty',
368+ 'empty_strided',
369+ }
330 VPP_CHUNK_0 = '0'370 VPP_CHUNK_0 = '0'
331 371 
332 PBAR_TOTAL = 100372 PBAR_TOTAL = 100
@@ -365,12 +405,15 @@ def calculate_list(path_n, path_b, prefix_str=Const.RANK, mode=GraphConst.INTERS
365 计算列表,可取交集、并集405 计算列表,可取交集、并集
366 """406 """
367 if prefix_str not in [Const.RANK, Const.STEP]:407 if prefix_str not in [Const.RANK, Const.STEP]:
368- logger.error(f'Parameter "prefix_str" is expected to be {Const.RANK} or {Const.STEP}, '408+ logger.error(
369- f'but in reality it is {prefix_str}.')409+ f'Parameter "prefix_str" is expected to be {Const.RANK} or {Const.STEP}, but in reality it is {prefix_str}.'
410+ )
370 raise MsprobeException(MsprobeException.INVALID_PARAM_ERROR)411 raise MsprobeException(MsprobeException.INVALID_PARAM_ERROR)
371 if mode not in [GraphConst.INTERSECTION, GraphConst.UNION]:412 if mode not in [GraphConst.INTERSECTION, GraphConst.UNION]:
372- logger.error(f'Parameter "mode" is expected to be {GraphConst.INTERSECTION} or {GraphConst.UNION}, '413+ logger.error(
373- f'but in reality it is {mode}.')414+ f'Parameter "mode" is expected to be {GraphConst.INTERSECTION} or {GraphConst.UNION}, '
415+ f'but in reality it is {mode}.'
416+ )
374 raise MsprobeException(MsprobeException.INVALID_PARAM_ERROR)417 raise MsprobeException(MsprobeException.INVALID_PARAM_ERROR)
375 418 
376 npu_list = check_and_return_dir_contents(path_n, prefix_str, skip_wrong_dir=True)419 npu_list = check_and_return_dir_contents(path_n, prefix_str, skip_wrong_dir=True)
@@ -404,6 +447,7 @@ class ProgressInfo:
404 """447 """
405 前端监测此类属性获取进度信息448 前端监测此类属性获取进度信息
406 """449 """
450+ 
407 current_progress = 0451 current_progress = 0
408 process_running = True452 process_running = True
409 error_msg = []453 error_msg = []
@@ -458,8 +502,9 @@ def monitor_progress(pbar_info, pbar, all_task_ids, is_parallel_merge=False):
458 task_progress[task_id] = 0502 task_progress[task_id] = 0
459 # 图合并模式,总进度卡99%,剩余的1%给db后处理函数; 否则总进度卡98%,剩余的1%给db后处理函数,1%给分布式节点分析关联503 # 图合并模式,总进度卡99%,剩余的1%给db后处理函数; 否则总进度卡98%,剩余的1%给db后处理函数,1%给分布式节点分析关联
460 global POST_DB_PROCESS504 global POST_DB_PROCESS
461- POST_DB_PROCESS = pbar_info.total - pbar_info.step_total if is_parallel_merge \505+ POST_DB_PROCESS = (
462- else pbar_info.total - 2 * pbar_info.step_total506+ pbar_info.total - pbar_info.step_total if is_parallel_merge else pbar_info.total - 2 * pbar_info.step_total
507+ )
463 all_complete = False508 all_complete = False
464 509 
465 while not all_complete:510 while not all_complete:
@@ -513,7 +558,7 @@ def update_pbar_info(pbar_info, current_data, total_data, step_ratio=0.01, updat
513 return558 return
514 559 
515 progress = (current_data * pbar_info.total // total_data) // pbar_info.stage_total560 progress = (current_data * pbar_info.total // total_data) // pbar_info.stage_total
516- progress += (pbar_info.stage_progress * (pbar_info.current_stage_dict.get(pbar_info.task_id) - 1))561+ progress += pbar_info.stage_progress * (pbar_info.current_stage_dict.get(pbar_info.task_id) - 1)
517 if pbar_info.pbar is not None:562 if pbar_info.pbar is not None:
518 if progress == pbar_info.total:563 if progress == pbar_info.total:
519 # 卡99%,剩余的1%给db后处理函数564 # 卡99%,剩余的1%给db后处理函数
@@ -0,0 +1,456 @@
1+# coding=utf-8
2+# -------------------------------------------------------------------------
3+# This file is part of the MindStudio project.
4+# Copyright (c) 2025 Huawei Technologies Co.,Ltd.
5+#
6+# MindStudio is licensed under Mulan PSL v2.
7+# You can use this software according to the terms and conditions of the Mulan PSL v2.
8+# You may obtain a copy of Mulan PSL v2 at:
9+#
10+# http://license.coscl.org.cn/MulanPSL2
11+#
12+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
13+# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
14+# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
15+# See the Mulan PSL v2 for more details.
16+# -------------------------------------------------------------------------
17+ 
18+import unittest
19+from unittest import mock
20+ 
21+from msprobe.core.compare.algorithm.algorithm_scheduler import AlgorithmScheduler
22+ 
23+EXPECTED_BUILTIN_ALGORITHMS = {
24+ "cosine_similarity",
25+ "euclidean_distance",
26+ "five_thousand_error",
27+ "max_absolute_error",
28+ "max_relative_error",
29+ "one_thousand_error",
30+}
31+EXPECTED_BUILTIN_COLUMN_NAMES = {
32+ "Cosine",
33+ "EucDist",
34+ "Five Thousandths Err Ratio",
35+ "MaxAbsErr",
36+ "MaxRelativeErr",
37+ "One Thousandth Err Ratio",
38+}
39+ 
40+ 
41+def _two_param_func(n_value, b_value):
42+ return 0.0, ""
43+ 
44+ 
45+def _three_param_func(n_value, b_value, args):
46+ return 0.0, ""
47+ 
48+ 
49+def _one_param_func(n_value):
50+ return 0.0, ""
51+ 
52+ 
53+class TestAlgorithmScheduler(unittest.TestCase):
54+ """AlgorithmScheduler 单例/发现/校验/调度行为单测。"""
55+ 
56+ def setUp(self):
57+ # 每个用例隔离单例状态,避免相互污染
58+ AlgorithmScheduler._instance = None
59+ 
60+ def tearDown(self):
61+ AlgorithmScheduler._instance = None
62+ 
63+ # ------------------------------------------------------------------
64+ # 辅助:构造一个未经 __init__ 的纯净实例,用于测试纯方法/受控状态
65+ # ------------------------------------------------------------------
66+ @staticmethod
67+ def _bare_instance():
68+ inst = object.__new__(AlgorithmScheduler)
69+ inst._module_cache = {}
70+ inst.build_in_support_algorithm = []
71+ inst.custom_support_algorithm = []
72+ inst.algorithm_names = []
73+ inst._column_name_algorithm_mapping = {}
74+ inst.algorithm_column_names = []
75+ return inst
76+ 
77+ # ------------------------------------------------------------------
78+ # 单例行为
79+ # ------------------------------------------------------------------
80+ def test_get_instance_given_multiple_calls_when_any_then_same_instance(self):
81+ inst1 = AlgorithmScheduler.get_instance()
82+ inst2 = AlgorithmScheduler.get_instance()
83+ inst3 = AlgorithmScheduler.get_instance()
84+ self.assertIs(inst1, inst2)
85+ self.assertIs(inst2, inst3)
86+ 
87+ def test_direct_new_given_any_when_invoke_then_runtime_error(self):
88+ with self.assertRaises(RuntimeError):
89+ AlgorithmScheduler()
90+ 
91+ # ------------------------------------------------------------------
92+ # builtin 算法自动发现
93+ # ------------------------------------------------------------------
94+ def test_make_support_algorithm_given_builtin_dir_when_load_then_all_discovered(self):
95+ inst = AlgorithmScheduler.get_instance()
96+ self.assertEqual(set(inst.build_in_support_algorithm), EXPECTED_BUILTIN_ALGORITHMS)
97+ self.assertEqual(inst.custom_support_algorithm, [])
98+ 
99+ def test_validate_column_names_given_builtin_when_load_then_no_duplicate(self):
100+ # __init__ 已执行 _validate_column_names,未抛异常即说明列名唯一
101+ inst = AlgorithmScheduler.get_instance()
102+ self.assertEqual(set(inst.algorithm_column_names), EXPECTED_BUILTIN_COLUMN_NAMES)
103+ self.assertEqual(len(inst.algorithm_column_names), len(set(inst.algorithm_column_names)))
104+ 
105+ def test_get_algorithm_names_given_loaded_when_call_then_returns_all(self):
106+ inst = AlgorithmScheduler.get_instance()
107+ self.assertEqual(set(inst.get_algorithm_names()), EXPECTED_BUILTIN_ALGORITHMS)
108+ 
109+ def test_get_algorithm_column_names_given_loaded_when_call_then_returns_all(self):
110+ inst = AlgorithmScheduler.get_instance()
111+ self.assertEqual(set(inst.get_algorithm_column_names()), EXPECTED_BUILTIN_COLUMN_NAMES)
112+ 
113+ # ------------------------------------------------------------------
114+ # column_name 去重校验
115+ # ------------------------------------------------------------------
116+ def test_validate_column_names_given_duplicate_when_validate_then_value_error(self):
117+ # 跳过真实加载,构造受控状态后直接测试 _validate_column_names
118+ with mock.patch.object(AlgorithmScheduler, "_make_support_algorithm", lambda self: None):
119+ inst = AlgorithmScheduler.get_instance()
120+ inst.build_in_support_algorithm = ["alg_a", "alg_b"]
121+ inst.custom_support_algorithm = []
122+ inst.algorithm_names = ["alg_a", "alg_b"]
123+ inst._column_name_algorithm_mapping = {}
124+ dummy_module = mock.Mock()
125+ with mock.patch.object(inst, "_get_module", return_value=dummy_module), \
126+ mock.patch.object(inst, "_get_column_name", side_effect=lambda name, mod: "dup_col"):
127+ with self.assertRaisesRegex(ValueError, "duplicated"):
128+ inst._validate_column_names()
129+ 
130+ def test_validate_column_names_given_unique_when_validate_then_mapping_built(self):
131+ with mock.patch.object(AlgorithmScheduler, "_make_support_algorithm", lambda self: None):
132+ inst = AlgorithmScheduler.get_instance()
133+ inst.build_in_support_algorithm = ["alg_a", "alg_b"]
134+ inst.custom_support_algorithm = []
135+ inst.algorithm_names = ["alg_a", "alg_b"]
136+ inst._column_name_algorithm_mapping = {}
137+ dummy_module = mock.Mock()
138+ col_names = {"alg_a": "ColA", "alg_b": "ColB"}
139+ with mock.patch.object(inst, "_get_module", return_value=dummy_module), \
140+ mock.patch.object(inst, "_get_column_name", side_effect=lambda name, mod: col_names[name]):
141+ inst._validate_column_names()
142+ self.assertEqual(inst._column_name_algorithm_mapping, {"ColA": "alg_a", "ColB": "alg_b"})
143+ 
144+ # ------------------------------------------------------------------
145+ # _is_real_number
146+ # ------------------------------------------------------------------
147+ def test_is_real_number_given_int_when_check_then_true(self):
148+ inst = self._bare_instance()
149+ self.assertTrue(inst._is_real_number(1))
150+ 
151+ def test_is_real_number_given_float_when_check_then_true(self):
152+ inst = self._bare_instance()
153+ self.assertTrue(inst._is_real_number(1.5))
154+ 
155+ def test_is_real_number_given_bool_when_check_then_false(self):
156+ inst = self._bare_instance()
157+ self.assertFalse(inst._is_real_number(True))
158+ self.assertFalse(inst._is_real_number(False))
159+ 
160+ def test_is_real_number_given_non_number_when_check_then_false(self):
161+ inst = self._bare_instance()
162+ self.assertFalse(inst._is_real_number("1"))
163+ self.assertFalse(inst._is_real_number(None))
164+ self.assertFalse(inst._is_real_number([1]))
165+ 
166+ # ------------------------------------------------------------------
167+ # _validate_return_value
168+ # ------------------------------------------------------------------
169+ def test_validate_return_value_given_int_str_when_valid_then_true(self):
170+ inst = self._bare_instance()
171+ self.assertTrue(inst._validate_return_value((1, "msg"), "alg"))
172+ 
173+ def test_validate_return_value_given_float_str_when_valid_then_true(self):
174+ inst = self._bare_instance()
175+ self.assertTrue(inst._validate_return_value((1.5, "msg"), "alg"))
176+ 
177+ def test_validate_return_value_given_str_str_when_valid_then_true(self):
178+ inst = self._bare_instance()
179+ self.assertTrue(inst._validate_return_value(("err", "msg"), "alg"))
180+ 
181+ def test_validate_return_value_given_non_tuple_when_invalid_then_value_error(self):
182+ inst = self._bare_instance()
183+ for bad in (5, "abc", [1, "a"], None):
184+ with self.assertRaises(ValueError):
185+ inst._validate_return_value(bad, "alg")
186+ 
187+ def test_validate_return_value_given_wrong_length_when_invalid_then_value_error(self):
188+ inst = self._bare_instance()
189+ for bad in ((1,), (1, "a", "b"), tuple()):
190+ with self.assertRaises(ValueError):
191+ inst._validate_return_value(bad, "alg")
192+ 
193+ def test_validate_return_value_given_bool_first_when_invalid_then_value_error(self):
194+ # bool 被排除在实数之外,且不属于 str,应当拦截
195+ inst = self._bare_instance()
196+ with self.assertRaises(ValueError):
197+ inst._validate_return_value((True, "msg"), "alg")
198+ 
199+ def test_validate_return_value_given_invalid_first_type_when_invalid_then_value_error(self):
200+ inst = self._bare_instance()
201+ for bad_first in ([1], {"a": 1}, None, object()):
202+ with self.assertRaises(ValueError):
203+ inst._validate_return_value((bad_first, "msg"), "alg")
204+ 
205+ def test_validate_return_value_given_invalid_second_type_when_invalid_then_value_error(self):
206+ inst = self._bare_instance()
207+ for bad_second in (1, 1.5, None, [1]):
208+ with self.assertRaises(ValueError):
209+ inst._validate_return_value((1, bad_second), "alg")
210+ 
211+ # ------------------------------------------------------------------
212+ # _get_module
213+ # ------------------------------------------------------------------
214+ def test_get_module_given_unsupported_name_when_call_then_value_error(self):
215+ inst = self._bare_instance()
216+ with self.assertRaisesRegex(ValueError, "is not supported"):
217+ inst._get_module("not_an_algorithm")
218+ 
219+ def test_get_module_given_loaded_when_call_twice_then_cached(self):
220+ inst = AlgorithmScheduler.get_instance()
221+ inst._module_cache.clear()
222+ m1 = inst._get_module("cosine_similarity")
223+ m2 = inst._get_module("cosine_similarity")
224+ self.assertIs(m1, m2)
225+ self.assertIn("cosine_similarity", inst._module_cache)
226+ 
227+ def test_get_module_given_missing_compare_when_load_then_attribute_error(self):
228+ inst = self._bare_instance()
229+ inst.build_in_support_algorithm = ["fake_alg"]
230+ fake_module = mock.Mock(spec=[]) # 无 compare 属性
231+ with mock.patch("importlib.import_module", return_value=fake_module):
232+ with self.assertRaisesRegex(AttributeError, "must define a compare"):
233+ inst._get_module("fake_alg")
234+ 
235+ def test_get_module_given_compare_not_callable_when_load_then_type_error(self):
236+ inst = self._bare_instance()
237+ inst.build_in_support_algorithm = ["fake_alg"]
238+ fake_module = mock.Mock()
239+ fake_module.compare = "not_callable"
240+ with mock.patch("importlib.import_module", return_value=fake_module):
241+ with self.assertRaisesRegex(TypeError, "not a callable"):
242+ inst._get_module("fake_alg")
243+ 
244+ def test_get_module_given_wrong_arg_count_when_load_then_value_error(self):
245+ inst = self._bare_instance()
246+ inst.build_in_support_algorithm = ["fake_alg"]
247+ for func in (_one_param_func, _three_param_func):
248+ fake_module = mock.Mock()
249+ fake_module.compare = func
250+ with mock.patch("importlib.import_module", return_value=fake_module):
251+ with self.assertRaisesRegex(ValueError, "positional arguments"):
252+ inst._get_module("fake_alg")
253+ inst._module_cache.clear()
254+ 
255+ def test_get_module_given_valid_signature_when_load_then_cached(self):
256+ inst = self._bare_instance()
257+ inst.build_in_support_algorithm = ["fake_alg"]
258+ fake_module = mock.Mock()
259+ fake_module.compare = _two_param_func
260+ with mock.patch("importlib.import_module", return_value=fake_module) as mock_import:
261+ module = inst._get_module("fake_alg")
262+ self.assertIs(module, fake_module)
263+ self.assertEqual(mock_import.call_count, 1)
264+ # 第二次命中缓存,不应再次 import
265+ inst._get_module("fake_alg")
266+ self.assertEqual(mock_import.call_count, 1)
267+ 
268+ def test_get_module_given_import_failure_when_load_then_propagates(self):
269+ inst = self._bare_instance()
270+ inst.custom_support_algorithm = ["fake_alg"]
271+ with mock.patch("importlib.import_module", side_effect=ModuleNotFoundError("No module")):
272+ with self.assertRaises(ModuleNotFoundError):
273+ inst._get_module("fake_alg")
274+ 
275+ def test_get_module_given_custom_when_load_then_uses_custom_path(self):
276+ inst = self._bare_instance()
277+ inst.custom_support_algorithm = ["fake_alg"]
278+ fake_module = mock.Mock()
279+ fake_module.compare = _two_param_func
280+ with mock.patch("importlib.import_module", return_value=fake_module) as mock_import:
281+ inst._get_module("fake_alg")
282+ called_module_name = mock_import.call_args[0][0]
283+ self.assertIn(AlgorithmScheduler.CUSTOM_ALGORITHM_DIR_NAME, called_module_name)
284+ 
285+ def test_get_module_given_builtin_when_load_then_uses_builtin_path(self):
286+ inst = self._bare_instance()
287+ inst.build_in_support_algorithm = ["fake_alg"]
288+ fake_module = mock.Mock()
289+ fake_module.compare = _two_param_func
290+ with mock.patch("importlib.import_module", return_value=fake_module) as mock_import:
291+ inst._get_module("fake_alg")
292+ called_module_name = mock_import.call_args[0][0]
293+ self.assertIn(AlgorithmScheduler.BUILT_IN_ALGORITHM_DIR_NAME, called_module_name)
294+ 
295+ # ------------------------------------------------------------------
296+ # _get_column_name
297+ # ------------------------------------------------------------------
298+ def test_get_column_name_given_missing_func_when_call_then_attribute_error(self):
299+ inst = self._bare_instance()
300+ fake_module = mock.Mock(spec=[])
301+ with self.assertRaisesRegex(AttributeError, "must define a column_name"):
302+ inst._get_column_name("alg", fake_module)
303+ 
304+ def test_get_column_name_given_not_callable_when_call_then_type_error(self):
305+ inst = self._bare_instance()
306+ fake_module = mock.Mock()
307+ fake_module.column_name = "not_callable"
308+ with self.assertRaisesRegex(TypeError, "not a callable"):
309+ inst._get_column_name("alg", fake_module)
310+ 
311+ def test_get_column_name_given_func_raises_when_call_then_value_error(self):
312+ inst = self._bare_instance()
313+ fake_module = mock.Mock()
314+ fake_module.column_name = mock.Mock(side_effect=RuntimeError("boom"))
315+ with self.assertRaisesRegex(ValueError, "Error occurred"):
316+ inst._get_column_name("alg", fake_module)
317+ 
318+ def test_get_column_name_given_empty_return_when_call_then_value_error(self):
319+ inst = self._bare_instance()
320+ fake_module = mock.Mock()
321+ fake_module.column_name = lambda: ""
322+ with self.assertRaisesRegex(ValueError, "non"):
323+ inst._get_column_name("alg", fake_module)
324+ 
325+ def test_get_column_name_given_non_string_return_when_call_then_value_error(self):
326+ inst = self._bare_instance()
327+ fake_module = mock.Mock()
328+ fake_module.column_name = lambda: 123
329+ with self.assertRaises(ValueError):
330+ inst._get_column_name("alg", fake_module)
331+ 
332+ def test_get_column_name_given_valid_when_call_then_returns_name(self):
333+ inst = self._bare_instance()
334+ fake_module = mock.Mock()
335+ fake_module.column_name = lambda: "ColA"
336+ self.assertEqual(inst._get_column_name("alg", fake_module), "ColA")
337+ 
338+ # ------------------------------------------------------------------
339+ # _call_algorithm
340+ # ------------------------------------------------------------------
341+ 
342+ def test_call_algorithm_given_valid_return_when_call_then_returns_result(self):
343+ inst = self._bare_instance()
344+ fake_module = mock.Mock()
345+ fake_module.compare = lambda n, b: (0.5, "")
346+ with mock.patch.object(inst, "_get_module", return_value=fake_module):
347+ result = inst._call_algorithm("alg", 1, 2)
348+ self.assertEqual(result, (0.5, ""))
349+ 
350+ # ------------------------------------------------------------------
351+ # compare 编排
352+ # ------------------------------------------------------------------
353+ def test_compare_given_unknown_column_when_call_then_unsupported(self):
354+ inst = self._bare_instance()
355+ inst._column_name_algorithm_mapping = {"ColA": "alg_a", "ColB": "alg_b"}
356+ inst.algorithm_column_names = ["ColA", "ColB"]
357+ results, errors = inst.compare(1, 2, column_names=["Unknown"])
358+ self.assertEqual(results, ["unsupported"])
359+ self.assertIn("No available algorithm", errors[0])
360+ 
361+ def test_compare_given_none_columns_when_call_then_uses_all(self):
362+ inst = self._bare_instance()
363+ inst._column_name_algorithm_mapping = {"ColA": "alg_a", "ColB": "alg_b"}
364+ inst.algorithm_column_names = ["ColA", "ColB"]
365+ with mock.patch.object(inst, "_call_algorithm", return_value=(0.0, "")) as mock_call:
366+ results, errors = inst.compare(1, 2, column_names=None)
367+ self.assertEqual(len(results), 2)
368+ self.assertEqual(mock_call.call_count, 2)
369+ 
370+ def test_compare_given_empty_columns_when_call_then_uses_all(self):
371+ inst = self._bare_instance()
372+ inst._column_name_algorithm_mapping = {"ColA": "alg_a", "ColB": "alg_b"}
373+ inst.algorithm_column_names = ["ColA", "ColB"]
374+ with mock.patch.object(inst, "_call_algorithm", return_value=(0.0, "")) as mock_call:
375+ results, errors = inst.compare(1, 2, column_names=[])
376+ self.assertEqual(len(results), 2)
377+ self.assertEqual(mock_call.call_count, 2)
378+ 
379+ def test_compare_given_subset_columns_when_call_then_only_subset_invoked(self):
380+ inst = self._bare_instance()
381+ inst._column_name_algorithm_mapping = {"ColA": "alg_a", "ColB": "alg_b"}
382+ inst.algorithm_column_names = ["ColA", "ColB"]
383+ with mock.patch.object(inst, "_call_algorithm", return_value=(0.0, "")) as mock_call:
384+ results, errors = inst.compare(1, 2, column_names=["ColA"])
385+ self.assertEqual(len(results), 1)
386+ self.assertEqual(mock_call.call_count, 1)
387+ 
388+ def test_compare_given_error_msg_when_call_then_error_msgs_collected(self):
389+ inst = self._bare_instance()
390+ inst._column_name_algorithm_mapping = {"ColA": "alg_a"}
391+ inst.algorithm_column_names = ["ColA"]
392+ with mock.patch.object(inst, "_call_algorithm", return_value=(0.0, "some error")):
393+ results, errors = inst.compare(1, 2, column_names=["ColA"])
394+ self.assertEqual(errors, ["some error"])
395+ 
396+ # ------------------------------------------------------------------
397+ # 自定义/内置算法目录加载(含空目录、非法文件名跳过)
398+ # ------------------------------------------------------------------
399+ def test_make_support_algorithm_from_dir_given_nonexistent_when_call_then_skip(self):
400+ inst = self._bare_instance()
401+ algos = []
402+ with mock.patch("os.path.exists", return_value=False):
403+ inst._make_support_algorithm_from_dir("/no/such/dir", algos)
404+ self.assertEqual(algos, [])
405+ 
406+ def test_make_support_algorithm_from_dir_given_empty_when_call_then_no_algorithm(self):
407+ inst = self._bare_instance()
408+ algos = []
409+ with mock.patch("os.path.exists", return_value=True), \
410+ mock.patch("os.listdir", return_value=[]), \
411+ mock.patch("os.path.isfile", return_value=True):
412+ inst._make_support_algorithm_from_dir("/fake/dir", algos)
413+ self.assertEqual(algos, [])
414+ 
415+ def test_make_support_algorithm_from_dir_given_mixed_names_when_call_then_only_valid_kept(self):
416+ inst = self._bare_instance()
417+ algos = []
418+ # alg_OK.py 含大写,不匹配 [a-z0-9_];subdir 无 .py 后缀;not_match.py 不以 alg_ 开头
419+ files = ["alg_valid.py", "not_match.py", "alg_OK.py", "alg_another.py", "subdir"]
420+ with mock.patch("os.path.exists", return_value=True), \
421+ mock.patch("os.listdir", return_value=files), \
422+ mock.patch("os.path.isfile", return_value=True):
423+ inst._make_support_algorithm_from_dir("/fake/dir", algos)
424+ self.assertEqual(algos, ["valid", "another"])
425+ 
426+ def test_add_algorithm_file_to_list_given_valid_name_when_call_then_appended(self):
427+ algos = []
428+ with mock.patch("os.path.isfile", return_value=True):
429+ result = AlgorithmScheduler._add_algorithm_file_to_list("/x/alg_test.py", algos)
430+ self.assertTrue(result)
431+ self.assertEqual(algos, ["test"])
432+ 
433+ def test_add_algorithm_file_to_list_given_invalid_name_when_call_then_skipped(self):
434+ algos = []
435+ with mock.patch("os.path.isfile", return_value=True):
436+ result = AlgorithmScheduler._add_algorithm_file_to_list("/x/not_match.py", algos)
437+ self.assertFalse(result)
438+ self.assertEqual(algos, [])
439+ 
440+ def test_add_algorithm_file_to_list_given_non_file_when_call_then_skipped(self):
441+ algos = []
442+ with mock.patch("os.path.isfile", return_value=False):
443+ result = AlgorithmScheduler._add_algorithm_file_to_list("/x/alg_test.py", algos)
444+ self.assertFalse(result)
445+ self.assertEqual(algos, [])
446+ 
447+ def test_add_algorithm_file_to_list_given_uppercase_when_call_then_skipped(self):
448+ algos = []
449+ with mock.patch("os.path.isfile", return_value=True):
450+ result = AlgorithmScheduler._add_algorithm_file_to_list("/x/alg_OK.py", algos)
451+ self.assertFalse(result)
452+ self.assertEqual(algos, [])
453+ 
454+ 
455+if __name__ == "__main__":
456+ unittest.main()
@@ -113,15 +113,11 @@ class TestCompareRealData(unittest.TestCase):
113 cross_frame = False113 cross_frame = False
114 compare_real_data = CompareRealData(file_reader, mode_config, cross_frame)114 compare_real_data = CompareRealData(file_reader, mode_config, cross_frame)
115 115 
116- comparison_result = ComparisonResult(116+ results = {CompareConst.COSINE: [0.99, 0.98], CompareConst.MAX_ABS_ERR: [0.01, 0.02],
117- cos_result=[0.99, 0.98],117+ CompareConst.MAX_RELATIVE_ERR: [0.001, 0.002], CompareConst.EUC_DIST: [0.5, 0.49],
118- max_err_result=[0.01, 0.02],118+ CompareConst.ONE_THOUSANDTH_ERR_RATIO: [0.1, 0.2], CompareConst.FIVE_THOUSANDTHS_ERR_RATIO: [0.05, 0.1]}
119- max_relative_err_result=[0.001, 0.002],119+ comparison_result = ComparisonResult(results, ['', 'Error in comparison'])
120- euc_dist_result=[0.5, 0.49],120+ 
121- one_thousand_err_ratio_result=[0.1, 0.2],
122- five_thousand_err_ratio_result=[0.05, 0.1],
123- err_msgs=['', 'Error in comparison']
124- )
125 offset = 0121 offset = 0
126 updated_df = compare_real_data._save_cmp_result(offset, comparison_result, self.result_df, self.lock)122 updated_df = compare_real_data._save_cmp_result(offset, comparison_result, self.result_df, self.lock)
127 123 
@@ -135,18 +131,13 @@ class TestCompareRealData(unittest.TestCase):
135 cross_frame = False131 cross_frame = False
136 compare_real_data = CompareRealData(file_reader, mode_config, cross_frame)132 compare_real_data = CompareRealData(file_reader, mode_config, cross_frame)
137 133 
138- comparison_result = ComparisonResult(134+ results = {CompareConst.COSINE: [0.99], CompareConst.MAX_ABS_ERR: [0.98],
139- cos_result=[0.99],135+ CompareConst.MAX_RELATIVE_ERR: [0.001], CompareConst.EUC_DIST: [0.5],
140- max_err_result=[],136+ CompareConst.ONE_THOUSANDTH_ERR_RATIO: [0.1], CompareConst.FIVE_THOUSANDTHS_ERR_RATIO: [0.05]}
141- max_relative_err_result=[0.001],137+ comparison_result = ComparisonResult(results, [""])
142- euc_dist_result=[0.5],
143- one_thousand_err_ratio_result=[0.1],
144- five_thousand_err_ratio_result=[0.05],
145- err_msgs=['']
146- )
147 with self.assertRaises(CompareException) as context:138 with self.assertRaises(CompareException) as context:
148 compare_real_data._save_cmp_result(0, comparison_result, self.result_df, self.lock)139 compare_real_data._save_cmp_result(0, comparison_result, self.result_df, self.lock)
149- self.assertEqual(context.exception.code, CompareException.INDEX_OUT_OF_BOUNDS_ERROR)140+ self.assertEqual(context.exception.code, CompareException.INVALID_DATA_ERROR)
150 141 
151 def test_compare_by_op_bench_normal(self):142 def test_compare_by_op_bench_normal(self):
152 npu_op_name = 'Functional.linear.0.forward.input.0'143 npu_op_name = 'Functional.linear.0.forward.input.0'
@@ -81,9 +81,6 @@ class TestModeAdapter(unittest.TestCase):
81 81 
82 self.adapter.compare_mode = GraphConst.REAL_DATA_COMPARE82 self.adapter.compare_mode = GraphConst.REAL_DATA_COMPARE
83 tips = self.adapter.get_tool_tip()83 tips = self.adapter.get_tool_tip()
84- self.assertEqual(tips, json.dumps({84+ algorithm_columns = CompareConst.ALL_COMPARE_INDEX
85- CompareConst.ONE_THOUSANDTH_ERR_RATIO: ToolTip.ONE_THOUSANDTH_ERR_RATIO,85+ expect_tips = {col: "" for col in algorithm_columns}
86- CompareConst.FIVE_THOUSANDTHS_ERR_RATIO: ToolTip.FIVE_THOUSANDTHS_ERR_RATIO,86+ self.assertEqual(tips, json.dumps(expect_tips))
87- CompareConst.COSINE: ToolTip.COSINE,
88- CompareConst.MAX_ABS_ERR: ToolTip.MAX_ABS_ERR,
89- CompareConst.MAX_RELATIVE_ERR: ToolTip.MAX_RELATIVE_ERR}))