已合并
adapt oversize dump name #498
starchen_创建于 27 天前
adapt oversize dump name #498
已合并
共 12 个文件变更+788-33
| @@ -155,4 +155,4 @@ repos: | |||
| 155 | # 后续遇新误判词追加即可（codespell 按小写匹配）。 | 155 | # 后续遇新误判词追加即可（codespell 按小写匹配）。 |
| 156 | args: | 156 | args: |
| 157 | - --skip=*.bin,*.xlsx,*.png,*.svg,*.lock,*.whl,*.pyc,./build,./bundle,./submodule,./third_party,./node_modules | 157 | - --skip=*.bin,*.xlsx,*.png,*.svg,*.lock,*.whl,*.pyc,./build,./bundle,./submodule,./third_party,./node_modules |
| 158 | - - --ignore-words-list=cann,oam,msprof,aicerr,hccl,tbe,te,nin,fo,optin,alog,abl,OpEnd,mata | 158 | + - --ignore-words-list=cann,oam,msprof,aicerr,hccl,tbe,te,nin,fo,optin,alog,abl,OpEnd,mata,VALU |
| @@ -33,6 +33,7 @@ class AicErrorInfo: | |||
| 33 | self.task_id = "" | 33 | self.task_id = "" |
| 34 | self.stream_id = "" | 34 | self.stream_id = "" |
| 35 | self.node_name = "" | 35 | self.node_name = "" |
| 36 | + self.data_name = "" # plog解析出的原始data_name，可能超过NAME_MAX | ||
| 36 | self.kernel_name = "" | 37 | self.kernel_name = "" |
| 37 | self.flip_num = "" | 38 | self.flip_num = "" |
| 38 | self.instr = "" | 39 | self.instr = "" |
| @@ -388,6 +388,8 @@ class AicoreErrorParser: | |||
| 388 | info.extra_info = self._get_extra_info(aic_err_ret.pop('extra_info')) | 388 | info.extra_info = self._get_extra_info(aic_err_ret.pop('extra_info')) |
| 389 | info.aic_error_info = aic_err_ret | 389 | info.aic_error_info = aic_err_ret |
| 390 | 390 | ||
| 391 | + info.data_name = data_name | ||
| 392 | + | ||
| 391 | # 此处附加判断是L0还是L1 | 393 | # 此处附加判断是L0还是L1 |
| 392 | error_info = self._get_node_and_kernel_name(data_name) | 394 | error_info = self._get_node_and_kernel_name(data_name) |
| 393 | # 访问具名元组的属性 | 395 | # 访问具名元组的属性 |
| @@ -37,6 +37,9 @@ class Collection: | |||
| 37 | self.collect_level = 0 | 37 | self.collect_level = 0 |
| 38 | self.ffts_flag = False | 38 | self.ffts_flag = False |
| 39 | self.is_sk = False | 39 | self.is_sk = False |
| 40 | + # 超长文件名场景：dump文件被框架重命名为随机数字串 | ||
 | |||
| 41 | + self.dump_file_rename = "" # 映射后的实际文件名 | ||
| 42 | + self._mapping_csv_path = "" # mapping.csv路径，首次解析后缓存，避免重复find | ||
| 40 | 43 | ||
| 41 | 44 | ||
| 42 | def get_sk_kernel_name(plog_dir) -> str: | 45 | def get_sk_kernel_name(plog_dir) -> str: |
| @@ -156,6 +159,50 @@ class Collection: | |||
| 156 | err_time, device_id, data_name = dump_data_ret[0] | 159 | err_time, device_id, data_name = dump_data_ret[0] |
| 157 | return err_time, device_id, data_name | 160 | return err_time, device_id, data_name |
| 158 | 161 | ||
| 162 | + | ||
| 163 | + def _is_oversize_name(data_name: str) -> bool: | ||
| 164 | + # 超过NAME_MAX的算子名，异常dump框架会将落盘文件重命名为随机数字串 | ||
| 165 | + # NAME_MAX是字节上限，多字节文件名下需按编码后的字节数判断 | ||
| 166 | + return len(os.fsencode(data_name)) > Constant.MAX_FILE_NAME_LEN | ||
| 167 | + | ||
| 168 | + def _get_dump_mapping_csv_path(self, device_id) -> str: | ||
| 169 | + find_mapping_cmd = ['find', self.report_path, '-name', Constant.MAPPING_CSV_FILE] | ||
| 170 | + regexp = r"[_\.\-/0-9a-zA-Z.]{1,}" | ||
| 171 | + mapping_csv_list = utils.get_inquire_result(find_mapping_cmd, regexp) | ||
| 172 | + if not mapping_csv_list: | ||
| 173 | + return "" | ||
| 174 | + # mapping.csv与dump文件同级，只取报错device对应的那一份。 | ||
| 175 | + # 此处匹配到目录分隔符与文件名，避免device_id为0时误命中data-dump/01/ | ||
| 176 | + device_dump_suffix = os.path.join("data-dump", str(device_id), Constant.MAPPING_CSV_FILE) | ||
| 177 | + for mapping_csv in mapping_csv_list: | ||
| 178 | + if mapping_csv.endswith(device_dump_suffix): | ||
| 179 | + return mapping_csv | ||
| 180 | + # 不回退到其它device的映射表：各device的随机名互不相通，用错会静默收集到别的卡的dump | ||
| 181 | + utils.print_warn_log( | ||
| 182 | + f"{Constant.MAPPING_CSV_FILE} of device {device_id} cannot be found in {self.report_path}, " | ||
| 183 | + f"{len(mapping_csv_list)} {Constant.MAPPING_CSV_FILE} of other devices are ignored.") | ||
| 184 | + return "" | ||
| 185 | + | ||
| 186 | + def _resolve_dump_file_rename(self, device_id, data_name) -> str: | ||
| 187 | + # 长度未超限时不查mapping.csv，避免大目录下不必要的全盘扫描 | ||
| 188 | + if not self._is_oversize_name(data_name): | ||
| 189 | + return "" | ||
| 190 | + mapping_csv_path = self._get_dump_mapping_csv_path(device_id) | ||
| 191 | + self._mapping_csv_path = mapping_csv_path | ||
| 192 | + dump_name_mapping = utils.parse_name_mapping_csv(mapping_csv_path) | ||
| 193 | + rename = dump_name_mapping.get(data_name, "") | ||
| 194 | + if rename: | ||
| 195 | + utils.print_info_log(f"The dump file name exceeds {Constant.MAX_FILE_NAME_LEN}, " | ||
| 196 | + f"use mapped name {rename} instead of {data_name}.") | ||
| 197 | + elif not mapping_csv_path: | ||
| 198 | + utils.print_warn_log(f"The dump file name {data_name} exceeds {Constant.MAX_FILE_NAME_LEN}, " | ||
| 199 | + f"but {Constant.MAPPING_CSV_FILE} cannot be found in {self.report_path}.") | ||
| 200 | + else: | ||
| 201 | + utils.print_warn_log(f"The dump file name {data_name} exceeds {Constant.MAX_FILE_NAME_LEN}, " | ||
| 202 | + f"but it is not recorded in {mapping_csv_path}.") | ||
| 203 | + self.dump_file_rename = rename | ||
| 204 | + return rename | ||
| 205 | + | ||
| 159 | def collect_plog_file(self): | 206 | def collect_plog_file(self): |
| 160 | find_path_cmd = ['grep', r'\[Dump\]\[Exception\]', '-inrE', self.report_path] | 207 | find_path_cmd = ['grep', r'\[Dump\]\[Exception\]', '-inrE', self.report_path] |
| 161 | find_path_regexp = r"(/[_\-/0-9a-zA-Z.]{1,}.[log|txt]):" | 208 | find_path_regexp = r"(/[_\-/0-9a-zA-Z.]{1,}.[log|txt]):" |
| @@ -257,10 +304,11 @@ class Collection: | |||
| 257 | utils.copy_src_to_dest(original_files, dest_path) | 304 | utils.copy_src_to_dest(original_files, dest_path) |
| 258 | return dest_path | 305 | return dest_path |
| 259 | 306 | ||
| 260 | - def collect_data_dump(self, device_id, data_name): | 307 | + def collect_data_dump(self, device_id, data_name, rename=""): |
| 261 | dest_path = os.path.join(self.output_path, "collection", "dump") | 308 | dest_path = os.path.join(self.output_path, "collection", "dump") |
| 309 | + find_name = rename or data_name | ||
| 262 | find_path_cmd = ['find', self.report_path, '-name', | 310 | find_path_cmd = ['find', self.report_path, '-name', |
| 263 | - f"{data_name}"] | 311 | + f"{find_name}"] |
| 264 | regexp = r"[_\.\-/0-9a-zA-Z.]{1,}" | 312 | regexp = r"[_\.\-/0-9a-zA-Z.]{1,}" |
| 265 | original_files = utils.get_inquire_result(find_path_cmd, regexp) | 313 | original_files = utils.get_inquire_result(find_path_cmd, regexp) |
| 266 | if not original_files: | 314 | if not original_files: |
| @@ -269,24 +317,28 @@ class Collection: | |||
| 269 | ) | 317 | ) |
| 270 | return '' | 318 | return '' |
| 271 | 319 | ||
| 272 | - # 如果找到大于1个data, 则匹配日志中的data_dump和device_id | 320 | + # 如果找到大于1个data, 则按文件自身所在的data-dump/<device_id>/目录取报错device的那一份。 |
| 321 | + # 此处匹配完整目录段，避免device_id为0时误命中data-dump/01/ | ||
| 273 | if len(original_files) > 1: | 322 | if len(original_files) > 1: |
| 274 | - plog_dir = os.path.join(self.output_path, 'collection', 'plog') | 323 | + device_dump_dir = os.path.join("data-dump", str(device_id)) |
| 275 | - for file in original_files: | 324 | + matched_files = [file for file in original_files |
| 276 | - data_dump_cmd = ['grep', os.path.basename(file), '-nr', plog_dir] | 325 | + if os.path.dirname(file).endswith(device_dump_dir)] |
| 277 | - dump_data_regexp = r".*?extra-info\/data-dump\/(\d+)\/[\w.]+" | 326 | + if matched_files: |
| 278 | - data_dump_ret = utils.get_inquire_result(data_dump_cmd, dump_data_regexp) | 327 | + utils.print_info_log(f"Find dump file {os.path.basename(matched_files[0])}.") |
| 279 | - if (device_id != data_dump_ret[0]): | 328 | + original_files = matched_files[:1] |
| 280 | - continue | ||
| 281 | - utils.print_info_log(f"Find dump file {os.path.basename(file)}.") | ||
| 282 | - original_files = [file] | ||
| 283 | 329 | ||
| 284 | utils.check_path_valid(dest_path, isdir=True, output=True) | 330 | utils.check_path_valid(dest_path, isdir=True, output=True) |
| 285 | utils.copy_src_to_dest(original_files, dest_path) | 331 | utils.copy_src_to_dest(original_files, dest_path) |
| 332 | + if rename: | ||
| 333 | + # 超长场景下dump文件保持随机名，需一并收集mapping.csv供解析侧还原 | ||
| 334 | + # 复用_resolve_dump_file_rename中已解析的路径，避免重复find | ||
| 335 | + mapping_csv_path = self._mapping_csv_path or self._get_dump_mapping_csv_path(device_id) | ||
| 336 | + if mapping_csv_path: | ||
| 337 | + utils.copy_src_to_dest([mapping_csv_path], dest_path) | ||
| 286 | return dest_path | 338 | return dest_path |
级别：一般 问题：collect_data_dump 中当 rename 非空时再次调用 _get_dump_mapping_csv_path，而该路径在 _resolve_dump_file_rename 中已经解析过一次，导致同一 find 命令在 collect 流程中重复执行。 影响：在 report_path 目录较大的场景下（如数十 GB dump 数据），重复全盘 find 可能显著增加收集耗时。 修复建议：将首次解析得到的 mapping_csv 路径缓存为实例变量（如 self._mapping_csv_path），collect_data_dump 中直接复用，避免二次搜索。   | |||
| 287 | 339 | ||
| 288 | - def check_dump_data_is_valid(self, err_time, data_name): | 340 | + def check_dump_data_is_valid(self, err_time, data_name, rename=""): |
| 289 | - find_dump_data_cmd = ['find', self.report_path, '-name', data_name] | 341 | + find_dump_data_cmd = ['find', self.report_path, '-name', rename or data_name] |
| 290 | regexp = r".*?\/data-dump\/\d+\/([\w.]+)" | 342 | regexp = r".*?\/data-dump\/\d+\/([\w.]+)" |
| 291 | dump_data_file_list = utils.get_inquire_result(find_dump_data_cmd, regexp) | 343 | dump_data_file_list = utils.get_inquire_result(find_dump_data_cmd, regexp) |
| 292 | home = os.environ.get("HOME") | 344 | home = os.environ.get("HOME") |
| @@ -305,13 +357,14 @@ class Collection: | |||
| 305 | ) | 357 | ) |
| 306 | raise utils.AicErrException(Constant.MS_AICERR_INVALID_DUMP_DATA_ERROR) | 358 | raise utils.AicErrException(Constant.MS_AICERR_INVALID_DUMP_DATA_ERROR) |
| 307 | 359 | ||
| 308 | - def check_host_and_device_kernel_name(self, data_name): | 360 | + def check_host_and_device_kernel_name(self, data_name, rename=""): |
| 309 | if self.is_sk: | 361 | if self.is_sk: |
| 310 | # SK场景下没有device .o，跳过host/device一致性检查 | 362 | # SK场景下没有device .o，跳过host/device一致性检查 |
| 311 | return True | 363 | return True |
| 312 | - kernel_cmd = ['find', self.report_path, '-name', data_name] | 364 | + find_name = rename or data_name |
| 365 | + kernel_cmd = ['find', self.report_path, '-name', find_name] | ||
| 313 | _, kernel_info = utils.execute_command(kernel_cmd) | 366 | _, kernel_info = utils.execute_command(kernel_cmd) |
| 314 | - kernel_path = kernel_info.split(data_name)[0] | 367 | + kernel_path = kernel_info.split(find_name)[0] |
| 315 | res = os.listdir(kernel_path) | 368 | res = os.listdir(kernel_path) |
| 316 | host_kernel_name = '' | 369 | host_kernel_name = '' |
| 317 | device_kernel_name = '' | 370 | device_kernel_name = '' |
| @@ -345,8 +398,9 @@ class Collection: | |||
| 345 | utils.print_info_log('Step 2. Obtain the name and path of the flushed data file from the log.') | 398 | utils.print_info_log('Step 2. Obtain the name and path of the flushed data file from the log.') |
| 346 | try: | 399 | try: |
| 347 | err_time, device_id, data_name = self.get_dump_data_info() | 400 | err_time, device_id, data_name = self.get_dump_data_info() |
| 348 | - self.check_dump_data_is_valid(err_time, data_name) | 401 | + rename = self._resolve_dump_file_rename(device_id, data_name) |
| 349 | - check_result = self.check_host_and_device_kernel_name(data_name) | 402 | + self.check_dump_data_is_valid(err_time, data_name, rename) |
| 403 | + check_result = self.check_host_and_device_kernel_name(data_name, rename) | ||
| 350 | if not check_result: | 404 | if not check_result: |
| 351 | utils.print_error_log(f"The kernel load on the host is different from the device.") | 405 | utils.print_error_log(f"The kernel load on the host is different from the device.") |
| 352 | return False | 406 | return False |
| @@ -355,7 +409,7 @@ class Collection: | |||
| 355 | 409 | ||
| 356 | # collect dump | 410 | # collect dump |
| 357 | utils.print_info_log('Step 3. Obtain the operator name from the log.') | 411 | utils.print_info_log('Step 3. Obtain the operator name from the log.') |
| 358 | - self.collect_data_dump(device_id, data_name) | 412 | + self.collect_data_dump(device_id, data_name, rename) |
| 359 | 413 | ||
| 360 | # get kernel_name | 414 | # get kernel_name |
| 361 | utils.print_info_log('Step 4. Obtain the compilation file based on the operator name.') | 415 | utils.print_info_log('Step 4. Obtain the compilation file based on the operator name.') |
| @@ -67,6 +67,9 @@ class Constant: | |||
| 67 | MAX_READ_FILE_BYTES = 1024 * 1024 # 1M | 67 | MAX_READ_FILE_BYTES = 1024 * 1024 # 1M |
| 68 | MAX_TAR_SIZE = 1 * 1024 * 1024 * 1024 # 1G | 68 | MAX_TAR_SIZE = 1 * 1024 * 1024 * 1024 # 1G |
| 69 | 69 | ||
| 70 | + MAX_FILE_NAME_LEN = 255 # Linux NAME_MAX，单文件名上限 | ||
| 71 | + MAPPING_CSV_FILE = "mapping.csv" # 超长文件名映射表，每行 {映射后},{映射前} | ||
| 72 | + | ||
| 70 | DIR_PLOG = 'plog' | 73 | DIR_PLOG = 'plog' |
| 71 | 74 | ||
| 72 | AIC_ERROR_TUPLE_LEN = 9 | 75 | AIC_ERROR_TUPLE_LEN = 9 |
| @@ -21,8 +21,10 @@ DumpDataParser class. This class mainly involves the parser_dump_data function. | |||
| 21 | Copyright Information: | 21 | Copyright Information: |
| 22 | Huawei Technologies Co., Ltd. All Rights Reserved © 2020 | 22 | Huawei Technologies Co., Ltd. All Rights Reserved © 2020 |
| 23 | """ | 23 | """ |
| 24 | +import csv | ||
| 24 | import json | 25 | import json |
| 25 | import os | 26 | import os |
| 27 | +import random | ||
| 26 | import struct | 28 | import struct |
| 27 | import ctypes | 29 | import ctypes |
| 28 | import traceback | 30 | import traceback |
| @@ -297,13 +299,46 @@ class DumpDataParser: | |||
| 297 | array = array.reshape(shape) | 299 | array = array.reshape(shape) |
| 298 | return array, np_dtype | 300 | return array, np_dtype |
| 299 | 301 | ||
| 302 | + def _load_name_mapping(self): | ||
| 303 | + """ | ||
| 304 | + Load the {original name: renamed name} mapping written by the dump framework. | ||
| 305 | + | ||
| 306 | + """ | ||
| 307 | + return utils.parse_name_mapping_csv(os.path.join(self.dump_path, Constant.MAPPING_CSV_FILE)) | ||
| 308 | + | ||
| 309 | + | ||
| 310 | + def _gen_random_numeric_name(file_dir, suffix): | ||
| 311 | + # 与异常dump框架一致，用随机数字串命名，保留后缀以便下游按扩展名读取 | ||
| 312 | + while True: | ||
| 313 | + name = str(random.randint(10 ** 15, 10 ** 16 - 1)) + suffix | ||
| 314 | + if not os.path.exists(os.path.join(file_dir, name)): | ||
| 315 | + return name | ||
| 316 | + | ||
| 317 | + | ||
| 318 | + def _record_mapping(file_dir, renamed, original_name): | ||
| 319 | + # 追加一行 {映射后随机数字串},{映射前文件名} 到同级mapping.csv | ||
| 320 | + mapping_csv = os.path.join(file_dir, Constant.MAPPING_CSV_FILE) | ||
| 321 | + with open(mapping_csv, 'a', newline='') as csv_file: | ||
| 322 | + csv.writer(csv_file).writerow([renamed, original_name]) | ||
| 323 | + | ||
| 324 | + def _check_file_name_len(self, dst_file_name): | ||
| 325 | + # NAME_MAX是字节上限，多字节文件名下需按编码后的字节数判断 | ||
| 326 | + if len(os.fsencode(os.path.basename(dst_file_name))) <= Constant.MAX_FILE_NAME_LEN: | ||
| 327 | + return dst_file_name | ||
| 328 | + file_dir, file_name = os.path.split(dst_file_name) | ||
| 329 | + _, suffix = os.path.splitext(file_name) | ||
| 330 | + renamed = self._gen_random_numeric_name(file_dir, suffix) | ||
| 331 | + self._record_mapping(file_dir, renamed, file_name) | ||
| 332 | + utils.print_warn_log(f"The output file name is too long, rename {file_name} to {renamed}.") | ||
| 333 | + return os.path.join(file_dir, renamed) | ||
| 334 | + | ||
| 300 | def _build_dst_file_name(self, dump_file_path, parse_type, index, dtype, np_dtype): | 335 | def _build_dst_file_name(self, dump_file_path, parse_type, index, dtype, np_dtype): |
| 301 | name_parts = [self.info.kernel_name, parse_type, str(index)] | 336 | name_parts = [self.info.kernel_name, parse_type, str(index)] |
| 302 | if dtype: | 337 | if dtype: |
| 303 | name_parts.append(dtype) | 338 | name_parts.append(dtype) |
| 304 | # numpy supported dtype is saved as npy, others keep the raw bin format | 339 | # numpy supported dtype is saved as npy, others keep the raw bin format |
| 305 | name_parts.append("npy" if np_dtype is not None else "bin") | 340 | name_parts.append("npy" if np_dtype is not None else "bin") |
| 306 | - return os.path.join(dump_file_path, ".".join(name_parts)) | 341 | + return self._check_file_name_len(os.path.join(dump_file_path, ".".join(name_parts))) |
| 307 | 342 | ||
| 308 | def _save_array(self, array, dst_file_name, parse_type, np_dtype): | 343 | def _save_array(self, array, dst_file_name, parse_type, np_dtype): |
| 309 | if np_dtype is not None: | 344 | if np_dtype is not None: |
| @@ -469,8 +504,14 @@ class DumpDataParser: | |||
| 469 | else: | 504 | else: |
| 470 | match_dump_list = [] | 505 | match_dump_list = [] |
| 471 | match_name = self.info.node_name | 506 | match_name = self.info.node_name |
| 507 | + name_mapping = self._load_name_mapping() | ||
| 508 | + if name_mapping: | ||
| 509 | + # 超长场景下落盘文件已被重命名，按原始data_name反查随机数字串 | ||
| 510 | + match_name = name_mapping.get(self.info.data_name or self.info.node_name, match_name) | ||
| 472 | for top, _, files in os.walk(self.dump_path): | 511 | for top, _, files in os.walk(self.dump_path): |
| 473 | for name in files: | 512 | for name in files: |
| 513 | + if name == Constant.MAPPING_CSV_FILE: | ||
| 514 | + continue | ||
| 474 | if match_name in name: | 515 | if match_name in name: |
| 475 | match_dump_list.append(os.path.join(top, name)) | 516 | match_dump_list.append(os.path.join(top, name)) |
| 476 | 517 | ||
| @@ -21,6 +21,7 @@ This file mainly involves the common function. | |||
| 21 | Copyright Information: | 21 | Copyright Information: |
| 22 | Huawei Technologies Co., Ltd. All Rights Reserved © 2020 | 22 | Huawei Technologies Co., Ltd. All Rights Reserved © 2020 |
| 23 | """ | 23 | """ |
| 24 | +import csv | ||
| 24 | import inspect | 25 | import inspect |
| 25 | import os | 26 | import os |
| 26 | import os.path | 27 | import os.path |
| @@ -297,6 +298,35 @@ def copy_src_to_dest(src_file_list: list, dest_path: str): | |||
| 297 | print_warn_log(f"Failed to copy {file} to {dest_file}. {error}.") | 298 | print_warn_log(f"Failed to copy {file} to {dest_file}. {error}.") |
| 298 | 299 | ||
| 299 | 300 | ||
| 301 | +def _iter_csv_rows(csv_path: str): | ||
| 302 | + """ | ||
| 303 | + yield the rows of a csv file one by one | ||
| 304 | + :param csv_path: the csv path | ||
| 305 | + """ | ||
| 306 | + # 显式指定utf-8，避免受locale影响；算子名可能含非utf-8字节，需兜住解码失败 | ||
| 307 | + with open(csv_path, 'r', encoding='utf-8') as csv_file: | ||
| 308 | + yield from csv.reader(csv_file) | ||
| 309 | + | ||
| 310 | + | ||
| 311 | +def parse_name_mapping_csv(csv_path: str) -> dict: | ||
| 312 | + """ | ||
| 313 | + parse the mapping.csv generated when a file name exceeds NAME_MAX, | ||
| 314 | + each line of which is {renamed random digits},{original file name} | ||
| 315 | + :param csv_path: the mapping.csv path | ||
| 316 | + :return: {original file name: renamed random digits}, empty when the csv is absent | ||
| 317 | + """ | ||
| 318 | + mapping = {} | ||
| 319 | + if not csv_path or not os.path.isfile(csv_path): | ||
| 320 | + return mapping | ||
| 321 | + try: | ||
| 322 | + for row in _iter_csv_rows(csv_path): | ||
| 323 | + if len(row) == 2: | ||
| 324 | + mapping[row[1].strip()] = row[0].strip() | ||
| 325 | + except (OSError, IOError, csv.Error, UnicodeDecodeError) as error: | ||
| 326 | + print_warn_log(f"Failed to read {csv_path}. {error}.") | ||
| 327 | + return mapping | ||
| 328 | + | ||
| 329 | + | ||
| 300 | def write_file(output_path: str, file_content: str, write_mode="w") -> None: | 330 | def write_file(output_path: str, file_content: str, write_mode="w") -> None: |
| 301 | """ | 331 | """ |
| 302 | write text to output file | 332 | write text to output file |
| @@ -18,6 +18,7 @@ | |||
| 18 | 18 | ||
| 19 | from ms_interface import utils | 19 | from ms_interface import utils |
| 20 | from ms_interface.collection import Collection | 20 | from ms_interface.collection import Collection |
| 21 | +from ms_interface.constant import Constant | ||
| 21 | import os | 22 | import os |
| 22 | import sys | 23 | import sys |
| 23 | import pytest | 24 | import pytest |
| @@ -331,6 +332,57 @@ class TestUtilsMethods(CommonAssert): | |||
| 331 | self.assertIn(utils.ExceptionRootCause().format_causes(), | 332 | self.assertIn(utils.ExceptionRootCause().format_causes(), |
| 332 | """Failed to get node name in plog. Cannot run L1 test""") | 333 | """Failed to get node name in plog. Cannot run L1 test""") |
| 333 | 334 | ||
| 335 | + def test_run_collect_oversize_data_name(self): | ||
| 336 | + """ | ||
| 337 | + 测试超长算子名场景：dump文件被框架重命名为随机数字串， | ||
| 338 | + collect通过mapping.csv找到实际落盘文件，并把mapping.csv一并收集 | ||
| 339 | + """ | ||
| 340 | + data_name = "a" * 250 + ".42.1.1726159207469285" | ||
| 341 | + rename = "1234567890123456" | ||
| 342 | + output_path = self.temp.joinpath(f"info_{CUR_TIME_STR}") | ||
| 343 | + input_path = self.temp.joinpath(f"asys_output_{CUR_TIME_STR}") | ||
| 344 | + input_path.mkdir(parents=True, exist_ok=True) | ||
| 345 | + dump_path = input_path.joinpath("extra-info/data-dump/0") | ||
| 346 | + dump_path.mkdir(parents=True, exist_ok=True) | ||
| 347 | + dump_path.joinpath(rename).touch() | ||
| 348 | + dump_path.joinpath("te_gatherv2.o").touch() | ||
| 349 | + dump_path.joinpath("te_gatherv2_host.o").touch() | ||
| 350 | + dump_path.joinpath(Constant.MAPPING_CSV_FILE).write_text(f"{rename},{data_name}\n") | ||
| 351 | + write_log_keyword_to_file(input_path, [ | ||
| 352 | + DUMP_EXCEPTION_STR, EXCEPTION_INFO_DUMP_ARGS_DATA, AICORE_KERNEL_EXECUTE_FAILED, | ||
| 353 | + '[ERROR] IDEDD(1592077,python3):2024-09-12-16:40:08.360.226 [dump_args.cpp:807]' | ||
| 354 | + '[tid:1592077] [1] dump exception to file, file: ' | ||
| 355 | + f'./new/extra-info/data-dump/0/{data_name}']) | ||
| 356 | + collection = Collection(input_path, output_path) | ||
| 357 | + res = collection.collect() | ||
| 358 | + self.assertEqual(res, True) | ||
| 359 | + self.assertEqual(collection.dump_file_rename, rename) | ||
| 360 | + self.assertEqual(bool(list(output_path.rglob(f"collection/dump/{rename}"))), True) | ||
| 361 | + self.assertEqual( | ||
| 362 | + bool(list(output_path.rglob(f"collection/dump/{Constant.MAPPING_CSV_FILE}"))), True) | ||
| 363 | + self.assertIn(self.debug_info.read_text(encoding="utf-8"), | ||
| 364 | + f"use mapped name {rename} instead of") | ||
| 365 | + | ||
| 366 | + def test_run_collect_oversize_data_name_no_mapping(self): | ||
| 367 | + """ | ||
| 368 | + 测试超长算子名但缺失mapping.csv：退化为按原始名查找，报dump文件找不到 | ||
| 369 | + """ | ||
| 370 | + data_name = "a" * 250 + ".42.1.1726159207469285" | ||
| 371 | + output_path = self.temp.joinpath(f"info_{CUR_TIME_STR}") | ||
| 372 | + input_path = self.temp.joinpath(f"asys_output_{CUR_TIME_STR}") | ||
| 373 | + input_path.mkdir(parents=True, exist_ok=True) | ||
| 374 | + write_log_keyword_to_file(input_path, [ | ||
| 375 | + DUMP_EXCEPTION_STR, EXCEPTION_INFO_DUMP_ARGS_DATA, | ||
| 376 | + '[ERROR] IDEDD(1592077,python3):2024-09-12-16:40:08.360.226 [dump_args.cpp:807]' | ||
| 377 | + '[tid:1592077] [1] dump exception to file, file: ' | ||
| 378 | + f'./new/extra-info/data-dump/0/{data_name}']) | ||
| 379 | + collection = Collection(input_path, output_path) | ||
| 380 | + res = collection.collect() | ||
| 381 | + self.assertEqual(res, False) | ||
| 382 | + self.assertEqual(collection.dump_file_rename, "") | ||
| 383 | + self.assertIn(self.debug_info.read_text(encoding="utf-8"), | ||
| 384 | + f"Cannot find dump file {data_name}") | ||
| 385 | + | ||
| 334 | def test_get_node_and_kernel_name_l1_get_node_name_have_multiple_dump(self): | 386 | def test_get_node_and_kernel_name_l1_get_node_name_have_multiple_dump(self): |
| 335 | """ | 387 | """ |
| 336 | 测试L1无法找到node_name失败报错 | 388 | 测试L1无法找到node_name失败报错 |
| @@ -29,6 +29,7 @@ import pytest | |||
| 29 | sys.path.append(MSAICERR_PATH) | 29 | sys.path.append(MSAICERR_PATH) |
| 30 | 30 | ||
| 31 | from ms_interface.aic_error_info import AicErrorInfo | 31 | from ms_interface.aic_error_info import AicErrorInfo |
| 32 | +from ms_interface.constant import Constant | ||
| 32 | from ms_interface.dump_data_parser import DumpDataParser, BigDumpDataParser | 33 | from ms_interface.dump_data_parser import DumpDataParser, BigDumpDataParser |
| 33 | 34 | ||
| 34 | dump_file = "exception_info.2.1.20250609144925349" | 35 | dump_file = "exception_info.2.1.20250609144925349" |
| @@ -51,6 +52,18 @@ class Selfliberr(): | |||
| 51 | return 1 | 52 | return 1 |
| 52 | 53 | ||
| 53 | 54 | ||
| 55 | +class SelflibWriteJson(): | ||
| 56 | + """在被调用时才生成json，模拟解析so的落盘时机""" | ||
| 57 | + | ||
| 58 | + def __init__(self, dump_json): | ||
| 59 | + self.dump_json = dump_json | ||
| 60 | + | ||
| 61 | + def ParseDumpProtoToJson(self, data_ptr, data_size, path_ptr): | ||
| 62 | + with open(path_ptr.decode('utf-8'), 'w') as json_file: | ||
| 63 | + json_file.write(json.dumps(self.dump_json)) | ||
| 64 | + return 0 | ||
| 65 | + | ||
| 66 | + | ||
| 54 | class TestUtilsMethods(CommonAssert): | 67 | class TestUtilsMethods(CommonAssert): |
| 55 | 68 | ||
| 56 | def change_test_dir(self, tmp_path): | 69 | def change_test_dir(self, tmp_path): |
| @@ -242,6 +255,49 @@ class TestUtilsMethods(CommonAssert): | |||
| 242 | assert parser.get_bin_data()[0].endswith("input.0.hifloat8.bin") | 255 | assert parser.get_bin_data()[0].endswith("input.0.hifloat8.bin") |
| 243 | self.assertIn(res, "If dtype is hifloat8, summary is: ") | 256 | self.assertIn(res, "If dtype is hifloat8, summary is: ") |
| 244 | 257 | ||
| 258 | + def test_parser_dump_file_by_mapped_name(self, mocker): | ||
| 259 | + """超长场景: dump目录下只有随机名文件和mapping.csv，按原始data_name反查映射名完成解析""" | ||
| 260 | + data_name = "a" * 250 + ".42.1.1726159207469285" | ||
| 261 | + rename = "1234567890123456" | ||
| 262 | + dump_dir = self.temp.joinpath("dump") | ||
| 263 | + dump_dir.mkdir(parents=True, exist_ok=True) | ||
| 264 | + dump_json = self._make_dump_json(output_data_type=1, output_size='8', output_dim=['2']) | ||
| 265 | + # 解析so会在dump文件同级生成json，此处在调用时写入，避免提前污染待匹配目录 | ||
| 266 | + mocker.patch('ctypes.CDLL', return_value=SelflibWriteJson(dump_json)) | ||
| 267 | + create_dump_file(str(dump_dir.joinpath(rename)), 10, 200) | ||
| 268 | + dump_dir.joinpath(Constant.MAPPING_CSV_FILE).write_text(f"{rename},{data_name}\n") | ||
| 269 | + info = AicErrorInfo() | ||
| 270 | + info.node_name = "GatherV2" # L1场景下node_name是plog中的短名，与映射key不同 | ||
| 271 | + info.data_name = data_name | ||
| 272 | + info.json_file = str(RES_PATH.joinpath("ori_data/collect_json/test.json")) | ||
| 273 | + parser = DumpDataParser(str(dump_dir), info) | ||
| 274 | + parser.parse() | ||
| 275 | + # 命中随机名文件，且mapping.csv没有被当成dump文件解析 | ||
| 276 | + self.assertEqual(info.dump_file, [str(dump_dir.joinpath(rename))]) | ||
| 277 | + self.assertIn(info.dump_info, f"{rename}.output.0.float32.npy") | ||
| 278 | + | ||
| 279 | + def test_parser_dump_file_oversize_result_renamed(self, mocker): | ||
| 280 | + """超长场景: 解析结果文件名超过NAME_MAX时重命名为随机数字串，并记录到同级mapping.csv""" | ||
| 281 | + dump_json = self._make_dump_json(output_data_type=1, output_size='8', output_dim=['2']) | ||
| 282 | + self.common_mock(mocker, dump_json) | ||
| 283 | + create_dump_file(dump_file, 10, 200) | ||
| 284 | + info = AicErrorInfo() | ||
| 285 | + info.kernel_name = "a" * 260 # 拼装后的结果文件名必然超长 | ||
| 286 | + info.json_file = str(RES_PATH.joinpath("ori_data/collect_json/test.json")) | ||
| 287 | + parser = DumpDataParser(dump_file, info) | ||
| 288 | + parser.parse() | ||
| 289 | + npy_files = [f for f in parser.get_bin_data() if f.endswith(".npy")] | ||
| 290 | + self.assertEqual(len(npy_files), 1) | ||
| 291 | + file_name = os.path.basename(npy_files[0]) | ||
| 292 | + # 落盘成功，文件名为随机数字串 + 原后缀 | ||
| 293 | + assert len(file_name) <= Constant.MAX_FILE_NAME_LEN | ||
| 294 | + assert file_name[:-len(".npy")].isdigit() | ||
| 295 | + self.assertEqual(os.path.isfile(npy_files[0]), True) | ||
| 296 | + self.assertEqual(str(np.load(npy_files[0]).dtype), "float32") | ||
| 297 | + # mapping.csv中记录了 {映射后},{映射前} | ||
| 298 | + mapping_text = self.temp.joinpath(Constant.MAPPING_CSV_FILE).read_text() | ||
| 299 | + self.assertIn(mapping_text, f"{file_name},{info.kernel_name}.output.0.float32.npy") | ||
| 300 | + | ||
| 245 | def test_parser_dump_file_bfloat16_dtype_success(self, mocker): | 301 | def test_parser_dump_file_bfloat16_dtype_success(self, mocker): |
| 246 | dump_json = {'version': '2.0', 'dump_time': '1749451765349986', 'output': [{'data_type': 27, 'format': 0, 'shape': {'dim': ['2', '2048']}, 'data': '', 'size': '10', 'sub_format': 0, 'address': '0', 'dim_range': [], 'offset': '3'}], 'input': [{'data_type': 0, 'format': 0, 'shape': {'dim': ['10240', '2048']}, 'data': '', 'size': '10', 'sub_format': 0, 'address': '0', 'offset': '0', 'arg_index': 0, 'input_type': 2}, {'data_type': 0, 'format': 0, 'shape': {'dim': ['2']}, 'data': '', 'size': '32', 'sub_format': 0, 'address': '0', 'offset': '0', 'arg_index': 1, 'input_type': 2}, {'data_type': 0, 'format': 0, 'shape': {'dim': ['1']}, 'data': '', 'size': '32', 'sub_format': 0, 'address': '0', 'offset': '0', 'arg_index': 2, 'input_type': 2}, {'data_type': 0, 'format': 0, 'data': '', 'size': '10', 'sub_format': 0, 'address': '0', 'offset': '0', 'arg_index': 5, 'input_type': 7}], 'buffer': [], 'op_name': '', 'attr': [], 'space': [{'type': 0, 'data': '', 'size': '10'}], 'dfx_message': '[AIC_INFO] args(0 to 20) after execute:0x12c200000000, 0x12d340000000, 0x12c1c0000518, 0x12d340000200, 0x12d340004400, 0x12c1c0000438, 0x12c100011000, 0x285a, 0x2, 0x1, 0, 0x2000, 0x8, 0x1, 0x1, 0x2800, 0x2, 0x800, 0x1, 0x1, \n[AIC_INFO] args(20 to 39) after execute:0x2, 0x1, 0x1, 0x1, 0x1, 0x800, 0x1, 0x1, 0x2, 0x1, 0x2, 0xa5a5a5a500000000, 0, 0, 0, 0, 0, 0, 0, \n[Dump][Exception] begin to load normal tensor, index:0\n[Dump][Exception] exception info dump args data, addr:0x12c200000000; size:83886080 bytes\n[Dump][Exception] end to load normal tensor, index:0\n[Dump][Exception] begin to load normal tensor, index:1\n[Dump][Exception] exception info dump args data, addr:0x12d340000000; size:32 bytes\n[Dump][Exception] end to load normal tensor, index:1\n[Dump][Exception] begin to load normal tensor, index:2\n[Dump][Exception] exception info dump args data, addr:0x12c1c0000518; size:32 bytes\n[Dump][Exception] end to load normal tensor, index:2\n[Dump][Exception] begin to load normal tensor, index:3\n[Dump][Exception] exception info dump args data, addr:0x12d340000200; size:16384 bytes\n[Dump][Exception] end to load normal tensor, index:3\n[Dump][Exception] exception info dump args data, addr:0x12d340004400; size:76832 bytes\n[Dump][Exception] exception info dump args data, addr:0x12c1c0000438; size:200 bytes\n'} | 302 | dump_json = {'version': '2.0', 'dump_time': '1749451765349986', 'output': [{'data_type': 27, 'format': 0, 'shape': {'dim': ['2', '2048']}, 'data': '', 'size': '10', 'sub_format': 0, 'address': '0', 'dim_range': [], 'offset': '3'}], 'input': [{'data_type': 0, 'format': 0, 'shape': {'dim': ['10240', '2048']}, 'data': '', 'size': '10', 'sub_format': 0, 'address': '0', 'offset': '0', 'arg_index': 0, 'input_type': 2}, {'data_type': 0, 'format': 0, 'shape': {'dim': ['2']}, 'data': '', 'size': '32', 'sub_format': 0, 'address': '0', 'offset': '0', 'arg_index': 1, 'input_type': 2}, {'data_type': 0, 'format': 0, 'shape': {'dim': ['1']}, 'data': '', 'size': '32', 'sub_format': 0, 'address': '0', 'offset': '0', 'arg_index': 2, 'input_type': 2}, {'data_type': 0, 'format': 0, 'data': '', 'size': '10', 'sub_format': 0, 'address': '0', 'offset': '0', 'arg_index': 5, 'input_type': 7}], 'buffer': [], 'op_name': '', 'attr': [], 'space': [{'type': 0, 'data': '', 'size': '10'}], 'dfx_message': '[AIC_INFO] args(0 to 20) after execute:0x12c200000000, 0x12d340000000, 0x12c1c0000518, 0x12d340000200, 0x12d340004400, 0x12c1c0000438, 0x12c100011000, 0x285a, 0x2, 0x1, 0, 0x2000, 0x8, 0x1, 0x1, 0x2800, 0x2, 0x800, 0x1, 0x1, \n[AIC_INFO] args(20 to 39) after execute:0x2, 0x1, 0x1, 0x1, 0x1, 0x800, 0x1, 0x1, 0x2, 0x1, 0x2, 0xa5a5a5a500000000, 0, 0, 0, 0, 0, 0, 0, \n[Dump][Exception] begin to load normal tensor, index:0\n[Dump][Exception] exception info dump args data, addr:0x12c200000000; size:83886080 bytes\n[Dump][Exception] end to load normal tensor, index:0\n[Dump][Exception] begin to load normal tensor, index:1\n[Dump][Exception] exception info dump args data, addr:0x12d340000000; size:32 bytes\n[Dump][Exception] end to load normal tensor, index:1\n[Dump][Exception] begin to load normal tensor, index:2\n[Dump][Exception] exception info dump args data, addr:0x12c1c0000518; size:32 bytes\n[Dump][Exception] end to load normal tensor, index:2\n[Dump][Exception] begin to load normal tensor, index:3\n[Dump][Exception] exception info dump args data, addr:0x12d340000200; size:16384 bytes\n[Dump][Exception] end to load normal tensor, index:3\n[Dump][Exception] exception info dump args data, addr:0x12d340004400; size:76832 bytes\n[Dump][Exception] exception info dump args data, addr:0x12c1c0000438; size:200 bytes\n'} |
| 247 | self.common_mock(mocker, dump_json) | 303 | self.common_mock(mocker, dump_json) |
| @@ -565,6 +565,375 @@ class TestUtilsMethods(CommonAssert): | |||||||
| 565 | res = collection.collect_kernel_file(kernel_name1) | 565 | res = collection.collect_kernel_file(kernel_name1) | ||||
| 566 | self.assertEqual('', res) | 566 | self.assertEqual('', res) | ||||
| 567 | 567 | ||||||
| 568 | + | ||||||
| 569 | + "name_len, expected", | ||||||
| 570 | + [ | ||||||
| 571 | + (Constant.MAX_FILE_NAME_LEN, False), # 等于上限，不算超长 | ||||||
| 572 | + (Constant.MAX_FILE_NAME_LEN + 1, True) # 超过上限 | ||||||
| 573 | + ] | ||||||
| 574 | + ) | ||||||
| 575 | + def test_is_oversize_name(self, name_len, expected): | ||||||
| 576 | + """ | ||||||
| 577 | + 测试超长名字长度初判的边界 | ||||||
| 578 | + """ | ||||||
| 579 | + collection = Collection(self.temp, self.temp) | ||||||
| 580 | + self.assertEqual(collection._is_oversize_name("a" * name_len), expected) | ||||||
| 581 | + | ||||||
| 582 | + def test_get_dump_mapping_csv_path(self): | ||||||
| 583 | + """ | ||||||
| 584 | + 测试按device_id定位data-dump目录下的mapping.csv | ||||||
| 585 | + """ | ||||||
| 586 | + output_path = self.temp.joinpath(f"info_{CUR_TIME_STR}") | ||||||
| 587 | + input_path = self.temp.joinpath(f"asys_output_{CUR_TIME_STR}") | ||||||
| 588 | + for device_id in ("0", "1"): | ||||||
| 589 | + dump_path = input_path.joinpath(f"extra-info/data-dump/{device_id}") | ||||||
| 590 | + dump_path.mkdir(parents=True, exist_ok=True) | ||||||
| 591 | + dump_path.joinpath(Constant.MAPPING_CSV_FILE).write_text(f"1234,{device_id}\n") | ||||||
| 592 | + collection = Collection(input_path, output_path) | ||||||
| 593 | + res = collection._get_dump_mapping_csv_path("1") | ||||||
| 594 | + self.assertIn(res, "data-dump/1/mapping.csv") | ||||||
 🟠 High Priority
同一文件中存在 6 处相同模式的错误： 后果：这些断言要么永远通过（若返回值为空串或碰巧是短串的子串），要么永远失败（若返回值是完整路径），无法验证实际意图。 建议：交换 改动建议
不准确？ | |||||||
| 595 | + | ||||||
| 596 | + def test_get_dump_mapping_csv_path_device_id_prefix(self): | ||||||
| 597 | + """ | ||||||
| 598 | + 测试device_id为0时不会误命中data-dump/01/下的mapping.csv | ||||||
| 599 | + """ | ||||||
| 600 | + output_path = self.temp.joinpath(f"info_{CUR_TIME_STR}") | ||||||
| 601 | + input_path = self.temp.joinpath(f"asys_output_{CUR_TIME_STR}") | ||||||
| 602 | + for device_id in ("01", "0"): | ||||||
| 603 | + dump_path = input_path.joinpath(f"extra-info/data-dump/{device_id}") | ||||||
| 604 | + dump_path.mkdir(parents=True, exist_ok=True) | ||||||
| 605 | + dump_path.joinpath(Constant.MAPPING_CSV_FILE).write_text(f"1234,{device_id}\n") | ||||||
| 606 | + collection = Collection(input_path, output_path) | ||||||
| 607 | + res = collection._get_dump_mapping_csv_path("0") | ||||||
| 608 | + self.assertIn(res, "data-dump/0/mapping.csv") | ||||||
| 609 | + self.assertNotIn(res, "data-dump/01/mapping.csv") | ||||||
| 610 | + | ||||||
| 611 | + def test_get_dump_mapping_csv_path_not_exist(self): | ||||||
| 612 | + """ | ||||||
| 613 | + 测试report_path下没有mapping.csv时返回空 | ||||||
| 614 | + """ | ||||||
| 615 | + output_path = self.temp.joinpath(f"info_{CUR_TIME_STR}") | ||||||
| 616 | + input_path = self.temp.joinpath(f"asys_output_{CUR_TIME_STR}") | ||||||
| 617 | + input_path.mkdir(parents=True, exist_ok=True) | ||||||
| 618 | + collection = Collection(input_path, output_path) | ||||||
| 619 | + self.assertEqual(collection._get_dump_mapping_csv_path("0"), "") | ||||||
| 620 | + | ||||||
| 621 | + def test_get_dump_mapping_csv_path_other_device_only(self): | ||||||
| 622 | + """ | ||||||
| 623 | + 测试只有其它device的mapping.csv时返回空，不回退使用别的卡的映射表 | ||||||
| 624 | + """ | ||||||
| 625 | + output_path = self.temp.joinpath(f"info_{CUR_TIME_STR}") | ||||||
| 626 | + input_path = self.temp.joinpath(f"asys_output_{CUR_TIME_STR}") | ||||||
| 627 | + for device_id in ("1", "2"): | ||||||
| 628 | + dump_path = input_path.joinpath(f"extra-info/data-dump/{device_id}") | ||||||
| 629 | + dump_path.mkdir(parents=True, exist_ok=True) | ||||||
| 630 | + dump_path.joinpath(Constant.MAPPING_CSV_FILE).write_text(f"1234,{device_id}\n") | ||||||
| 631 | + collection = Collection(input_path, output_path) | ||||||
| 632 | + self.assertEqual(collection._get_dump_mapping_csv_path("0"), "") | ||||||
| 633 | + self.assertIn(self.debug_info.read_text(), | ||||||
| 634 | + f"{Constant.MAPPING_CSV_FILE} of device 0 cannot be found in") | ||||||
| 635 | + | ||||||
| 636 | + def test_collect_oversize_scene_other_device_mapping_only(self): | ||||||
| 637 | + """ | ||||||
| 638 | + 测试报错device无mapping.csv、其它device有时，不会误用其映射名收集到别的卡的dump。 | ||||||
| 639 | + 退化为按原始名查找，dump文件找不到时collect返回False | ||||||
| 640 | + """ | ||||||
| 641 | + data_name = "a" * 250 + ".42.1.1726159207469285" | ||||||
| 642 | + rename = "1234567890123456" | ||||||
| 643 | + output_path = self.temp.joinpath(f"info_{CUR_TIME_STR}") | ||||||
| 644 | + input_path = self.temp.joinpath(f"asys_output_{CUR_TIME_STR}") | ||||||
| 645 | + # 报错device为0，但只有device1落了随机名dump和mapping.csv | ||||||
| 646 | + dump_path = input_path.joinpath("extra-info/data-dump/1") | ||||||
| 647 | + dump_path.mkdir(parents=True, exist_ok=True) | ||||||
| 648 | + dump_path.joinpath(rename).touch() | ||||||
| 649 | + dump_path.joinpath(Constant.MAPPING_CSV_FILE).write_text(f"{rename},{data_name}\n") | ||||||
| 650 | + input_path.joinpath("plog.txt").write_text( | ||||||
| 651 | + "[ERROR] IDEDD(1592077,python3):2024-09-12-16:40:08.360.226 [dump_args.cpp:807]" | ||||||
| 652 | + "[tid:1592077] [Dump][Exception] dump exception to file, file: " | ||||||
| 653 | + f"./new/extra-info/data-dump/0/{data_name}") | ||||||
| 654 | + collection = Collection(input_path, output_path) | ||||||
| 655 | + self.assertEqual(collection.collect(), False) | ||||||
| 656 | + self.assertEqual(collection.dump_file_rename, "") | ||||||
| 657 | + # device1的dump文件没有被误收集 | ||||||
| 658 | + self.assertEqual(bool(list(output_path.rglob(f'collection/dump/{rename}'))), False) | ||||||
| 659 | + | ||||||
| 660 | + def test_resolve_dump_file_rename_normal_name(self, mocker): | ||||||
| 661 | + """ | ||||||
| 662 | + 测试普通名字走快路径，不触发find mapping.csv | ||||||
| 663 | + """ | ||||||
| 664 | + collection = Collection(self.temp, self.temp) | ||||||
| 665 | + get_csv = mocker.patch.object(collection, "_get_dump_mapping_csv_path") | ||||||
| 666 | + self.assertEqual(collection._resolve_dump_file_rename("0", "short_name"), "") | ||||||
| 667 | + self.assertEqual(get_csv.called, False) | ||||||
| 668 | + | ||||||
| 669 | + def test_resolve_dump_file_rename_oversize_matched(self, mocker): | ||||||
| 670 | + """ | ||||||
| 671 | + 测试超长名字命中映射，返回映射后的随机数字串 | ||||||
| 672 | + """ | ||||||
| 673 | + data_name = "a" * 250 + ".42.1.1726159207469285" | ||||||
| 674 | + collection = Collection(self.temp, self.temp) | ||||||
| 675 | + mocker.patch.object(collection, "_get_dump_mapping_csv_path", return_value="mapping.csv") | ||||||
| 676 | + mocker.patch.object(utils, "parse_name_mapping_csv", | ||||||
| 677 | + return_value={data_name: "1234567890123456"}) | ||||||
| 678 | + self.assertEqual(collection._resolve_dump_file_rename("0", data_name), "1234567890123456") | ||||||
| 679 | + self.assertEqual(collection.dump_file_rename, "1234567890123456") | ||||||
| 680 | + | ||||||
| 681 | + def test_is_oversize_name_multi_byte(self): | ||||||
| 682 | + """ | ||||||
| 683 | + 测试多字节文件名按字节数判定，字符数未超但字节数已超 | ||||||
| 684 | + """ | ||||||
| 685 | + collection = Collection(self.temp, self.temp) | ||||||
| 686 | + name = "算" * 100 # 100个字符，UTF-8编码为300字节 | ||||||
| 687 | + self.assertEqual(len(name) > Constant.MAX_FILE_NAME_LEN, False) | ||||||
| 688 | + self.assertEqual(collection._is_oversize_name(name), True) | ||||||
| 689 | + | ||||||
| 690 | + def test_resolve_dump_file_rename_oversize_not_matched(self, mocker): | ||||||
| 691 | + """ | ||||||
| 692 | + 测试超长名字未命中映射，返回空串退化为原始名，并打印未命中原因 | ||||||
| 693 | + """ | ||||||
| 694 | + data_name = "a" * 250 + ".42.1.1726159207469285" | ||||||
| 695 | + collection = Collection(self.temp, self.temp) | ||||||
| 696 | + mocker.patch.object(collection, "_get_dump_mapping_csv_path", return_value="mapping.csv") | ||||||
| 697 | + mocker.patch.object(utils, "parse_name_mapping_csv", return_value={"other": "1234"}) | ||||||
| 698 | + self.assertEqual(collection._resolve_dump_file_rename("0", data_name), "") | ||||||
| 699 | + self.assertIn(self.debug_info.read_text(), "it is not recorded in mapping.csv") | ||||||
| 700 | + | ||||||
| 701 | + def test_resolve_dump_file_rename_oversize_no_mapping_csv(self, mocker): | ||||||
| 702 | + """ | ||||||
| 703 | + 测试超长名字但mapping.csv缺失，日志可区分于"有mapping.csv但未命中" | ||||||
| 704 | + """ | ||||||
| 705 | + data_name = "a" * 250 + ".42.1.1726159207469285" | ||||||
| 706 | + collection = Collection(self.temp, self.temp) | ||||||
| 707 | + mocker.patch.object(collection, "_get_dump_mapping_csv_path", return_value="") | ||||||
| 708 | + self.assertEqual(collection._resolve_dump_file_rename("0", data_name), "") | ||||||
| 709 | + self.assertIn(self.debug_info.read_text(), | ||||||
| 710 | + f"but {Constant.MAPPING_CSV_FILE} cannot be found in") | ||||||
| 711 | + | ||||||
| 712 | + def test_check_dump_data_is_valid_with_rename(self): | ||||||
| 713 | + """ | ||||||
| 714 | + 测试超长场景下按映射名校验dump文件存在性 | ||||||
| 715 | + """ | ||||||
| 716 | + data_name = "a" * 250 + ".42.1.1726159207469285" | ||||||
| 717 | + rename = "1234567890123456" | ||||||
| 718 | + output_path = self.temp.joinpath(f"info_{CUR_TIME_STR}") | ||||||
| 719 | + input_path = self.temp.joinpath(f"asys_output_{CUR_TIME_STR}") | ||||||
| 720 | + dump_path = input_path.joinpath("extra-info/data-dump/0") | ||||||
| 721 | + dump_path.mkdir(parents=True, exist_ok=True) | ||||||
| 722 | + dump_path.joinpath(rename).touch() | ||||||
| 723 | + collection = Collection(input_path, output_path) | ||||||
| 724 | + # 按映射名可以找到，不抛异常 | ||||||
| 725 | + collection.check_dump_data_is_valid("2024-09-12-16:40:08.360.226", data_name, rename) | ||||||
| 726 | + # 不传rename时按原始名查找，找不到 | ||||||
| 727 | + with pytest.raises(utils.AicErrException) as e: | ||||||
| 728 | + collection.check_dump_data_is_valid("2024-09-12-16:40:08.360.226", data_name) | ||||||
| 729 | + self.assertEqual(str(e), str(Constant.MS_AICERR_INVALID_DUMP_DATA_ERROR)) | ||||||
🟠 High Priority L688-690 的代码结构为： with pytest.raises(utils.AicErrException) as e: collection.check_dump_data_is_valid("2024-09-12-16:40:08.360.226", data_name) # L689 self.assertEqual(str(e), str(Constant.MS_AICERR_INVALID_DUMP_DATA_ERROR)) # L690 若 L689 抛出异常，控制流立即跳出 另外，即使把断言移到 正确写法： with pytest.raises(utils.AicErrException) as e: collection.check_dump_data_is_valid("2024-09-12-16:40:08.360.226", data_name) self.assertEqual(str(e.value), str(Constant.MS_AICERR_INVALID_DUMP_DATA_ERROR)) 建议：将 不准确？ | |||||||
| 730 | + | ||||||
| 731 | + def test_check_host_and_device_kernel_name_with_rename(self): | ||||||
| 732 | + """ | ||||||
| 733 | + 测试超长场景下用映射名定位dump文件所在目录，目录内.o校验逻辑不变 | ||||||
| 734 | + """ | ||||||
| 735 | + data_name = "a" * 250 + ".42.1.1726159207469285" | ||||||
| 736 | + rename = "1234567890123456" | ||||||
| 737 | + output_path = self.temp.joinpath(f"info_{CUR_TIME_STR}") | ||||||
| 738 | + input_path = self.temp.joinpath(f"asys_output_{CUR_TIME_STR}/dump") | ||||||
| 739 | + input_path.mkdir(parents=True, exist_ok=True) | ||||||
| 740 | + input_path.joinpath(rename).touch() | ||||||
| 741 | + input_path.joinpath("te_gatherv2.o").touch() | ||||||
| 742 | + input_path.joinpath("te_gatherv2_host.o").touch() | ||||||
| 743 | + collection = Collection(input_path, output_path) | ||||||
| 744 | + self.assertEqual(collection.check_host_and_device_kernel_name(data_name, rename), True) | ||||||
| 745 | + | ||||||
| 746 | + def test_collect_data_dump_with_rename(self): | ||||||
| 747 | + """ | ||||||
| 748 | + 测试超长场景下按映射名收集dump文件，并一并收集mapping.csv | ||||||
| 749 | + """ | ||||||
| 750 | + data_name = "a" * 250 + ".42.1.1726159207469285" | ||||||
| 751 | + rename = "1234567890123456" | ||||||
| 752 | + output_path = self.temp.joinpath(f"info_{CUR_TIME_STR}") | ||||||
| 753 | + input_path = self.temp.joinpath(f"asys_output_{CUR_TIME_STR}") | ||||||
| 754 | + dump_path = input_path.joinpath("extra-info/data-dump/0") | ||||||
| 755 | + dump_path.mkdir(parents=True, exist_ok=True) | ||||||
| 756 | + dump_path.joinpath(rename).touch() | ||||||
| 757 | + dump_path.joinpath(Constant.MAPPING_CSV_FILE).write_text(f"{rename},{data_name}\n") | ||||||
| 758 | + collection = Collection(input_path, output_path) | ||||||
| 759 | + collection.collect_data_dump("0", data_name, rename) | ||||||
| 760 | + self.assertEqual( | ||||||
| 761 | + bool(list(output_path.rglob(f'collection/dump/{rename}'))), True) | ||||||
| 762 | + self.assertEqual( | ||||||
| 763 | + bool(list(output_path.rglob(f'collection/dump/{Constant.MAPPING_CSV_FILE}'))), True) | ||||||
| 764 | + | ||||||
| 765 | + def test_collect_data_dump_with_rename_multiple(self, mocker): | ||||||
| 766 | + """ | ||||||
| 767 | + 测试超长场景下找到多个dump文件时，回查plog的grep关键字用原始名 | ||||||
| 768 | + """ | ||||||
| 769 | + data_name = "a" * 250 + ".42.1.1726159207469285" | ||||||
| 770 | + rename = "1234567890123456" | ||||||
| 771 | + output_path = self.temp.joinpath(f"info_{CUR_TIME_STR}") | ||||||
| 772 | + collection_plog_path = output_path.joinpath('collection/plog') | ||||||
| 773 | + collection_plog_path.mkdir(parents=True, exist_ok=True) | ||||||
| 774 | + collection_plog_path.joinpath("dump.log").write_text( | ||||||
| 775 | + f"extra-info/data-dump/0/{data_name}") | ||||||
| 776 | + input_path = self.temp.joinpath(f"asys_output_{CUR_TIME_STR}") | ||||||
| 777 | + for device_id in ("0", "1"): | ||||||
| 778 | + dump_path = input_path.joinpath(f"extra-info/data-dump/{device_id}") | ||||||
| 779 | + dump_path.mkdir(parents=True, exist_ok=True) | ||||||
| 780 | + dump_path.joinpath(rename).touch() | ||||||
| 781 | + collection = Collection(input_path, output_path) | ||||||
| 782 | + collection.collect_data_dump("0", data_name, rename) | ||||||
| 783 | + self.assertEqual( | ||||||
| 784 | + bool(list(output_path.rglob(f'collection/dump/{rename}'))), True) | ||||||
| 785 | + self.assertIn(self.debug_info.read_text(), f"Find dump file {rename}.") | ||||||
| 786 | + | ||||||
| 787 | + def test_collect_data_dump_multiple_no_plog_match(self): | ||||||
| 788 | + """ | ||||||
| 789 | + 测试找到多个dump文件但plog中查不到data-dump记录时不抛IndexError | ||||||
| 790 | + """ | ||||||
| 791 | + data_name = "GatherV2.GatherV21.1.1733469426252033" | ||||||
| 792 | + output_path = self.temp.joinpath(f"info_{CUR_TIME_STR}") | ||||||
| 793 | + collection_plog_path = output_path.joinpath('collection/plog') | ||||||
| 794 | + collection_plog_path.mkdir(parents=True, exist_ok=True) | ||||||
| 795 | + collection_plog_path.joinpath("dump.log").write_text("no data-dump record here") | ||||||
| 796 | + input_path = self.temp.joinpath(f"asys_output_{CUR_TIME_STR}") | ||||||
| 797 | + for device_id in ("0", "1"): | ||||||
| 798 | + dump_path = input_path.joinpath(f"extra-info/data-dump/{device_id}") | ||||||
| 799 | + dump_path.mkdir(parents=True, exist_ok=True) | ||||||
| 800 | + dump_path.joinpath(data_name).touch() | ||||||
| 801 | + collection = Collection(input_path, output_path) | ||||||
| 802 | + res = collection.collect_data_dump("0", data_name) | ||||||
| 803 | + self.assertEqual(res, os.path.join(str(output_path), "collection", "dump")) | ||||||
| 804 | + self.assertEqual( | ||||||
| 805 | + bool(list(output_path.rglob(f'collection/dump/{data_name}'))), True) | ||||||
| 806 | + | ||||||
| 807 | + | ||||||
| 808 | + def test_collect_data_dump_multiple_picks_target_device(self, target_device): | ||||||
| 809 | + """ | ||||||
| 810 | + 测试多个device目录下存在同名dump文件时，按报错device筛选。 | ||||||
| 811 | + 两个device各跑一次，结果必须各自命中，不能受目录遍历顺序影响 | ||||||
| 812 | + """ | ||||||
| 813 | + data_name = "GatherV2.GatherV21.1.1733469426252033" | ||||||
| 814 | + output_path = self.temp.joinpath(f"info_{CUR_TIME_STR}_{target_device}") | ||||||
| 815 | + collection_plog_path = output_path.joinpath('collection/plog') | ||||||
| 816 | + collection_plog_path.mkdir(parents=True, exist_ok=True) | ||||||
| 817 | + collection_plog_path.joinpath("dump.log").write_text( | ||||||
| 818 | + f"extra-info/data-dump/{target_device}/{data_name}") | ||||||
| 819 | + input_path = self.temp.joinpath(f"asys_output_{CUR_TIME_STR}") | ||||||
| 820 | + for device_id in ("0", "1"): | ||||||
| 821 | + dump_path = input_path.joinpath(f"extra-info/data-dump/{device_id}") | ||||||
| 822 | + dump_path.mkdir(parents=True, exist_ok=True) | ||||||
| 823 | + dump_path.joinpath(data_name).write_text(f"device{device_id}") | ||||||
| 824 | + collection = Collection(input_path, output_path) | ||||||
| 825 | + collection.collect_data_dump(target_device, data_name) | ||||||
| 826 | + collected = list(output_path.rglob(f'collection/dump/{data_name}')) | ||||||
| 827 | + # 只收集报错device的那一份 | ||||||
| 828 | + self.assertEqual(len(collected), 1) | ||||||
| 829 | + self.assertEqual(collected[0].read_text(), f"device{target_device}") | ||||||
| 830 | + | ||||||
| 831 | + def test_collect_data_dump_multiple_device_id_prefix(self): | ||||||
| 832 | + """ | ||||||
| 833 | + 测试device_id为0时不会误命中data-dump/01/下的同名dump文件 | ||||||
| 834 | + """ | ||||||
| 835 | + data_name = "GatherV2.GatherV21.1.1733469426252033" | ||||||
| 836 | + output_path = self.temp.joinpath(f"info_{CUR_TIME_STR}") | ||||||
| 837 | + input_path = self.temp.joinpath(f"asys_output_{CUR_TIME_STR}") | ||||||
| 838 | + for device_id in ("01", "0"): | ||||||
| 839 | + dump_path = input_path.joinpath(f"extra-info/data-dump/{device_id}") | ||||||
| 840 | + dump_path.mkdir(parents=True, exist_ok=True) | ||||||
| 841 | + dump_path.joinpath(data_name).write_text(f"device{device_id}") | ||||||
| 842 | + collection = Collection(input_path, output_path) | ||||||
| 843 | + collection.collect_data_dump("0", data_name) | ||||||
| 844 | + collected = list(output_path.rglob(f'collection/dump/{data_name}')) | ||||||
| 845 | + self.assertEqual(len(collected), 1) | ||||||
| 846 | + self.assertEqual(collected[0].read_text(), "device0") | ||||||
| 847 | + | ||||||
| 848 | + | ||||||
| 849 | + def test_collect_data_dump_with_rename_multiple_device(self, target_device): | ||||||
| 850 | + """ | ||||||
| 851 | + 测试超长场景下多个device目录存在相同映射名时，只收集报错device的那一份。 | ||||||
| 852 | + 映射名在plog中不存在，无法靠回查plog区分，必须按文件所在目录筛选 | ||||||
| 853 | + """ | ||||||
| 854 | + data_name = "a" * 250 + ".42.1.1726159207469285" | ||||||
| 855 | + rename = "1234567890123456" | ||||||
| 856 | + output_path = self.temp.joinpath(f"info_{CUR_TIME_STR}_{target_device}") | ||||||
| 857 | + input_path = self.temp.joinpath(f"asys_output_{CUR_TIME_STR}") | ||||||
| 858 | + for device_id in ("0", "1"): | ||||||
| 859 | + dump_path = input_path.joinpath(f"extra-info/data-dump/{device_id}") | ||||||
| 860 | + dump_path.mkdir(parents=True, exist_ok=True) | ||||||
| 861 | + dump_path.joinpath(rename).write_text(f"device{device_id}") | ||||||
| 862 | + collection = Collection(input_path, output_path) | ||||||
| 863 | + collection.collect_data_dump(target_device, data_name, rename) | ||||||
| 864 | + collected = list(output_path.rglob(f'collection/dump/{rename}')) | ||||||
| 865 | + self.assertEqual(len(collected), 1) | ||||||
| 866 | + self.assertEqual(collected[0].read_text(), f"device{target_device}") | ||||||
| 867 | + | ||||||
| 868 | + def test_collect_oversize_scene(self): | ||||||
| 869 | + """ | ||||||
| 870 | + 测试超长名字场景走完整collect流程：按映射名收集dump文件并带上mapping.csv | ||||||
| 871 | + """ | ||||||
| 872 | + data_name = "a" * 250 + ".42.1.1726159207469285" | ||||||
| 873 | + rename = "1234567890123456" | ||||||
| 874 | + output_path = self.temp.joinpath(f"info_{CUR_TIME_STR}") | ||||||
| 875 | + input_path = self.temp.joinpath(f"asys_output_{CUR_TIME_STR}") | ||||||
| 876 | + dump_path = input_path.joinpath("extra-info/data-dump/0") | ||||||
| 877 | + dump_path.mkdir(parents=True, exist_ok=True) | ||||||
| 878 | + dump_path.joinpath(rename).touch() | ||||||
| 879 | + dump_path.joinpath(Constant.MAPPING_CSV_FILE).write_text(f"{rename},{data_name}\n") | ||||||
| 880 | + input_path.joinpath("plog.txt").write_text( | ||||||
| 881 | + "[ERROR] IDEDD(1592077,python3):2024-09-12-16:40:08.360.226 [dump_args.cpp:807]" | ||||||
| 882 | + "[tid:1592077] [Dump][Exception] dump exception to file, file: " | ||||||
| 883 | + f"./new/extra-info/data-dump/0/{data_name}") | ||||||
| 884 | + collection = Collection(input_path, output_path) | ||||||
| 885 | + res = collection.collect() | ||||||
| 886 | + self.assertEqual(res, True) | ||||||
| 887 | + self.assertEqual(collection.dump_file_rename, rename) | ||||||
| 888 | + self.assertEqual( | ||||||
| 889 | + bool(list(output_path.rglob(f'collection/dump/{rename}'))), True) | ||||||
| 890 | + self.assertEqual( | ||||||
| 891 | + bool(list(output_path.rglob(f'collection/dump/{Constant.MAPPING_CSV_FILE}'))), True) | ||||||
| 892 | + | ||||||
| 893 | + def test_collect_oversize_scene_mapping_csv_found_once(self, mocker): | ||||||
| 894 | + """ | ||||||
| 895 | + 测试超长名字场景下mapping.csv只解析一次，collect_data_dump复用缓存不再重复find | ||||||
| 896 | + """ | ||||||
| 897 | + data_name = "a" * 250 + ".42.1.1726159207469285" | ||||||
| 898 | + rename = "1234567890123456" | ||||||
| 899 | + output_path = self.temp.joinpath(f"info_{CUR_TIME_STR}") | ||||||
| 900 | + input_path = self.temp.joinpath(f"asys_output_{CUR_TIME_STR}") | ||||||
| 901 | + dump_path = input_path.joinpath("extra-info/data-dump/0") | ||||||
| 902 | + dump_path.mkdir(parents=True, exist_ok=True) | ||||||
| 903 | + dump_path.joinpath(rename).touch() | ||||||
| 904 | + dump_path.joinpath(Constant.MAPPING_CSV_FILE).write_text(f"{rename},{data_name}\n") | ||||||
| 905 | + input_path.joinpath("plog.txt").write_text( | ||||||
| 906 | + "[ERROR] IDEDD(1592077,python3):2024-09-12-16:40:08.360.226 [dump_args.cpp:807]" | ||||||
| 907 | + "[tid:1592077] [Dump][Exception] dump exception to file, file: " | ||||||
| 908 | + f"./new/extra-info/data-dump/0/{data_name}") | ||||||
| 909 | + collection = Collection(input_path, output_path) | ||||||
| 910 | + get_csv = mocker.spy(collection, "_get_dump_mapping_csv_path") | ||||||
| 911 | + self.assertEqual(collection.collect(), True) | ||||||
| 912 | + self.assertEqual(get_csv.call_count, 1) | ||||||
| 913 | + self.assertEqual( | ||||||
| 914 | + bool(list(output_path.rglob(f'collection/dump/{Constant.MAPPING_CSV_FILE}'))), True) | ||||||
| 915 | + | ||||||
| 916 | + def test_collect_data_dump_with_rename_no_cache(self, mocker): | ||||||
| 917 | + """ | ||||||
| 918 | + 测试直接调用collect_data_dump（未经_resolve_dump_file_rename）时缓存为空， | ||||||
| 919 | + 仍会按需解析mapping.csv | ||||||
| 920 | + """ | ||||||
| 921 | + data_name = "a" * 250 + ".42.1.1726159207469285" | ||||||
| 922 | + rename = "1234567890123456" | ||||||
| 923 | + output_path = self.temp.joinpath(f"info_{CUR_TIME_STR}") | ||||||
| 924 | + input_path = self.temp.joinpath(f"asys_output_{CUR_TIME_STR}") | ||||||
| 925 | + dump_path = input_path.joinpath("extra-info/data-dump/0") | ||||||
| 926 | + dump_path.mkdir(parents=True, exist_ok=True) | ||||||
| 927 | + dump_path.joinpath(rename).touch() | ||||||
| 928 | + dump_path.joinpath(Constant.MAPPING_CSV_FILE).write_text(f"{rename},{data_name}\n") | ||||||
| 929 | + collection = Collection(input_path, output_path) | ||||||
| 930 | + self.assertEqual(collection._mapping_csv_path, "") | ||||||
| 931 | + get_csv = mocker.spy(collection, "_get_dump_mapping_csv_path") | ||||||
| 932 | + collection.collect_data_dump("0", data_name, rename) | ||||||
| 933 | + self.assertEqual(get_csv.call_count, 1) | ||||||
| 934 | + self.assertEqual( | ||||||
| 935 | + bool(list(output_path.rglob(f'collection/dump/{Constant.MAPPING_CSV_FILE}'))), True) | ||||||
| 936 | + | ||||||
| 568 | 937 | ||||||
| 569 | "graph_name, expected", | 938 | "graph_name, expected", | ||||
| 570 | [ | 939 | [ | ||||
| @@ -470,3 +470,107 @@ class TestUtilsMethods(CommonAssert): | |||||||
| 470 | dump_data_parser._save_data_to_bin_file({'input': [{'shape': {'dim':['1', '2']}, 'size':'2', 'data': struct.pack('Q', 10)}]}, 'input', {'input': {}}, dump_file) | 470 | dump_data_parser._save_data_to_bin_file({'input': [{'shape': {'dim':['1', '2']}, 'size':'2', 'data': struct.pack('Q', 10)}]}, 'input', {'input': {}}, dump_file) | ||||
| 471 | except Exception as e: | 471 | except Exception as e: | ||||
| 472 | self.assertEqual(str(e), '4') | 472 | self.assertEqual(str(e), '4') | ||||
| 473 | + | ||||||
| 474 | + def test_build_dst_file_name_normal_len(self): | ||||||
| 475 | + """不超长的解析结果文件名原样返回，不生成mapping.csv""" | ||||||
| 476 | + info = AicErrorInfo() | ||||||
| 477 | + info.kernel_name = 'GatherV2' | ||||||
| 478 | + dump_data_parser = DumpDataParser(str(self.temp), info) | ||||||
| 479 | + res = dump_data_parser._build_dst_file_name(str(self.temp), 'input', 0, 'float32', np.dtype('float32')) | ||||||
| 480 | + self.assertEqual(res, os.path.join(str(self.temp), 'GatherV2.input.0.float32.npy')) | ||||||
| 481 | + self.assertEqual(self.temp.joinpath(Constant.MAPPING_CSV_FILE).exists(), False) | ||||||
| 482 | + | ||||||
| 483 | + def test_build_dst_file_name_oversize_renamed(self): | ||||||
| 484 | + """超长的解析结果文件名被重命名为随机数字串，并记录到同级mapping.csv""" | ||||||
| 485 | + info = AicErrorInfo() | ||||||
| 486 | + info.kernel_name = 'a' * 260 | ||||||
| 487 | + dump_data_parser = DumpDataParser(str(self.temp), info) | ||||||
| 488 | + res = dump_data_parser._build_dst_file_name(str(self.temp), 'input', 0, 'float32', np.dtype('float32')) | ||||||
| 489 | + file_name = os.path.basename(res) | ||||||
| 490 | + # 重命名后为随机数字串 + 原后缀，落盘不会超过NAME_MAX | ||||||
| 491 | + assert len(file_name) <= Constant.MAX_FILE_NAME_LEN | ||||||
| 492 | + assert file_name.endswith('.npy') | ||||||
| 493 | + assert file_name[:-len('.npy')].isdigit() | ||||||
| 494 | + # 回读mapping.csv，确认记录了 {映射后},{映射前} | ||||||
| 495 | + mapping_text = self.temp.joinpath(Constant.MAPPING_CSV_FILE).read_text() | ||||||
| 496 | + self.assertIn(mapping_text, f'{file_name},{info.kernel_name}.input.0.float32.npy') | ||||||
🟠 High Priority
同一文件中 L535-536 存在相同错误： 后果：mapping_text（含换行符的完整内容）几乎不会是单行字符串的子串，断言预期会失败，无法验证实际意图。 建议：交换 改动建议
不准确？ | |||||||
| 497 | + # 重命名后的文件名可以正常落盘 | ||||||
| 498 | + np.save(res, np.zeros(2, dtype=np.float32)) | ||||||
| 499 | + self.assertEqual(os.path.isfile(res), True) | ||||||
| 500 | + | ||||||
| 501 | + def test_build_dst_file_name_oversize_multi_byte(self): | ||||||
| 502 | + """多字节kernel_name按字节数判超长，字符数未超但字节数已超时同样重命名""" | ||||||
| 503 | + info = AicErrorInfo() | ||||||
| 504 | + info.kernel_name = '算' * 100 # 100个字符，UTF-8编码为300字节 | ||||||
| 505 | + dump_data_parser = DumpDataParser(str(self.temp), info) | ||||||
| 506 | + res = dump_data_parser._build_dst_file_name(str(self.temp), 'input', 0, 'float32', np.dtype('float32')) | ||||||
| 507 | + file_name = os.path.basename(res) | ||||||
| 508 | + assert len(os.fsencode(file_name)) <= Constant.MAX_FILE_NAME_LEN | ||||||
| 509 | + assert file_name[:-len('.npy')].isdigit() | ||||||
| 510 | + np.save(res, np.zeros(2, dtype=np.float32)) | ||||||
| 511 | + self.assertEqual(os.path.isfile(res), True) | ||||||
| 512 | + | ||||||
| 513 | + def test_build_dst_file_name_oversize_bin_keeps_suffix(self): | ||||||
| 514 | + """超长的bin结果同样保留.bin后缀，保证下游按裸字节读取""" | ||||||
| 515 | + info = AicErrorInfo() | ||||||
| 516 | + info.kernel_name = 'a' * 260 | ||||||
| 517 | + dump_data_parser = DumpDataParser(str(self.temp), info) | ||||||
| 518 | + res = dump_data_parser._build_dst_file_name(str(self.temp), 'input', 0, 'hifloat8', None) | ||||||
| 519 | + assert os.path.basename(res).endswith('.bin') | ||||||
| 520 | + | ||||||
| 521 | + def test_gen_random_numeric_name(self, mocker): | ||||||
| 522 | + """随机数字串与已有文件冲突时重新生成""" | ||||||
| 523 | + dump_data_parser = DumpDataParser(str(self.temp), AicErrorInfo()) | ||||||
| 524 | + self.temp.joinpath('1111111111111111.npy').touch() | ||||||
| 525 | + mocker.patch('random.randint', side_effect=[1111111111111111, 2222222222222222]) | ||||||
| 526 | + res = dump_data_parser._gen_random_numeric_name(str(self.temp), '.npy') | ||||||
| 527 | + self.assertEqual(res, '2222222222222222.npy') | ||||||
| 528 | + | ||||||
| 529 | + def test_record_mapping_append(self): | ||||||
| 530 | + """多条映射追加写入同一份mapping.csv""" | ||||||
| 531 | + dump_data_parser = DumpDataParser(str(self.temp), AicErrorInfo()) | ||||||
| 532 | + dump_data_parser._record_mapping(str(self.temp), '1111111111111111.npy', 'long_name_a.npy') | ||||||
| 533 | + dump_data_parser._record_mapping(str(self.temp), '2222222222222222.bin', 'long_name_b.bin') | ||||||
| 534 | + mapping_text = self.temp.joinpath(Constant.MAPPING_CSV_FILE).read_text() | ||||||
| 535 | + self.assertIn(mapping_text, '1111111111111111.npy,long_name_a.npy') | ||||||
| 536 | + self.assertIn(mapping_text, '2222222222222222.bin,long_name_b.bin') | ||||||
| 537 | + | ||||||
| 538 | + def test_load_name_mapping(self): | ||||||
| 539 | + """读取dump目录下的mapping.csv，构造 原名 -> 映射名 的查找字典""" | ||||||
| 540 | + dump_data_parser = DumpDataParser(str(self.temp), AicErrorInfo()) | ||||||
| 541 | + self.temp.joinpath(Constant.MAPPING_CSV_FILE).write_text( | ||||||
| 542 | + '1234567890123456,long_name_a\nbad_line\n') | ||||||
| 543 | + self.assertEqual(dump_data_parser._load_name_mapping(), {'long_name_a': '1234567890123456'}) | ||||||
| 544 | + | ||||||
| 545 | + def test_load_name_mapping_not_exist(self): | ||||||
| 546 | + dump_data_parser = DumpDataParser(str(self.temp), AicErrorInfo()) | ||||||
| 547 | + self.assertEqual(dump_data_parser._load_name_mapping(), {}) | ||||||
| 548 | + | ||||||
| 549 | + def test_parse_matches_by_mapped_name(self, mocker): | ||||||
| 550 | + """dump目录下只有随机名文件 + mapping.csv时，按data_name反查映射名命中""" | ||||||
| 551 | + data_name = 'a' * 250 + '.42.1.1726159207469285' | ||||||
| 552 | + rename = '1234567890123456' | ||||||
| 553 | + dump_dir = self.temp.joinpath('dump') | ||||||
| 554 | + dump_dir.mkdir(parents=True, exist_ok=True) | ||||||
| 555 | + dump_dir.joinpath(rename).touch() | ||||||
| 556 | + dump_dir.joinpath(Constant.MAPPING_CSV_FILE).write_text(f'{rename},{data_name}\n') | ||||||
| 557 | + info = AicErrorInfo() | ||||||
| 558 | + info.node_name = 'GatherV2' | ||||||
| 559 | + info.data_name = data_name | ||||||
| 560 | + dump_data_parser = DumpDataParser(str(dump_dir), info) | ||||||
| 561 | + mocker.patch.object(dump_data_parser, 'parse_dump_data', return_value='') | ||||||
| 562 | + dump_data_parser.parse() | ||||||
| 563 | + self.assertEqual(info.dump_file, [str(dump_dir.joinpath(rename))]) | ||||||
| 564 | + | ||||||
| 565 | + def test_parse_skips_mapping_csv(self, mocker): | ||||||
| 566 | + """mapping.csv不会被当成dump文件解析""" | ||||||
| 567 | + dump_dir = self.temp.joinpath('dump') | ||||||
| 568 | + dump_dir.mkdir(parents=True, exist_ok=True) | ||||||
| 569 | + dump_dir.joinpath('GatherV2.1.1.123').touch() | ||||||
| 570 | + dump_dir.joinpath(Constant.MAPPING_CSV_FILE).write_text('1234,GatherV2.mapping\n') | ||||||
| 571 | + info = AicErrorInfo() | ||||||
| 572 | + info.node_name = 'GatherV2' | ||||||
| 573 | + dump_data_parser = DumpDataParser(str(dump_dir), info) | ||||||
| 574 | + mocker.patch.object(dump_data_parser, 'parse_dump_data', return_value='') | ||||||
| 575 | + dump_data_parser.parse() | ||||||
| 576 | + self.assertEqual(info.dump_file, [str(dump_dir.joinpath('GatherV2.1.1.123'))]) | ||||||
| @@ -16,6 +16,16 @@ | |||
| 16 | # limitations under the License. | 16 | # limitations under the License. |
| 17 | # ---------------------------------------------------------------------------- | 17 | # ---------------------------------------------------------------------------- |
| 18 | 18 | ||
| 19 | +import csv | ||
| 20 | +import os | ||
| 21 | +import pytest | ||
| 22 | +import sys | ||
| 23 | +import shutil | ||
| 24 | +import subprocess | ||
| 25 | +from unittest import mock | ||
| 26 | +from unittest.mock import Mock | ||
| 27 | +from pathlib import Path | ||
| 28 | + | ||
| 19 | from ms_interface import utils | 29 | from ms_interface import utils |
| 20 | from ms_interface.constant import Constant | 30 | from ms_interface.constant import Constant |
| 21 | from ms_interface.constant import ModeCustom | 31 | from ms_interface.constant import ModeCustom |
| @@ -26,17 +36,6 @@ from ms_interface.single_op_test_frame.common.ascend_tbe_op import AscendOpKerne | |||
| 26 | from ms_interface.run_dirty_ub import run_dirty_ub_tik | 36 | from ms_interface.run_dirty_ub import run_dirty_ub_tik |
| 27 | 37 | ||
| 28 | from conftest import MSAICERR_PATH, CommonAssert | 38 | from conftest import MSAICERR_PATH, CommonAssert |
| 29 | -import os | ||
| 30 | -import pytest | ||
| 31 | -import sys | ||
| 32 | -import shutil | ||
| 33 | -from unittest import mock | ||
| 34 | -from unittest.mock import Mock | ||
| 35 | -from pathlib import Path | ||
| 36 | -import subprocess | ||
| 37 | -te = Mock(name="te") | ||
| 38 | -te.__name__ = "te" | ||
| 39 | -sys.modules['te'] = te | ||
| 40 | sys.path.append(MSAICERR_PATH) | 39 | sys.path.append(MSAICERR_PATH) |
| 41 | 40 | ||
| 42 | cur_abspath = os.path.dirname(__file__) | 41 | cur_abspath = os.path.dirname(__file__) |
| @@ -142,6 +141,50 @@ class TestUtilsMethods(CommonAssert): | |||
| 142 | res = utils.get_inquire_result(['xxx'], 'asfdd', match_dict=True) | 141 | res = utils.get_inquire_result(['xxx'], 'asfdd', match_dict=True) |
| 143 | assert res == [] | 142 | assert res == [] |
| 144 | 143 | ||
| 144 | + def test_parse_name_mapping_csv(self, tmp_path): | ||
| 145 | + """ | ||
| 146 | + 测试超长文件名的mapping.csv解析为 原名 -> 映射名，非法行被忽略 | ||
| 147 | + """ | ||
| 148 | + csv_path = tmp_path.joinpath(Constant.MAPPING_CSV_FILE) | ||
| 149 | + csv_path.write_text("1234567890123456,long_name_a\n" | ||
| 150 | + "6543210987654321,long_name_b\n" | ||
| 151 | + "invalid_line_without_comma\n") | ||
| 152 | + res = utils.parse_name_mapping_csv(str(csv_path)) | ||
| 153 | + self.assertEqual(res, {"long_name_a": "1234567890123456", | ||
| 154 | + "long_name_b": "6543210987654321"}) | ||
| 155 | + | ||
| 156 | + def test_parse_name_mapping_csv_not_exist(self, tmp_path): | ||
| 157 | + self.assertEqual(utils.parse_name_mapping_csv(""), {}) | ||
| 158 | + self.assertEqual(utils.parse_name_mapping_csv( | ||
| 159 | + str(tmp_path.joinpath("not_exist.csv"))), {}) | ||
| 160 | + | ||
| 161 | + def test_parse_name_mapping_csv_read_failed(self, tmp_path, mocker): | ||
| 162 | + """ | ||
| 163 | + 测试mapping.csv读取异常时返回空字典而非向上抛异常 | ||
| 164 | + """ | ||
| 165 | + csv_path = tmp_path.joinpath(Constant.MAPPING_CSV_FILE) | ||
| 166 | + csv_path.write_text("1234567890123456,long_name_a\n") | ||
| 167 | + mocker.patch("csv.reader", side_effect=csv.Error("line contains NUL")) | ||
| 168 | + self.assertEqual(utils.parse_name_mapping_csv(str(csv_path)), {}) | ||
| 169 | + | ||
| 170 | + def test_parse_name_mapping_csv_invalid_encoding(self, tmp_path): | ||
| 171 | + """ | ||
| 172 | + 测试mapping.csv含非utf-8字节时返回空字典而非抛UnicodeDecodeError | ||
| 173 | + """ | ||
| 174 | + csv_path = tmp_path.joinpath(Constant.MAPPING_CSV_FILE) | ||
| 175 | + csv_path.write_bytes(b"1234567890123456,long_name_a\n" | ||
| 176 | + b"6543210987654321,long_name_\xff\xfe\n") | ||
| 177 | + self.assertEqual(utils.parse_name_mapping_csv(str(csv_path)), {}) | ||
| 178 | + | ||
| 179 | + def test_parse_name_mapping_csv_utf8_name(self, tmp_path): | ||
| 180 | + """ | ||
| 181 | + 测试多字节算子名不受locale影响，按utf-8解析成功 | ||
| 182 | + """ | ||
| 183 | + csv_path = tmp_path.joinpath(Constant.MAPPING_CSV_FILE) | ||
| 184 | + csv_path.write_bytes("1234567890123456,算子名称\n".encode("utf-8")) | ||
| 185 | + self.assertEqual(utils.parse_name_mapping_csv(str(csv_path)), | ||
| 186 | + {"算子名称": "1234567890123456"}) | ||
| 187 | + | ||
| 145 | 188 | ||
| 146 | def test_run_dirty_ub(self, mocker): | 189 | def test_run_dirty_ub(self, mocker): |
| 147 | temp_dir = Path(cur_abspath).joinpath("../test_run_dirty_ub") | 190 | temp_dir = Path(cur_abspath).joinpath("../test_run_dirty_ub") |
级别：提示 问题：self.dump_name_mapping 在 _resolve_dump_file_rename 中被赋值，但在整个生产代码路径中从未被读取（仅在 UT 断言中使用），属于死状态。 影响：增加维护者理解成本，可能误以为该字段在下游逻辑中被消费。 修复建议：若仅用于调试可保留并加注释说明；否则移除该实例变量，_resolve_dump_file_rename 内部用局部变量即可。