已合并
【Feature】【资料】适配【量化算子】类型精度比较 #628
shuhuipeng创建于 5月8日
【Feature】【资料】适配【量化算子】类型精度比较 #628
已合并
共 8 个文件变更+440-8
| @@ -101,6 +101,7 @@ msprobe acc_check -api_info ./dump_path/step{step_number}/rank{rank_number}/dump | |||
| 101 | black_list: [] | 101 | black_list: [] |
| 102 | error_data_path: './' | 102 | error_data_path: './' |
| 103 | precision: 14 | 103 | precision: 14 |
| 104 | + quantization_api_list: [] | ||
| 104 | ``` | 105 | ``` |
| 105 | 106 | ||
| 106 | | 参数 | 可选/必选 | 说明 | | 107 | | 参数 | 可选/必选 | 说明 | |
| @@ -109,6 +110,7 @@ msprobe acc_check -api_info ./dump_path/step{step_number}/rank{rank_number}/dump | |||
| 109 | | black_list | 可选 | API dump 黑名单，被指定的 API 不进行 dump。<br>**配置示例**：black_list=["conv1d", "conv2d"]。默认未配置黑名单，即 dump 全量 API 数据。 | | 110 | | black_list | 可选 | API dump 黑名单，被指定的 API 不进行 dump。<br>**配置示例**：black_list=["conv1d", "conv2d"]。默认未配置黑名单，即 dump 全量 API 数据。 | |
| 110 | | error_data_path | 可选 | 配置保存精度未达标的 API 输入输出数据路径。<br>**配置示例**："error_data_path": "./"。默认为当前路径。 | | 111 | | error_data_path | 可选 | 配置保存精度未达标的 API 输入输出数据路径。<br>**配置示例**："error_data_path": "./"。默认为当前路径。 | |
| 111 | | precision | 可选 | 浮点数表示位数，默认取小数点后14位。 | | 112 | | precision | 可选 | 浮点数表示位数，默认取小数点后14位。 | |
| 113 | + | quantization_api_list | 可选 | 浮点输入且整型输出的量化算子API列表，配置后表示该类型API只需满足绝对误差 ≤ 1即为PASS | | ||
| 112 | 114 | ||
| 113 | **说明**：white_list 和 black_list 同时配置时，二者配置的 API 名单若无交集，则白名单生效，若 API 名单存在交集，则白名单排除的部分以及交集的 API 不进行 dump。 | 115 | **说明**：white_list 和 black_list 同时配置时，二者配置的 API 名单若无交集，则白名单生效，若 API 名单存在交集，则白名单排除的部分以及交集的 API 不进行 dump。 |
| 114 | 116 | ||
| @@ -47,7 +47,8 @@ class Config: | |||
| 47 | 'white_list': list, | 47 | 'white_list': list, |
| 48 | 'black_list': list, | 48 | 'black_list': list, |
| 49 | 'error_data_path': str, | 49 | 'error_data_path': str, |
| 50 | - 'precision': int | 50 | + 'precision': int, |
| 51 | + 'quantization_api_list': list | ||
| 51 | } | 52 | } |
| 52 | if key not in validators: | 53 | if key not in validators: |
| 53 | raise ValueError(f"{key} must be one of {validators.keys()}") | 54 | raise ValueError(f"{key} must be one of {validators.keys()}") |
| @@ -63,6 +64,8 @@ class Config: | |||
| 63 | ACCCheckConfig.check_filter_list_config(key, value) | 64 | ACCCheckConfig.check_filter_list_config(key, value) |
| 64 | if key == 'error_data_path': | 65 | if key == 'error_data_path': |
| 65 | ACCCheckConfig.check_error_data_path_config(value) | 66 | ACCCheckConfig.check_error_data_path_config(value) |
| 67 | + if key == 'quantization_api_list': | ||
| 68 | + ACCCheckConfig.check_filter_list_config(key, value) | ||
| 66 | return value | 69 | return value |
| 67 | 70 | ||
| 68 | 71 | ||
| @@ -344,11 +344,15 @@ class Comparator: | |||
| 344 | message = "" | 344 | message = "" |
| 345 | if bench_output.size == 0: | 345 | if bench_output.size == 0: |
| 346 | return CompareConst.SKIP, compare_column, "There is not bench calculation result." | 346 | return CompareConst.SKIP, compare_column, "There is not bench calculation result." |
| 347 | - if bench_output.dtype in [bool, np.uint8, np.int8, np.int16, np.uint16, np.uint32, np.int32, | 347 | + if bench_output.dtype in [bool, np.uint8, np.int8, np.int16, np.uint16, np.uint32, np.int32, np.int64, np.uint64]: |
| 348 | - np.int64, np.uint64]: | 348 | + # 输出为整型且为配置的量化算子，使用量化比较逻辑：最大绝对误差<=1的判定标准 |
| 349 | - message += f"Current {bench_output.dtype} data, only perform binary comparison." | 349 | + # 获取输入为浮点型且输出为整形的量化算子清单：配置在config.yaml里面 |
| 350 | - err_rate, status, msg = compare_bool_tensor(bench_output, device_output) | 350 | + quantization_api_list = msCheckerConfig.quantization_api_list |
| 351 | - message += msg + "\n" | 351 | + if api_name in quantization_api_list: |
| 352 | + status, compare_column, message = self._compare_quantization_tensor(bench_output, device_output, compare_column, npu_dtype) | ||
| 353 | + return status, compare_column, message | ||
| 354 | + err_rate, status, msg = compare_bool_tensor(bench_output, device_output) | ||
| 355 | + message += f"Current {bench_output.dtype} data, only perform binary comparison." | ||
| 352 | compare_column.error_rate = err_rate | 356 | compare_column.error_rate = err_rate |
| 353 | return status, compare_column, message | 357 | return status, compare_column, message |
| 354 | else: | 358 | else: |
| @@ -360,6 +364,23 @@ class Comparator: | |||
| 360 | compare_column, in_and_out_dtype) | 364 | compare_column, in_and_out_dtype) |
| 361 | return status, compare_column, message | 365 | return status, compare_column, message |
| 362 | 366 | ||
| 367 | + | ||
| 368 | + def _compare_quantization_tensor(self, bench_output, device_output, compare_column, dtype): | ||
| 369 | + """处理量化算子的比较逻辑，使用最大绝对误差<=1的判定标准""" | ||
| 370 | + message = "" | ||
| 371 | + # 使用最大绝对误差<=1的判定逻辑 | ||
| 372 | + abs_err = get_abs_err(bench_output, device_output) | ||
| 373 | + max_abs_res, max_abs_status = get_max_abs_err(abs_err) | ||
| 374 | + | ||
| 375 | + compare_column.max_abs_err = max_abs_res | ||
| 376 | + | ||
| 377 | + if max_abs_res <= 1: | ||
| 378 | + message += f"Quantization type: Max abs error is {max_abs_res:.2f}, less than or equal to 1, consider as pass.\n" | ||
| 379 | + return CompareConst.PASS, compare_column, message | ||
| 380 | + else: | ||
| 381 | + message += f"Quantization type: Max abs error is {max_abs_res:.2f}, greater than 1, consider as error.\n" | ||
| 382 | + return CompareConst.ERROR, compare_column, message | ||
| 383 | + | ||
| 363 | def _perform_comparison(self, api_name, input_data, dtype, in_dtype): | 384 | def _perform_comparison(self, api_name, input_data, dtype, in_dtype): |
| 364 | comparison_func = self.registry.get_comparison_function(api_name, dtype, in_dtype) | 385 | comparison_func = self.registry.get_comparison_function(api_name, dtype, in_dtype) |
| 365 | comparison_func(input_data) | 386 | comparison_func(input_data) |
| @@ -2,4 +2,5 @@ white_list: [] | |||
| 2 | black_list: [] | 2 | black_list: [] |
| 3 | error_data_path: './' | 3 | error_data_path: './' |
| 4 | precision: 14 | 4 | precision: 14 |
| 5 | +quantization_api_list: ["npu_rms_norm_dynamic_quant", "npu_dynamic_quant", "npu_quant_lightning_indexer"] | ||
| 5 | 6 | ||
| @@ -1308,6 +1308,9 @@ torch_npu: | |||
| 1308 | - npu_sparse_lightning_indexer_grad_kl_loss | 1308 | - npu_sparse_lightning_indexer_grad_kl_loss |
| 1309 | - npu_swiglu_mx_quant | 1309 | - npu_swiglu_mx_quant |
| 1310 | - npu_fused_causal_conv1d | 1310 | - npu_fused_causal_conv1d |
| 1311 | + - npu_quant_lightning_indexer | ||
| 1312 | + - npu_rms_norm_dynamic_quant | ||
| 1313 | + - npu_dynamic_quant | ||
| 1311 | 1314 | ||
| 1312 | aten: | 1315 | aten: |
| 1313 | - signbit | 1316 | - signbit |
| @@ -48,6 +48,13 @@ class TestConfig(unittest.TestCase): | |||
| 48 | 48 | ||
| 49 | with self.assertRaises(Exception): | 49 | with self.assertRaises(Exception): |
| 50 | self.cfg.validate('white_list', ['invalid_api1', 'invalid_api2']) | 50 | self.cfg.validate('white_list', ['invalid_api1', 'invalid_api2']) |
| 51 | + | ||
| 52 | + def test_validate_quantization_api_list(self): | ||
| 53 | + validate_quantization_api_list = ['npu_rms_norm_dynamic_quant', 'npu_dynamic_quant', 'npu_quant_lightning_indexer'] | ||
| 54 | + self.assertEqual(self.cfg.validate('quantization_api_list', validate_quantization_api_list), validate_quantization_api_list) | ||
| 55 | + | ||
| 56 | + with self.assertRaises(Exception): | ||
| 57 | + self.cfg.validate('quantization_api_list', ['invalid_api1', 'invalid_api2']) | ||
| 51 | 58 | ||
| 52 | def test_CheckerConfig_init_with_defaults(self): | 59 | def test_CheckerConfig_init_with_defaults(self): |
| 53 | checker_config = CheckerConfig() | 60 | checker_config = CheckerConfig() |
| @@ -322,4 +322,296 @@ class TestAlgorithmMethods(unittest.TestCase): | |||
| 322 | self.assertEqual(result, CompareConst.PASS) | 322 | self.assertEqual(result, CompareConst.PASS) |
| 323 | self.assertEqual(msg, "") | 323 | self.assertEqual(msg, "") |
| 324 | 324 | ||
| 325 | - # ========== int8 类型测试用例 ========== | 325 | + def test_cosine_sim_int8(self): |
| 326 | + """测试纯int8数据的余弦相似度计算""" | ||
| 327 | + bench_output = np.array([1, 2, 3], dtype=np.int8) | ||
| 328 | + device_output = np.array([1, 2, 3], dtype=np.int8) | ||
| 329 | + cos, success, msg = alg.cosine_sim(bench_output, device_output) | ||
| 330 | + self.assertEqual(cos, 1.0) | ||
| 331 | + self.assertTrue(success) | ||
| 332 | + self.assertEqual(msg, '') | ||
| 333 | + | ||
| 334 | + def test_cosine_sim_int8_with_difference(self): | ||
| 335 | + """测试int8数据有差异时的余弦相似度""" | ||
| 336 | + bench_output = np.array([1, 2, 3], dtype=np.int8) | ||
| 337 | + device_output = np.array([2, 3, 4], dtype=np.int8) | ||
| 338 | + cos, success, msg = alg.cosine_sim(bench_output, device_output) | ||
| 339 | + # 计算期望的余弦相似度 | ||
| 340 | + expected_cos = np.dot(bench_output, device_output) / ( | ||
| 341 | + np.linalg.norm(bench_output) * np.linalg.norm(device_output)) | ||
| 342 | + self.assertAlmostEqual(cos, expected_cos, places=6) | ||
| 343 | + self.assertTrue(success) | ||
| 344 | + | ||
| 345 | + def test_get_abs_err_int8(self): | ||
| 346 | + """测试int8数据的绝对误差计算""" | ||
| 347 | + bench_data = np.array([10, 20, 30], dtype=np.int8) | ||
| 348 | + device_data = np.array([12, 18, 32], dtype=np.int8) | ||
| 349 | + abs_err = alg.get_abs_err(bench_data, device_data) | ||
| 350 | + self.assertListEqual(list(abs_err), [2.0, 2.0, 2.0]) | ||
| 351 | + | ||
| 352 | + def test_get_abs_bench_with_eps_int8(self): | ||
| 353 | + """测试int8数据的get_abs_bench_with_eps函数""" | ||
| 354 | + bench_data = np.array([1, 2, 3], dtype=np.int8) | ||
| 355 | + abs_bench, abs_bench_with_eps = alg.get_abs_bench_with_eps(bench_data, torch.int8) | ||
| 356 | + expected_abs_bench = np.abs(bench_data).astype(float) # 显式转换为float | ||
| 357 | + expected_abs_bench_with_eps = expected_abs_bench + np.finfo(np.float64).eps # 使用float64的epsilon | ||
| 358 | + self.assertTrue(np.array_equal(abs_bench, expected_abs_bench)) | ||
| 359 | + self.assertTrue(np.allclose(abs_bench_with_eps, expected_abs_bench_with_eps)) | ||
| 360 | + | ||
| 361 | + def test_get_rel_err_origin_int8(self): | ||
| 362 | + """测试int8数据的原始相对误差计算""" | ||
| 363 | + bench_data = np.array([10, 20, 30], dtype=np.int8) | ||
| 364 | + device_data = np.array([12, 18, 32], dtype=np.int8) | ||
| 365 | + abs_err = np.abs(device_data - bench_data) | ||
| 366 | + rel_err = alg.get_rel_err_origin(abs_err, bench_data) | ||
| 367 | + expected_rel_err = np.abs(abs_err / bench_data) | ||
| 368 | + self.assertTrue(np.allclose(rel_err, expected_rel_err)) | ||
| 369 | + | ||
| 370 | + def test_get_finite_and_infinite_mask_int8(self): | ||
| 371 | + """测试int8数据的有限值和无限值掩码""" | ||
| 372 | + bench_data = np.array([1, 2, 3], dtype=np.int8) | ||
| 373 | + device_data = np.array([4, 5, 6], dtype=np.int8) | ||
| 374 | + both_finite_mask, inf_nan_mask = alg.get_finite_and_infinite_mask(bench_data, device_data) | ||
| 375 | + self.assertListEqual(list(both_finite_mask), [True, True, True]) | ||
| 376 | + self.assertListEqual(list(inf_nan_mask), [False, False, False]) | ||
| 377 | + | ||
| 378 | + def test_check_inf_nan_value_int8(self): | ||
| 379 | + """测试int8数据的inf/nan值检查""" | ||
| 380 | + bench_data = np.array([1, 2, 3], dtype=np.int8) | ||
| 381 | + device_data = np.array([4, 5, 6], dtype=np.int8) | ||
| 382 | + both_finite_mask, inf_nan_mask = alg.get_finite_and_infinite_mask(bench_data, device_data) | ||
| 383 | + # int8数据不应该有inf/nan，所以应该返回0 | ||
| 384 | + result = alg.check_inf_nan_value(inf_nan_mask, bench_data, device_data, torch.int8, 0.001) | ||
| 385 | + self.assertEqual(result, 0) | ||
| 386 | + | ||
| 387 | + def test_get_max_abs_err_int8(self): | ||
| 388 | + """测试int8数据的最大绝对误差""" | ||
| 389 | + bench_data = np.array([10, 20, 30], dtype=np.int8) | ||
| 390 | + device_data = np.array([12, 18, 32], dtype=np.int8) | ||
| 391 | + abs_err = alg.get_abs_err(bench_data, device_data) | ||
| 392 | + max_err, has_nan_inf = alg.get_max_abs_err(abs_err) | ||
| 393 | + self.assertEqual(max_err, 2.0) | ||
| 394 | + self.assertFalse(has_nan_inf) | ||
| 395 | + | ||
| 396 | + def test_get_max_rel_err_int8(self): | ||
| 397 | + """测试int8数据的最大相对误差""" | ||
| 398 | + bench_data = np.array([10, 20, 30], dtype=np.int8) | ||
| 399 | + device_data = np.array([12, 18, 32], dtype=np.int8) | ||
| 400 | + abs_err = np.abs(device_data - bench_data) | ||
| 401 | + abs_bench = np.abs(bench_data).astype(float) # 显式转换为float | ||
| 402 | + eps = np.finfo(np.float64).eps # 使用float64的epsilon | ||
| 403 | + abs_bench_with_eps = abs_bench + eps | ||
| 404 | + rel_err = abs_err / abs_bench_with_eps | ||
| 405 | + # get_max_rel_err返回单个值，不是元组 | ||
| 406 | + max_err = alg.get_max_rel_err(rel_err) | ||
| 407 | + self.assertAlmostEqual(max_err, 0.2, places=3) # 最大相对误差约为0.2 | ||
| 408 | + | ||
| 409 | + def test_get_mean_rel_err_int8(self): | ||
| 410 | + """测试int8数据的平均相对误差""" | ||
| 411 | + bench_data = np.array([10, 20, 30], dtype=np.int8) | ||
| 412 | + device_data = np.array([12, 18, 32], dtype=np.int8) | ||
| 413 | + abs_err = np.abs(device_data - bench_data) | ||
| 414 | + abs_bench = np.abs(bench_data).astype(float) # 显式转换为float | ||
| 415 | + eps = np.finfo(np.float64).eps # 使用float64的epsilon | ||
| 416 | + abs_bench_with_eps = abs_bench + eps | ||
| 417 | + rel_err = abs_err / abs_bench_with_eps | ||
| 418 | + mean_err = alg.get_mean_rel_err(rel_err) | ||
| 419 | + expected_mean = np.mean(rel_err) | ||
| 420 | + self.assertAlmostEqual(mean_err, expected_mean, places=6) | ||
| 421 | + | ||
| 422 | + # ========== int8 边界值测试用例 ========== | ||
| 423 | + | ||
| 424 | + def test_cosine_sim_int8_boundary_values(self): | ||
| 425 | + """测试int8边界值的余弦相似度""" | ||
| 426 | + # 测试int8的最大值和最小值 | ||
| 427 | + bench_output = np.array([127, -128, 0], dtype=np.int8) | ||
| 428 | + device_output = np.array([127, -128, 0], dtype=np.int8) | ||
| 429 | + cos, success, msg = alg.cosine_sim(bench_output, device_output) | ||
| 430 | + self.assertEqual(cos, 1.0) | ||
| 431 | + self.assertTrue(success) | ||
| 432 | + | ||
| 433 | + def test_get_abs_bench_with_eps_int8_boundary(self): | ||
| 434 | + """测试int8边界值的get_abs_bench_with_eps""" | ||
| 435 | + bench_data = np.array([127, -128, 0], dtype=np.int8) | ||
| 436 | + abs_bench, abs_bench_with_eps = alg.get_abs_bench_with_eps(bench_data, torch.int8) | ||
| 437 | + expected_abs_bench = np.array([127, 128, 0], dtype=np.float64) | ||
| 438 | + expected_abs_bench_with_eps = expected_abs_bench + np.finfo(np.float64).eps # 使用float64的epsilon | ||
| 439 | + self.assertTrue(np.array_equal(abs_bench, expected_abs_bench)) | ||
| 440 | + self.assertTrue(np.allclose(abs_bench_with_eps, expected_abs_bench_with_eps)) | ||
| 441 | + | ||
| 442 | + def test_get_abs_err_int8_max_difference(self): | ||
| 443 | + """测试int8最大差值的绝对误差""" | ||
| 444 | + # 注意: int8范围是-128到127,127 - (-128) = 255会溢出 | ||
| 445 | + # 100 - (-100) = 200 也会溢出到 -56,abs后为56 | ||
| 446 | + bench_data = np.array([-50], dtype=np.int8) | ||
| 447 | + device_data = np.array([50], dtype=np.int8) | ||
| 448 | + abs_err = alg.get_abs_err(bench_data, device_data) | ||
| 449 | + self.assertEqual(abs_err[0], 100) | ||
| 450 | + | ||
| 451 | + def test_get_rel_err_origin_int8_zero_handling(self): | ||
| 452 | + """测试int8数据中包含0时的相对误差处理""" | ||
| 453 | + bench_data = np.array([10, 0, 30], dtype=np.int8) | ||
| 454 | + device_data = np.array([12, 0, 32], dtype=np.int8) | ||
| 455 | + abs_err = np.abs(device_data - bench_data) | ||
| 456 | + rel_err = alg.get_rel_err_origin(abs_err, bench_data) | ||
| 457 | + # 对于0值，相对误差应该为inf或特殊处理 | ||
| 458 | + # 实际实现中，0除以0会产生nan或inf | ||
| 459 | + # 测试非0值的相对误差计算 | ||
| 460 | + # 期望: [abs(12-10)/10, abs(0-0)/0, abs(32-30)/30] = [0.2, nan/inf, 0.0667] | ||
| 461 | + # 由于除零会产生inf/nan，我们只验证非0值 | ||
| 462 | + self.assertAlmostEqual(rel_err[0], 0.2, places=5) | ||
| 463 | + self.assertAlmostEqual(rel_err[2], 0.06666667, places=5) | ||
| 464 | + # 0值处的相对误差应该是inf或nan | ||
| 465 | + self.assertTrue(np.isinf(rel_err[1]) or np.isnan(rel_err[1])) | ||
| 466 | + | ||
| 467 | + def test_get_finite_and_infinite_mask_int8_boundary(self): | ||
| 468 | + """测试int8边界值的有限值掩码""" | ||
| 469 | + bench_data = np.array([127, -128, 0], dtype=np.int8) | ||
| 470 | + device_data = np.array([127, -128, 0], dtype=np.int8) | ||
| 471 | + both_finite_mask, inf_nan_mask = alg.get_finite_and_infinite_mask(bench_data, device_data) | ||
| 472 | + self.assertListEqual(list(both_finite_mask), [True, True, True]) | ||
| 473 | + self.assertListEqual(list(inf_nan_mask), [False, False, False]) | ||
| 474 | + | ||
| 475 | + # ========== int8 与 float16 混合场景测试用例 ========== | ||
| 476 | + | ||
| 477 | + def test_cosine_sim_int8_to_float16_conversion(self): | ||
| 478 | + """测试int8数据转换为float16后的余弦相似度""" | ||
| 479 | + bench_output = np.array([1, 2, 3], dtype=np.int8) | ||
| 480 | + device_output = np.array([1, 2, 3], dtype=np.int8) | ||
| 481 | + # 算法内部可能会处理类型转换 | ||
| 482 | + cos, success, msg = alg.cosine_sim(bench_output, device_output) | ||
| 483 | + self.assertEqual(cos, 1.0) | ||
| 484 | + self.assertTrue(success) | ||
| 485 | + | ||
| 486 | + def test_get_abs_err_int8_vs_float16(self): | ||
| 487 | + """测试int8和float16混合数据的绝对误差""" | ||
| 488 | + # 算法应该能够处理int8数据 | ||
| 489 | + bench_data = np.array([10, 20, 30], dtype=np.int8) | ||
| 490 | + device_data = np.array([12, 18, 32], dtype=np.int8) | ||
| 491 | + abs_err = alg.get_abs_err(bench_data, device_data) | ||
| 492 | + self.assertListEqual(list(abs_err), [2.0, 2.0, 2.0]) | ||
| 493 | + | ||
| 494 | + def test_get_rel_err_origin_int8_vs_float16(self): | ||
| 495 | + """测试int8数据的相对误差计算（模拟与float16对比场景）""" | ||
| 496 | + bench_data = np.array([10, 20, 30], dtype=np.int8) | ||
| 497 | + device_data = np.array([12, 18, 32], dtype=np.int8) | ||
| 498 | + abs_err = np.abs(device_data - bench_data) | ||
| 499 | + # 对于int8，相对误差计算应该正常工作 | ||
| 500 | + rel_err = alg.get_rel_err_origin(abs_err, bench_data) | ||
| 501 | + expected_rel_err = np.abs(abs_err / bench_data) | ||
| 502 | + self.assertTrue(np.allclose(rel_err, expected_rel_err)) | ||
| 503 | + | ||
| 504 | + def test_int8_array_operations(self): | ||
| 505 | + """测试int8数组的基本操作""" | ||
| 506 | + bench_data = np.array([1, 2, 3, 4, 5], dtype=np.int8) | ||
| 507 | + device_data = np.array([2, 3, 4, 5, 6], dtype=np.int8) | ||
| 508 | + | ||
| 509 | + # 测试绝对误差 | ||
| 510 | + abs_err = alg.get_abs_err(bench_data, device_data) | ||
| 511 | + self.assertTrue(np.array_equal(abs_err, np.array([1.0, 1.0, 1.0, 1.0, 1.0]))) | ||
| 512 | + | ||
| 513 | + # 测试相对误差 | ||
| 514 | + rel_err = alg.get_rel_err_origin(abs_err, bench_data) | ||
| 515 | + expected_rel_err = np.array([1.0, 0.5, 0.33333333, 0.25, 0.2]) | ||
| 516 | + self.assertTrue(np.allclose(rel_err, expected_rel_err, atol=1e-5)) | ||
| 517 | + | ||
| 518 | + # 测试最大绝对误差 | ||
| 519 | + max_abs_err, _ = alg.get_max_abs_err(abs_err) | ||
| 520 | + self.assertEqual(max_abs_err, 1.0) | ||
| 521 | + | ||
| 522 | + # 测试最大相对误差 - get_max_rel_err返回单个值，不是元组 | ||
| 523 | + max_rel_err = alg.get_max_rel_err(rel_err) | ||
| 524 | + self.assertAlmostEqual(max_rel_err, 1.0, places=3) | ||
| 525 | + | ||
| 526 | + def test_get_abs_bench_with_eps_int8_type_conversion(self): | ||
| 527 | + """测试int8数据的显式类型转换""" | ||
| 528 | + bench_data = np.array([10, 20, 30], dtype=np.int8) | ||
| 529 | + abs_bench, abs_bench_with_eps = alg.get_abs_bench_with_eps(bench_data, torch.int8) | ||
| 530 | + | ||
| 531 | + # 验证abs_bench的类型为float64 | ||
| 532 | + self.assertEqual(abs_bench.dtype, np.float64) | ||
| 533 | + self.assertEqual(abs_bench_with_eps.dtype, np.float64) | ||
| 534 | + | ||
| 535 | + # 验证类型一致性 | ||
| 536 | + self.assertEqual(abs_bench.dtype, abs_bench_with_eps.dtype) | ||
| 537 | + | ||
| 538 | + # 验证数值正确性 | ||
| 539 | + expected_abs_bench = np.array([10.0, 20.0, 30.0], dtype=np.float64) | ||
| 540 | + self.assertTrue(np.array_equal(abs_bench, expected_abs_bench)) | ||
| 541 | + | ||
| 542 | + # 验证eps值使用的是float64的机器epsilon | ||
| 543 | + float64_eps = np.finfo(np.float64).eps | ||
| 544 | + expected_abs_bench_with_eps = expected_abs_bench + float64_eps | ||
| 545 | + self.assertTrue(np.allclose(abs_bench_with_eps, expected_abs_bench_with_eps)) | ||
| 546 | + | ||
| 547 | + def test_get_abs_bench_with_eps_int16_type_conversion(self): | ||
| 548 | + """测试int16数据的显式类型转换""" | ||
| 549 | + bench_data = np.array([1000, 2000, 3000], dtype=np.int16) | ||
| 550 | + abs_bench, abs_bench_with_eps = alg.get_abs_bench_with_eps(bench_data, torch.int16) | ||
| 551 | + | ||
| 552 | + # 验证abs_bench的类型为float64 | ||
| 553 | + self.assertEqual(abs_bench.dtype, np.float64) | ||
| 554 | + self.assertEqual(abs_bench_with_eps.dtype, np.float64) | ||
| 555 | + | ||
| 556 | + # 验证类型一致性 | ||
| 557 | + self.assertEqual(abs_bench.dtype, abs_bench_with_eps.dtype) | ||
| 558 | + | ||
| 559 | + # 验证数值正确性 | ||
| 560 | + expected_abs_bench = np.array([1000.0, 2000.0, 3000.0], dtype=np.float64) | ||
| 561 | + self.assertTrue(np.array_equal(abs_bench, expected_abs_bench)) | ||
| 562 | + | ||
| 563 | + # 验证eps值使用的是float64的机器epsilon | ||
| 564 | + float64_eps = np.finfo(np.float64).eps | ||
| 565 | + expected_abs_bench_with_eps = expected_abs_bench + float64_eps | ||
| 566 | + self.assertTrue(np.allclose(abs_bench_with_eps, expected_abs_bench_with_eps)) | ||
| 567 | + | ||
| 568 | + def test_get_abs_bench_with_eps_int32_type_conversion(self): | ||
| 569 | + """测试int32数据的显式类型转换""" | ||
| 570 | + bench_data = np.array([100000, 200000, 300000], dtype=np.int32) | ||
| 571 | + abs_bench, abs_bench_with_eps = alg.get_abs_bench_with_eps(bench_data, torch.int32) | ||
| 572 | + | ||
| 573 | + # 验证abs_bench的类型为float64 | ||
| 574 | + self.assertEqual(abs_bench.dtype, np.float64) | ||
| 575 | + self.assertEqual(abs_bench_with_eps.dtype, np.float64) | ||
| 576 | + | ||
| 577 | + # 验证类型一致性 | ||
| 578 | + self.assertEqual(abs_bench.dtype, abs_bench_with_eps.dtype) | ||
| 579 | + | ||
| 580 | + # 验证数值正确性 | ||
| 581 | + expected_abs_bench = np.array([100000.0, 200000.0, 300000.0], dtype=np.float64) | ||
| 582 | + self.assertTrue(np.array_equal(abs_bench, expected_abs_bench)) | ||
| 583 | + | ||
| 584 | + # 验证eps值使用的是float64的机器epsilon | ||
| 585 | + float64_eps = np.finfo(np.float64).eps | ||
| 586 | + expected_abs_bench_with_eps = expected_abs_bench + float64_eps | ||
| 587 | + self.assertTrue(np.allclose(abs_bench_with_eps, expected_abs_bench_with_eps)) | ||
| 588 | + | ||
| 589 | + def test_get_abs_bench_with_eps_integer_vs_float_comparison(self): | ||
| 590 | + """测试整数类型和浮点类型的eps值差异""" | ||
| 591 | + # int8数据 | ||
| 592 | + int8_data = np.array([10, 20, 30], dtype=np.int8) | ||
| 593 | + int8_abs_bench, int8_abs_bench_with_eps = alg.get_abs_bench_with_eps(int8_data, torch.int8) | ||
| 594 | + | ||
| 595 | + # float32数据 | ||
| 596 | + float32_data = np.array([10.0, 20.0, 30.0], dtype=np.float32) | ||
| 597 | + float32_abs_bench, float32_abs_bench_with_eps = alg.get_abs_bench_with_eps(float32_data, torch.float32) | ||
| 598 | + | ||
| 599 | + # 验证两者都是float64类型 (整数转换为float64) | ||
| 600 | + self.assertEqual(int8_abs_bench.dtype, np.float64) | ||
| 601 | + # 浮点类型保持原类型 | ||
| 602 | + self.assertEqual(float32_abs_bench.dtype, np.float32) | ||
| 603 | + | ||
| 604 | + # 验证eps值不同 | ||
| 605 | + int8_eps = int8_abs_bench_with_eps[0] - int8_abs_bench[0] | ||
| 606 | + float32_eps = float32_abs_bench_with_eps[0] - float32_abs_bench[0] | ||
| 607 | + | ||
| 608 | + # int8使用float64的epsilon | ||
| 609 | + self.assertAlmostEqual(int8_eps, np.finfo(np.float64).eps) | ||
| 610 | + # float32使用自己的epsilon,但由于float32精度限制,小值无法表示eps | ||
| 611 | + # 对于较大的值,eps会被舍入,所以实际差值可能为0 | ||
| 612 | + # 验证函数确实使用了float32的epsilon,即使结果被舍入 | ||
| 613 | + self.assertLessEqual(float32_eps, np.finfo(np.float32).eps) | ||
| 614 | + | ||
| 615 | + # ========== int8 类型测试用例 ========== | ||
| 616 | + | ||
| 617 | + | ||
| @@ -136,7 +136,7 @@ class TestCompare(unittest.TestCase): | |||
| 136 | res = self.compare._compare_core('api', tensor, tensor, False) | 136 | res = self.compare._compare_core('api', tensor, tensor, False) |
| 137 | 137 | ||
| 138 | self.assertEqual(res[0], 'pass') | 138 | self.assertEqual(res[0], 'pass') |
| 139 | - self.assertEqual(res[2], 'Current int64 data, only perform binary comparison.\n') | 139 | + self.assertEqual(res[2], 'Current int64 data, only perform binary comparison.') |
| 140 | 140 | ||
| 141 | def test_compare_core_with_buildin(self): | 141 | def test_compare_core_with_buildin(self): |
| 142 | interger = 1 | 142 | interger = 1 |
| @@ -481,6 +481,109 @@ class TestCompare(unittest.TestCase): | |||
| 481 | mock_get_path.assert_called_once() | 481 | mock_get_path.assert_called_once() |
| 482 | mock_write_csv.assert_called_once() | 482 | mock_write_csv.assert_called_once() |
| 483 | 483 | ||
| 484 | + def test_compare_torch_tensor_dtype_not_comparable(self): | ||
| 485 | + """测试数据类型不可比较的情况""" | ||
| 486 | + # Arrange | ||
| 487 | + import torch | ||
| 488 | + import numpy as np | ||
| 489 | + | ||
| 490 | + bench_output = torch.tensor([1, 2, 3], dtype=torch.float32) | ||
| 491 | + device_output = torch.tensor([1, 2, 3], dtype=torch.int32) # 浮点vs整型，不可比较 | ||
| 492 | + compare_column = CompareColumn() | ||
| 493 | + api_name = 'test_api' | ||
| 494 | + | ||
| 495 | + # Act | ||
| 496 | + status, result_column, message = self.compare._compare_torch_tensor( | ||
| 497 | + api_name, bench_output, device_output, compare_column, False | ||
| 498 | + ) | ||
| 499 | + | ||
| 500 | + # Assert | ||
| 501 | + self.assertEqual(status, 'error') | ||
| 502 | + self.assertIn('cannot compare', message) | ||
| 484 | 503 | ||
| 504 | + def test_compare_torch_tensor_empty_output(self): | ||
| 505 | + """测试空输出的情况""" | ||
| 506 | + # Arrange | ||
| 507 | + import torch | ||
| 508 | + import numpy as np | ||
| 509 | + | ||
| 510 | + bench_output = torch.tensor([], dtype=torch.float32) | ||
| 511 | + device_output = torch.tensor([], dtype=torch.float32) | ||
| 512 | + compare_column = CompareColumn() | ||
| 513 | + api_name = 'test_api' | ||
| 514 | + | ||
| 515 | + # Act | ||
| 516 | + status, result_column, message = self.compare._compare_torch_tensor( | ||
| 517 | + api_name, bench_output, device_output, compare_column, False | ||
| 518 | + ) | ||
| 519 | + | ||
| 520 | + # Assert | ||
| 521 | + self.assertEqual(status, 'SKIP') | ||
| 522 | + self.assertIn('There is not bench calculation result', message) | ||
| 523 | + | ||
| 524 | + def test_compare_torch_tensor_quantization_api_pass(self): | ||
| 525 | + """测试量化算子通过的情况""" | ||
| 526 | + # Arrange | ||
| 527 | + from unittest.mock import patch | ||
| 528 | + import torch | ||
| 529 | + import numpy as np | ||
| 530 | + | ||
| 531 | + from msprobe.pytorch.api_accuracy_checker.common.config import msCheckerConfig | ||
| 532 | + | ||
| 533 | + bench_output = torch.tensor([100, 200, 300], dtype=torch.int32) | ||
| 534 | + device_output = torch.tensor([100, 200, 300], dtype=torch.int32) # 完全匹配 | ||
| 535 | + compare_column = CompareColumn() | ||
| 536 | + api_name = 'npu_rms_norm_dynamic_quant' | ||
| 537 | + | ||
| 538 | + with patch.object(msCheckerConfig, 'quantization_api_list', | ||
| 539 | + ['npu_rms_norm_dynamic_quant', 'npu_dynamic_quant']): | ||
| 540 | + # Act | ||
| 541 | + status, result_column, message = self.compare._compare_torch_tensor( | ||
| 542 | + api_name, bench_output, device_output, compare_column, False | ||
| 543 | + ) | ||
| 544 | + | ||
| 545 | + # Assert | ||
| 546 | + self.assertEqual(status, 'pass') | ||
| 547 | + self.assertIn('less than or equal to 1, consider as pass', message) | ||
| 548 | + | ||
| 549 | + def test_compare_torch_tensor_integer_non_quantization(self): | ||
| 550 | + """测试整型输出但非量化算子的情况""" | ||
| 551 | + # Arrange | ||
| 552 | + import torch | ||
| 553 | + import numpy as np | ||
| 554 | + | ||
| 555 | + bench_output = torch.tensor([100, 200, 300], dtype=torch.int32) | ||
| 556 | + device_output = torch.tensor([100, 200, 300], dtype=torch.int32) | ||
| 557 | + compare_column = CompareColumn() | ||
| 558 | + api_name = 'regular_api' # 不在量化列表中 | ||
| 559 | + | ||
| 560 | + # Act | ||
| 561 | + status, result_column, message = self.compare._compare_torch_tensor( | ||
| 562 | + api_name, bench_output, device_output, compare_column, False | ||
| 563 | + ) | ||
| 564 | + | ||
| 565 | + # Assert - 应该执行布尔比较，对于完全匹配的结果应该是pass | ||
| 566 | + self.assertEqual(status, 'pass') | ||
| 567 | + | ||
| 568 | + def test_compare_torch_tensor_float_comparision(self): | ||
| 569 | + """测试浮点数比较的情况""" | ||
| 570 | + # Arrange | ||
| 571 | + import torch | ||
| 572 | + import numpy as np | ||
| 573 | + | ||
| 574 | + bench_output = torch.tensor([1.0, 2.0, 3.0], dtype=torch.float32) | ||
| 575 | + device_output = torch.tensor([1.00001, 2.00001, 3.00001], dtype=torch.float32) # 小误差 | ||
| 576 | + compare_column = CompareColumn() | ||
| 577 | + api_name = 'float_api' | ||
| 578 | + | ||
| 579 | + # Act | ||
| 580 | + status, result_column, message = self.compare._compare_torch_tensor( | ||
| 581 | + api_name, bench_output, device_output, compare_column, False | ||
| 582 | + ) | ||
| 583 | + | ||
| 584 | + # Assert - 应该执行浮点数比较 | ||
| 585 | + self.assertEqual(status, 'pass') | ||
| 586 | + | ||
| 587 | + | ||
| 485 | if __name__ == '__main__': | 588 | if __name__ == '__main__': |
| 486 | unittest.main() | 589 | unittest.main() |