import logging
import torch
import datetime
import os
import sys
import numpy as np
logging.basicConfig(level=logging.INFO, format='%(message)s', force=True)
logger = logging.getLogger(__name__)
def cal_relative_diff_np_isclose(real_data, expect_data, type_str='fp16'):
diff = abs(float(real_data) - float(expect_data))
result = diff / (np.abs(expect_data) + 10e-10)
return result
def display_output_np_isclose(real_data, expect_data, start, end, expect_fp32_data=None):
def display_inner(idx):
j = idx + start
diff_rate = cal_relative_diff_np_isclose(
real_data[j], expect_data[j])
if "inf" in str(expect_data[j]) or "nan" in str(expect_data[j]):
diff_abs = "inf" if "inf" in str(expect_data[j]) else "nan"
if expect_fp32_data is not None:
print_log('%08d \t %-7s \t %-7s \t %-7s \t %-7s \t %-7s' % (
start + idx, expect_fp32_data[j], expect_data[j], real_data[j], diff_abs, diff_rate))
else:
print_log('%08d \t %-7s \t %-7s \t %-7s \t %-7s' % (
start + idx, expect_data[j], real_data[j], diff_abs, diff_rate))
else:
diff_abs = abs(np.float64(
expect_data[j]) - np.float64(real_data[j]))
if expect_fp32_data is not None:
print_log('%08d \t %0.7f \t %0.7f \t %0.7f \t %0.7f \t %0.7f' % (
start + idx, expect_fp32_data[j], expect_data[j], real_data[j], diff_abs, diff_rate))
else:
print_log('%08d \t %0.7f \t %0.7f \t %0.7f \t %0.7f' % (
start + idx, expect_data[j], real_data[j], diff_abs, diff_rate))
print_log(
'---------------------------------------------------------------------------------------')
if expect_fp32_data is not None:
print_log(
'Loop \t ExpFP32Out \t ExpFP16Out \t NPUOut \tFpDiff(min) \t RateDiff')
else:
print_log('Loop \t ExpectOut \t RealOut \t FpDiff \t RateDiff')
print_log(
'---------------------------------------------------------------------------------------')
split_count = int(end - start)
if split_count <= 20:
for i in range(split_count + 1):
display_inner(i)
else:
for i in range(10):
display_inner(i)
print_log('... \t ... \t ... \t ... \t ...')
for i in range(split_count - 10 + 1, split_count + 1):
display_inner(i)
def print_log(data=None, level='INFO'):
print("[%s] [%s]-%s:%s - %s" % (datetime.datetime.now().strftime(
"%Y/%m/%d %H:%M:%S"), level, os.path.basename(sys._getframe().f_back.f_code.co_filename),
str(sys._getframe().f_back.f_lineno).zfill(4), data))
def display_error_output(real_data, expect_data, err_idx, relative_diff):
print_log(
'Error Line-----------------------------------------------------------------------------')
print_log('Loop \t ExpectOut \t RealOut \t FpDiff \t RateDiff')
print_log(
'---------------------------------------------------------------------------------------')
count = 0
len_err = len(err_idx)
for i in err_idx:
count += 1
if count < 10 or (90 < count < 100):
print_log('%08d \t %.7f \t %.7f \t %.7f \t %.7f' % (
i, expect_data[i], real_data[i], abs(np.float64(
expect_data[i]) - np.float64(real_data[i])),
relative_diff[count - 1]))
elif count == 10 or (count == 100 and len_err > 100):
dot_3 = '...'
print_log('%08s \t %07s \t %07s \t %07s \t %07s' %
(dot_3, dot_3, dot_3, dot_3, dot_3))
elif count > 100:
break
print_log(
'Max-RE line:---------------------------------------------------------------------------')
max_error = max(relative_diff)
m_idx_list = err_idx[np.where(relative_diff == max_error)]
m_count = 0
for m_idx in m_idx_list:
m_count += 1
if m_count < 4:
print_log('%08d \t %.7f \t %.7f \t %.7f \t %.7f' % (
m_idx, expect_data[m_idx], real_data[m_idx],
abs(np.float64(expect_data[m_idx]) -
np.float64(real_data[m_idx])),
max_error))
else:
break
print_log(
'---------------------------------------------------------------------------------------')
def check_result(expect, npu_result):
diff_thd=0.005
pct_thd=0.005
max_diff_hd=10
rtol=0.005
atol=0.000025
max_error_idx = 10000000
real_data = npu_result.cpu().to(torch.float32).numpy()
data_compe = expect.cpu().to(torch.float32).numpy()
real_data = real_data.flatten()
data_compe = data_compe.flatten()
if real_data.size == 0 and real_data.size == data_compe.size:
print_log(
'The npu_output is [],and it is same as bm_output, the result of data_compare is \"Pass\"')
return "Pass", 100.0, 0
start = 0
end = real_data.size - 1
if end < start:
end = start
max_error = 0
result = "Failed"
if real_data.size != data_compe.size:
print_log(
'Error,the size of npu output[%s] and benchmark[%s] is not equal.' % (real_data.size, data_compe.size))
return result, 0.0, max_error
overflows_count = data_compe[np.isinf(data_compe)].size + data_compe[np.isnan(data_compe)].size
if overflows_count > 0:
print_log('Overflow,size:%s,benchmark_output:%s, %s' % (
overflows_count, data_compe[np.isinf(data_compe)][0:10], data_compe[np.isnan(data_compe)][0:10]))
split_count = int(end - start + 1) if end != start else 1
print_log('split_count:%s; max_diff_hd:%s;' %
(float(split_count), max_diff_hd))
has_nan_inf = False
if 'nan' in str(real_data) or 'inf' in str(real_data) or 'nan' in str(data_compe) or 'inf' in str(data_compe):
has_nan_inf = True
if npu_result.dtype == torch.bfloat16:
rtol=0.0078125
atol=0.0001
diff_result = np.isclose(real_data.astype(np.float32), data_compe.astype(np.float32), rtol=rtol, atol=atol,
equal_nan=True)
else:
diff_result = np.isclose(real_data, data_compe, rtol=rtol, atol=atol, equal_nan=True)
err_idx = np.where(diff_result != np.array((True,)))[0]
if str(data_compe.dtype) == 'bool':
data_compe = data_compe.astype(np.int8)
real_data = real_data.astype(np.int8)
diff_abs = abs(data_compe - real_data)
b1 = np.maximum(np.abs(real_data), (np.abs(data_compe)))
b2 = float((1.0 / (1 << 14)) / diff_thd)
b = np.add(np.maximum(b1, b2), 10e-10)
eps = 10e-10
err_diff = diff_abs / (b + eps)
err_diff = err_diff[err_idx]
fulfill_percent = float(split_count - err_idx.size) / \
float(split_count) * 100.0
display_output_np_isclose(real_data, data_compe, start, end)
pct_thd = (1 - pct_thd) * 100.0
result = "Pass" if (fulfill_percent >= pct_thd) else "Failed"
if len(err_diff) > 0:
max_error = max(err_diff[0:max_error_idx])
if max_error >= max_diff_hd:
result = "Failed"
print_log(
'---------------------------------------------------------------------------------------')
print_log('Rtol \t Atol \t PctThd \t PctRlt \t Result')
print_log(
'---------------------------------------------------------------------------------------')
print_log('%.4f \t %.6f \t %.2f%% \t %.6f%% \t %s' %
(rtol, atol, pct_thd, fulfill_percent, result))
if len(err_diff) > 0:
print_log('Max-RelativeError is: %s. Threshold is: %s.' %
(max_error, max_diff_hd))
if result == "Failed":
display_error_output(real_data, data_compe,
err_idx, err_diff[0:max_error_idx])
return result, fulfill_percent