from abc import ABC, abstractmethod
from typing import Optional
from atk.configs.results_config import TaskResult
from atk.configs.nodes_config import NodesConfig, Node
from atk.configs.case_report_config import CaseConfigReport
from atk.tasks.post_process.perf_compare_tool import PerfCompareExecutor
from atk.configs.standard_config import PerfStandard
from atk.configs.base_config import NONE_RET
from atk.common.log import Logger
logging = Logger().get_logger()
class BaseReportConfig(ABC):
def __init__(self, nodes_config: NodesConfig):
self.nodes_config = nodes_config
self.compare_nodes = nodes_config.get_compare_nodes()
self.report_title_factory = None
self.init_report_title_factory()
@abstractmethod
def init_report_title_factory(self):
"""
:self.report_title_factory = BaseReportTitleFactory(self):
"""
pass
def get_value(self, name, node=None):
return self.report_title_factory.get_value(name, node)
def get_print_name_list(self):
return self.report_title_factory.print_name_list
def get_print_table(self):
return self.report_title_factory.get_print_table()
class ResultReportConfig(BaseReportConfig):
perf_standard = None
def __init__(self, task_result: TaskResult):
super(ResultReportConfig, self).__init__(task_result.nodes)
self.task_result = task_result
self.float_format = ".4f"
if not ResultReportConfig.perf_standard:
ResultReportConfig.perf_standard = PerfStandard().get_standard(task_result.device, task_result.opp_perf)
@property
def name(self):
return self.case_report.name
@property
def case_report(self):
return CaseConfigReport(self.task_result.case_config)
@staticmethod
def function_titles():
return [
"create_dataset",
"save_dataset",
"load_dataset",
"init_resource",
"run_accuracy",
"run_e2e_performance",
"run_device_performance",
"run_opp_case",
"download_data",
"accuracy_calc",
]
def get_perf_ratio(self, base_perf, bm_perf) -> Optional[float]:
logging.debug(f"start get_perf_ratio, case_id: {self.case_report.id}, perf_name: {base_perf}")
if bm_perf is None or base_perf is None:
return NONE_RET
if base_perf == 0:
return 0.0
e2e_ratio = bm_perf / base_perf
return e2e_ratio
def init_report_title_factory(self):
from atk.tasks.report.report_title.result_report_title import ResultReportTitleFactory
self.report_title_factory = ResultReportTitleFactory(self)
def backend_perf(self, node, config):
if not self.task_result.performance:
return NONE_RET
attr = self.task_result.performance.performance_configs.get(node.get_backend_name())
if not attr:
return NONE_RET
ret = getattr(attr, config)
return ret
def get_compare_case_performance(self, main_perf, other_perf) -> Optional[bool]:
if (
not self.task_result.performance
or main_perf is None
or other_perf is None
):
return NONE_RET
return PerfCompareExecutor.compare_case_performance(main_perf, other_perf, self.perf_standard)
def get_compare_case_memory(self, main, other) -> Optional[bool]:
if (
not self.task_result.performance
or main is None
or other is None
):
return NONE_RET
return PerfCompareExecutor.compare_case_memory(main, other)
def get_function_time(self, node, fun_name):
if not self.task_result.performance:
return 0
perf_back = self.task_result.performance.performance_configs.get(node.get_backend_name())
if perf_back:
return getattr(perf_back.function_times, fun_name)
return 0
def run_times(self, node: Node):
func = self.function_titles()
ret = 0
for fun in func:
ret += self.get_function_time(node, fun)
return ret
def get_accuracy_pass(self, node: Node) -> Optional[bool]:
if not self.task_result.accuracy:
return NONE_RET
attr = self.task_result.accuracy.accuracy_configs.get(node.get_backend_name())
if not attr:
return NONE_RET
ret = True
for accuracy_config in attr:
if not accuracy_config.result:
ret = False
break
return ret
def get_e2e_config(self, node, name):
if not self.task_result.performance:
return NONE_RET
attr = self.task_result.performance.performance_configs.get(node.get_backend_name())
if not attr:
return NONE_RET
e2e_config = getattr(attr, "e2e_config")
if e2e_config:
return getattr(e2e_config, name)
else:
return NONE_RET
class CsvReport(ResultReportConfig):
def init_report_title_factory(self):
from atk.tasks.report.report_title.csv_report_title import CsvReportTitleFactory
self.report_title_factory = CsvReportTitleFactory(self)