import numpy as np
from pathlib import Path
import matplotlib.pyplot as plt
from ..tools.common import load_json
from ..tools.common import save_json
plt.switch_backend('agg')
class TrainingMonitor():
def __init__(self, file_dir, arch, add_test=False):
'''
:param startAt: 重新开始训练的epoch点
'''
if isinstance(file_dir, Path):
pass
else:
file_dir = Path(file_dir)
file_dir.mkdir(parents=True, exist_ok=True)
self.arch = arch
self.file_dir = file_dir
self.H = {}
self.add_test = add_test
self.json_path = file_dir / (arch + "_training_monitor.json")
def reset(self,start_at):
if start_at > 0:
if self.json_path is not None:
if self.json_path.exists():
self.H = load_json(self.json_path)
for k in self.H.keys():
self.H[k] = self.H[k][:start_at]
def epoch_step(self, logs={}):
for (k, v) in logs.items():
l = self.H.get(k, [])
if not isinstance(v, np.float):
v = round(float(v), 4)
l.append(v)
self.H[k] = l
if self.json_path is not None:
save_json(data = self.H,file_path=self.json_path)
if len(self.H["loss"]) == 1:
self.paths = {key: self.file_dir / (self.arch + f'_{key.upper()}') for key in self.H.keys()}
if len(self.H["loss"]) > 1:
keys = [key for key, _ in self.H.items() if '_' not in key]
for key in keys:
N = np.arange(0, len(self.H[key]))
plt.style.use("ggplot")
plt.figure()
plt.plot(N, self.H[key], label=f"train_{key}")
plt.plot(N, self.H[f"valid_{key}"], label=f"valid_{key}")
if self.add_test:
plt.plot(N, self.H[f"test_{key}"], label=f"test_{key}")
plt.legend()
plt.xlabel("Epoch #")
plt.ylabel(key)
plt.title(f"Training {key} [Epoch {len(self.H[key])}]")
plt.savefig(str(self.paths[key]))
plt.close()