""" 日志模块
"""
import logging.config
from datetime import datetime, timezone, timedelta
from enum import Enum
from typing import Optional
class CustomFormatter(logging.Formatter):
def format(self, record):
log_msg = ""
if record.levelno > logging.INFO:
current_time = datetime.now(timezone(timedelta(hours=8))).strftime('%Y-%m-%d %H:%M:%S')
log_msg += f"{current_time} "
gpu_rank = torch.distributed.get_rank()
log_msg += f"[NPU Rank: {gpu_rank}] "
file_name = record.pathname
func_name = record.funcName
line_no = record.lineno
log_msg += f"{file_name} - {func_name}():{line_no} - "
log_msg += f"{record.levelname}: "
log_msg += f"{record.msg}"
return log_msg
if torch.distributed.get_rank() == 0:
log_msg += f"{record.msg}"
return log_msg
return None
class CustomHandler(logging.StreamHandler):
def emit(self, record):
log_entry = self.format(record)
if log_entry:
self.stream.write(log_entry + self.terminator)
class Formatters(Enum):
DEFAULT = {
'standard_formatter': {
'format': '{message}',
'style': '{',
},
'verbose_formatter': {
'format': '{message}',
'style': '{',
}
}
PROFILE = {
'standard_formatter': {
'()': 'tinker.utils.logger.CustomFormatter'
},
'verbose_formatter': {
'()': 'tinker.utils.logger.CustomFormatter'
}
}
logger = logging.getLogger('default')
profile_logger = logging.getLogger('profile')
def get_default_config():
log_config = {
'version': 1,
'disable_existing_loggers': False,
'formatters': {
'standard_formatter': {},
'verbose_formatter': {}
},
'handlers': {
'console': {
'level': 'NOTSET',
'class': 'logging.StreamHandler',
'formatter': 'standard_formatter'
},
'logfile': {
'level': 'NOTSET',
'class': 'logging.handlers.RotatingFileHandler',
'filename': '',
'mode': 'a',
'maxBytes': 1024 * 1024 * 100,
'backupCount': 5,
'formatter': 'verbose_formatter',
'encoding': 'utf-8',
}
},
'loggers': {
}
}
return log_config
def init_log(log_path: Optional[str], log_level):
"""
此方法用于初始化&更新默认的log
:param log_path: 日志存的地址
:param log_level: 等级
:return:
"""
log_config = get_default_config()
formatter = Formatters.DEFAULT
log_config['loggers']['default'] = {
'handlers': ['console'],
'level': 'NOTSET',
'propagate': False
}
log_config['handlers']['console']['level'] = log_level
log_config['handlers']['logfile']['level'] = log_level
log_config['loggers']['default']['level'] = log_level
log_config['formatters'].update(formatter.value)
if log_path:
log_config['loggers']['default']['handlers'].append('logfile')
log_config['handlers']['logfile']['filename'] = log_path
else:
del log_config['handlers']['logfile']
logging.config.dictConfig(log_config)
def init_profile_log(log_level):
"""
在多进程环境中使用的log,不支持文件输出
:param log_level: 等级
:return:
"""
log_config = get_default_config()
formatter = Formatters.PROFILE
log_config['loggers']['profile'] = {
'handlers': ['console'],
'level': 'NOTSET',
'propagate': False
}
import torch
global torch
if not torch.distributed.is_initialized():
raise RuntimeError('Before using method \'init_profile_log\', torch should be initialized.')
log_config['handlers']['console']['level'] = log_level
log_config['handlers']['logfile']['level'] = log_level
log_config['loggers']['profile']['level'] = log_level
log_config['handlers']['console']['class'] = "tinker.utils.logger.CustomHandler"
log_config['formatters'].update(formatter.value)
del log_config['handlers']['logfile']
logging.config.dictConfig(log_config)