| @@ -54,6 +54,22 @@ FASTAPI_WORKERS=1 | |||
| 54 | # ── Log ─────────────────────────────────────────────────────────────────────── | 54 | # ── Log ─────────────────────────────────────────────────────────────────────── |
| 55 | LOG_LEVEL=INFO | 55 | LOG_LEVEL=INFO |
| 56 | LOG_DIR=logs | 56 | LOG_DIR=logs |
| 57 | +# 日志轮转大小(loguru格式,如 "20 MB") | ||
| 58 | +LOG_ROTATION_SIZE=20 MB | ||
| 59 | +# 日志保留天数(0=不按天数清理,默认7天) | ||
| 60 | +LOG_RETENTION_DAYS=7 | ||
| 61 | +# 日志总空间上限(字节,0=不按空间清理,默认500MB) | ||
| 62 | +LOG_MAX_TOTAL_SIZE=524288000 | ||
| 63 | + | ||
| 64 | +# ── SDK 日志(openjiuwen SDK 的日志清理配置)──────────────────────────── | ||
| 65 | +# SDK 日志归档文件保留数量(轮转后最多保留 N 个 .gz,默认20) | ||
| 66 | +JIUWEN_LOG_BACKUP_COUNT=20 | ||
| 67 | +# SDK 日志单文件大小阈值(字节,达到即触发轮转,默认20MB) | ||
| 68 | +JIUWEN_LOG_MAX_BYTES=20971520 | ||
| 69 | +# SDK 日志归档保留天数(0=不按天数清理,默认7天) | ||
| 70 | +JIUWEN_LOG_RETENTION_DAYS=7 | ||
| 71 | +# SDK 日志总空间上限(字节,0=不按空间清理,默认500MB) | ||
| 72 | +JIUWEN_LOG_MAX_TOTAL_SIZE=524288000 | ||
| 57 | 73 | ||
| 58 | # ── LLM(上游保留字段,供其他 Agent 参考;EDPAgent 不读此组)────────────── | 74 | # ── LLM(上游保留字段,供其他 Agent 参考;EDPAgent 不读此组)────────────── |
| 59 | LLM_PROVIDER=llm_OpenAI | 75 | LLM_PROVIDER=llm_OpenAI |
| @@ -20,6 +20,7 @@ import socket | |||
| 20 | import sys | 20 | import sys |
| 21 | import time | 21 | import time |
| 22 | from contextlib import asynccontextmanager | 22 | from contextlib import asynccontextmanager |
| 23 | +from datetime import datetime, timedelta, timezone | ||
| 23 | from pathlib import Path | 24 | from pathlib import Path |
| 24 | from typing import Any, Optional | 25 | from typing import Any, Optional |
| 25 | 26 | ||
| @@ -128,6 +129,69 @@ def dynamic_format(record): | |||
| 128 | ) | 129 | ) |
| 129 | 130 | ||
| 130 | 131 | ||
| 132 | +def _cleanup_logs(files): | ||
| 133 | + """ | ||
| 134 | + loguru retention 回调函数:按天数 + 按总空间清理。 | ||
| 135 | + | ||
| 136 | + loguru 在每次轮转后调用此函数,传入所有匹配的日志文件列表。 | ||
| 137 | + 注意:loguru 传入的可能是 Path 对象或字符串,统一转为 Path 处理。 | ||
| 138 | + | ||
| 139 | + 清理逻辑: | ||
| 140 | + 1. 按保留天数清理:删除修改时间超过 retention_days 的归档文件 | ||
| 141 | + 2. 按总空间清理:如果总空间超限,从最旧归档文件开始逐一删除 | ||
| 142 | + | ||
| 143 | + 注意:活跃文件(无 .gz 后缀的 .log 文件)不参与删除,只删 .gz 归档文件。 | ||
| 144 | + """ | ||
| 145 | + retention_days = int(os.getenv("LOG_RETENTION_DAYS", "7")) | ||
| 146 | + max_total_size = int(os.getenv("LOG_MAX_TOTAL_SIZE", "524288000")) | ||
🟡 Medium Priority
与此形成对比, 失败模式:如果环境变量被显式设置为空字符串(如 额外影响:默认值 建议:改用 ![]() ![]() 不准确? | |||
| 147 | + | ||
| 148 | + if not files: | ||
| 149 | + return | ||
| 150 | + | ||
| 151 | + # 统一转为 Path 对象(loguru 可能传入字符串) | ||
| 152 | + files = [Path(f) for f in files] | ||
| 153 | + | ||
| 154 | + # 按文件名区分活跃文件和归档文件(不用 mtime 猜测,避免误删刚压缩的归档) | ||
| 155 | + # 活跃文件:无 .gz 后缀的 .log 文件(正在被 loguru 写入) | ||
| 156 | + # 归档文件:.gz 后缀的压缩文件(已被轮转压缩) | ||
| 157 | + active_files = [f for f in files if f.exists() and not f.name.endswith(".gz")] | ||
| 158 | + archive_files = [f for f in files if f.exists() and f.name.endswith(".gz")] | ||
| 159 | + | ||
| 160 | + if not archive_files: | ||
| 161 | + return | ||
| 162 | + | ||
| 163 | + # 1. 按天数清理(只清理归档文件) | ||
| 164 | + if retention_days > 0: | ||
| 165 | + cutoff_time = (datetime.now(tz=timezone.utc) - timedelta(days=retention_days)).timestamp() | ||
| 166 | + for f in archive_files: | ||
| 167 | + try: | ||
| 168 | + if f.stat().st_mtime < cutoff_time: | ||
| 169 | + f.unlink() | ||
| 170 | + except OSError as e: | ||
| 171 | + # 用 print 到 stderr,不用 loguru(避免在 retention 回调中递归调用 loguru) | ||
| 172 | + sys.stderr.write(f"[log_cleanup] WARN 按天数清理删除失败: {f} -> {e}\n") | ||
| 173 | + | ||
| 174 | + # 2. 按总空间清理(活跃文件 + 剩余归档文件的总和) | ||
| 175 | + if max_total_size > 0: | ||
| 176 | + # 重新获取未被天数清理删掉的归档文件 | ||
| 177 | + remaining_archives = [(f.stat().st_mtime, f) for f in archive_files if f.exists()] | ||
| 178 | + active_size = sum(f.stat().st_size for f in active_files if f.exists()) | ||
| 179 | + archive_total = sum(f.stat().st_size for _, f in remaining_archives) | ||
| 180 | + total_size = active_size + archive_total | ||
| 181 | + | ||
| 182 | + if total_size > max_total_size: | ||
| 183 | + remaining_archives.sort(key=lambda x: x[0]) # 最旧在前 | ||
| 184 | + for _mtime, f in remaining_archives: | ||
| 185 | + if total_size <= max_total_size: | ||
| 186 | + break | ||
| 187 | + try: | ||
| 188 | + file_size = f.stat().st_size | ||
| 189 | + f.unlink() | ||
| 190 | + total_size -= file_size | ||
| 191 | + except OSError as e: | ||
| 192 | + sys.stderr.write(f"[log_cleanup] WARN 按空间清理删除失败: {f} -> {e}\n") | ||
| 193 | + | ||
| 194 | + | ||
| 131 | def setup_logging() -> None: | 195 | def setup_logging() -> None: |
| 132 | settings = get_settings() | 196 | settings = get_settings() |
| 133 | 197 | ||
| @@ -152,8 +216,8 @@ def setup_logging() -> None: | |||
| 152 | logger.add( | 216 | logger.add( |
| 153 | log_file, | 217 | log_file, |
| 154 | level=settings.log_level.upper() if settings.log_level else "INFO", | 218 | level=settings.log_level.upper() if settings.log_level else "INFO", |
| 155 | - rotation="20 MB", | 219 | + rotation=settings.log_rotation_size, |
| 156 | - retention="7 days", | 220 | + retention=_cleanup_logs, |
| 157 | compression="gz", | 221 | compression="gz", |
| 158 | format=dynamic_format, | 222 | format=dynamic_format, |
| 159 | filter=lambda record: len(record["extra"]) == 0 or "source" not in record["extra"] | 223 | filter=lambda record: len(record["extra"]) == 0 or "source" not in record["extra"] |
| @@ -162,8 +226,8 @@ def setup_logging() -> None: | |||
| 162 | logger.add( | 226 | logger.add( |
| 163 | audit_log, | 227 | audit_log, |
| 164 | level="INFO", | 228 | level="INFO", |
| 165 | - rotation="20 MB", | 229 | + rotation=settings.log_rotation_size, |
| 166 | - retention="30 days", | 230 | + retention=_cleanup_logs, |
| 167 | compression="gz", | 231 | compression="gz", |
| 168 | format="<green>{time:YYYY-MM-DD HH:mm:ss.SSS}</green> \x01 " | 232 | format="<green>{time:YYYY-MM-DD HH:mm:ss.SSS}</green> \x01 " |
| 169 | "<level>{level: <8}</level> \x01 " | 233 | "<level>{level: <8}</level> \x01 " |
| @@ -462,9 +526,23 @@ async def lifespan(fastapi_app: FastAPI): | |||
| 462 | "[A2AService] DB持久化已禁用(runtime_db_enabled=false),使用纯Redis模式", | 526 | "[A2AService] DB持久化已禁用(runtime_db_enabled=false),使用纯Redis模式", |
| 463 | ) | 527 | ) |
| 464 | 528 | ||
| 529 | + # 在 initialize() 之前,覆盖 SDK 日志配置(轮转大小、备份数量) | ||
| 530 | + from openjiuwen.core.common.logging.log_config import configure_log_config | ||
| 531 | + from openjiuwen.core.common.logging.default.constant import DEFAULT_INNER_LOG_CONFIG | ||
| 532 | + custom_log_config = DEFAULT_INNER_LOG_CONFIG.copy() | ||
| 533 | + custom_log_config["backup_count"] = int(os.getenv("JIUWEN_LOG_BACKUP_COUNT", 20)) | ||
| 534 | + custom_log_config["max_bytes"] = int(os.getenv("JIUWEN_LOG_MAX_BYTES", 20971520)) | ||
🟡 Medium Priority 在 而 失败模式:若环境变量被设为空字符串(如 建议:改用 ![]() ![]() 不准确? | |||
| 535 | + configure_log_config(custom_log_config) | ||
| 536 | + | ||
| 465 | await initialize() | 537 | await initialize() |
| 466 | logger.info("[A2AService] Agent 初始化完成") | 538 | logger.info("[A2AService] Agent 初始化完成") |
| 467 | 539 | ||
| 540 | + # SDK 日志清理:用 foundation Handler 替换 SDK 默认 Handler, | ||
| 541 | + # 为 SDK 的 21 个 logger 增加 gzip 压缩 + 按天数清理 + 按总空间清理能力 | ||
| 542 | + from common.sdk_log_cleaner import setup_sdk_log_cleaner | ||
| 543 | + setup_sdk_log_cleaner() | ||
| 544 | + logger.info("[A2AService] SDK 日志清理已启用(gzip压缩+按天数+按空间清理)") | ||
| 545 | + | ||
| 468 | # read=None 关闭 SSE 流读超时:子 Agent 进入并行工作流阶段事件流会出现 >5s 空档 | 546 | # read=None 关闭 SSE 流读超时:子 Agent 进入并行工作流阶段事件流会出现 >5s 空档 |
| 469 | # (VA 跑批),默认 5s read 超时会误杀子 Agent;整体时长由框架 sub_agent_timeout_seconds | 547 | # (VA 跑批),默认 5s read 超时会误杀子 Agent;整体时长由框架 sub_agent_timeout_seconds |
| 470 | # (asyncio.wait_for)兜底(见 ref_deploy_a2a_troubleshooting #6)。 | 548 | # (asyncio.wait_for)兜底(见 ref_deploy_a2a_troubleshooting #6)。 |
| @@ -0,0 +1,278 @@ | |||
| 1 | +# coding: utf-8 | ||
| 2 | +# Copyright (c) Huawei Technologies Co., Ltd. 2026-2026. All rights reserved | ||
| 3 | + | ||
| 4 | +"""SDK 日志清理器:用 foundation Handler 替换 SDK 默认 Handler。 | ||
| 5 | + | ||
| 6 | +设计说明 | ||
| 7 | +-------- | ||
| 8 | +- openjiuwen SDK 内部使用 Python logging(SafeRotatingFileHandler)记录日志 | ||
| 9 | +- SDK 默认只支持「按大小轮转 + 按数量保留」,不支持 gzip 压缩和按天/按空间清理 | ||
| 10 | +- foundation 的 CompressedRotatingFileHandler 支持轮转 + gzip 压缩 | ||
| 11 | +- 通过继承 SDK 的 DefaultLogger 重写 _setup_logger(),用 foundation Handler 替换 SDK Handler | ||
| 12 | +- 替换后 SDK 日志获得:gzip 压缩 + 按天数清理 + 按总空间清理 | ||
| 13 | + | ||
| 14 | +清理逻辑与 a2a/VA 保持一致: | ||
| 15 | +- 轮转时触发清理(不是定时任务) | ||
| 16 | +- 按天数清理:删除超过 retention_days 的 .gz 归档文件 | ||
| 17 | +- 按总空间清理:总空间超限时从最旧 .gz 归档开始逐一删除 | ||
| 18 | +- 活跃文件(无 .gz 后缀)不参与删除 | ||
| 19 | +""" | ||
| 20 | + | ||
| 21 | +import os | ||
| 22 | +import sys | ||
| 23 | +from pathlib import Path | ||
| 24 | +from datetime import datetime, timedelta, timezone | ||
| 25 | + | ||
| 26 | +# 从 foundation 导入 Handler(支持 gzip 压缩) | ||
| 27 | +from openjiuwen_runtime.foundation.log.handler import CompressedRotatingFileHandler | ||
| 28 | + | ||
| 29 | +# 从 SDK 导入 DefaultLogger 和相关类(不修改 SDK,只继承) | ||
| 30 | +from openjiuwen.core.common.logging.default.default_impl import ( | ||
| 31 | + DefaultLogger, | ||
| 32 | + SafeRotatingFileHandler, | ||
| 33 | + ContextFilter, | ||
| 34 | +) | ||
| 35 | +from openjiuwen.core.common.logging.manager import LogManager | ||
| 36 | + | ||
| 37 | + | ||
| 38 | +class CleanableCompressedRotatingFileHandler(CompressedRotatingFileHandler): | ||
| 39 | + """扩展 foundation 的 CompressedRotatingFileHandler,增加按天数和按空间清理。 | ||
| 40 | + | ||
| 41 | + 继承关系: | ||
| 42 | + CleanableCompressedRotatingFileHandler | ||
| 43 | + → CompressedRotatingFileHandler (foundation,提供 gzip 压缩) | ||
| 44 | + → RotatingFileHandler (Python logging) | ||
| 45 | + """ | ||
| 46 | + | ||
| 47 | + def __init__( | ||
| 48 | + self, | ||
| 49 | + filename: str, | ||
| 50 | + mode: str = "a", | ||
| 51 | + max_bytes: int = 0, | ||
| 52 | + backup_count: int = 0, | ||
| 53 | + encoding: str = "utf-8", | ||
| 54 | + delay: bool = False, | ||
| 55 | + retention_days: int = 0, | ||
| 56 | + max_total_size: int = 0, | ||
| 57 | + ): | ||
| 58 | + super().__init__( | ||
| 59 | + filename=filename, | ||
| 60 | + mode=mode, | ||
| 61 | + maxBytes=max_bytes, | ||
| 62 | + backupCount=backup_count, | ||
| 63 | + encoding=encoding, | ||
| 64 | + delay=delay, | ||
| 65 | + ) | ||
| 66 | + self.retention_days = retention_days | ||
| 67 | + self.max_total_size = max_total_size | ||
| 68 | + | ||
| 69 | + def doRollover(self) -> None: # noqa: N802 继承自标准库 RotatingFileHandler,必须保持原名 | ||
| 70 | + # 1. 调用 foundation 的轮转 + gzip 压缩逻辑 | ||
| 71 | + super().doRollover() | ||
| 72 | + | ||
| 73 | + # 2. 清理逻辑(与 a2a/VA 保持一致,只删 .gz) | ||
| 74 | + if self.retention_days > 0: | ||
| 75 | + self.cleanup_by_retention() | ||
| 76 | + if self.max_total_size > 0: | ||
| 77 | + self.cleanup_by_total_size() | ||
| 78 | + | ||
| 79 | + def cleanup_by_retention(self) -> None: | ||
| 80 | + """按天数清理(只清理 .gz 归档文件,不删活跃文件)""" | ||
| 81 | + log_path = Path(self.baseFilename) | ||
| 82 | + log_dir = log_path.parent | ||
| 83 | + log_name = log_path.name | ||
| 84 | + cutoff_time = (datetime.now(tz=timezone.utc) - timedelta(days=self.retention_days)).timestamp() | ||
| 85 | + | ||
| 86 | + for f in log_dir.glob(f"{log_name}*.gz"): | ||
| 87 | + if f.is_file(): | ||
| 88 | + try: | ||
| 89 | + if f.stat().st_mtime < cutoff_time: | ||
| 90 | + f.unlink() | ||
| 91 | + except OSError as e: | ||
| 92 | + # 用 print 到 stderr,不用 loguru(避免在 Handler 中递归调用日志) | ||
| 93 | + sys.stderr.write(f"[sdk_log_cleanup] WARN 按天数清理删除失败: {f} -> {e}\n") | ||
| 94 | + | ||
| 95 | + def cleanup_by_total_size(self) -> None: | ||
| 96 | + """按总空间清理(活跃文件计入总量但不删除,只删 .gz 归档文件)""" | ||
| 97 | + log_path = Path(self.baseFilename) | ||
| 98 | + log_dir = log_path.parent | ||
| 99 | + log_name = log_path.name | ||
| 100 | + active_file = log_path | ||
| 101 | + | ||
| 102 | + active_size = active_file.stat().st_size if active_file.exists() else 0 | ||
| 103 | + archive_files = [] | ||
| 104 | + for f in log_dir.glob(f"{log_name}*.gz"): | ||
| 105 | + if f.is_file(): | ||
| 106 | + try: | ||
| 107 | + archive_files.append((f.stat().st_mtime, f)) | ||
| 108 | + except OSError as e: | ||
| 109 | + sys.stderr.write(f"[sdk_log_cleanup] WARN 扫描归档文件 stat 失败: {f} -> {e}\n") | ||
| 110 | + continue | ||
| 111 | + | ||
| 112 | + archive_total = sum(f.stat().st_size for _, f in archive_files if f.exists()) | ||
| 113 | + total_size = active_size + archive_total | ||
| 114 | + if total_size <= self.max_total_size: | ||
| 115 | + return | ||
| 116 | + | ||
| 117 | + archive_files.sort(key=lambda x: x[0]) # 最旧在前 | ||
| 118 | + for mtime, f in archive_files: | ||
| 119 | + if total_size <= self.max_total_size: | ||
| 120 | + break | ||
| 121 | + try: | ||
| 122 | + file_size = f.stat().st_size | ||
| 123 | + f.unlink() | ||
| 124 | + total_size -= file_size | ||
| 125 | + except OSError as e: | ||
| 126 | + sys.stderr.write(f"[sdk_log_cleanup] WARN 按空间清理删除失败: {f} -> {e}\n") | ||
| 127 | + | ||
| 128 | + | ||
| 129 | +class CleanableDefaultLogger(DefaultLogger): | ||
| 130 | + """继承 SDK 的 DefaultLogger,在 _setup_logger() 中用 foundation Handler 替换 SDK Handler。 | ||
| 131 | + | ||
| 132 | + 继承关系: | ||
| 133 | + CleanableDefaultLogger | ||
| 134 | + → DefaultLogger (SDK) | ||
| 135 | + """ | ||
| 136 | + | ||
| 137 | + def __init__(self, log_type: str, config: dict, | ||
| 138 | + retention_days: int = 0, max_total_size: int = 0): | ||
| 139 | + self._retention_days = retention_days | ||
| 140 | + self._max_total_size = max_total_size | ||
| 141 | + super().__init__(log_type, config) | ||
| 142 | + | ||
| 143 | + def _setup_logger(self) -> None: | ||
| 144 | + """重写 _setup_logger: | ||
| 145 | + 1. 调用 SDK 父类完成基础配置(console handler、level 等) | ||
| 146 | + 2. 将 SafeRotatingFileHandler 替换为 foundation 的 CleanableCompressedRotatingFileHandler | ||
| 147 | + """ | ||
| 148 | + # 先调用 SDK 父类完成基础配置 | ||
| 149 | + super()._setup_logger() | ||
| 150 | + | ||
| 151 | + # 遍历 handlers,将 SDK 的 SafeRotatingFileHandler 替换为 foundation 的 Handler | ||
| 152 | + old_handlers = self._logger.handlers[:] | ||
| 153 | + for handler in old_handlers: | ||
| 154 | + if isinstance(handler, SafeRotatingFileHandler): | ||
| 155 | + # 移除 SDK 的 Handler | ||
| 156 | + self._logger.removeHandler(handler) | ||
| 157 | + handler.close() | ||
| 158 | + | ||
| 159 | + # 用 foundation 的 Handler 替换(获得压缩 + 清理能力) | ||
| 160 | + # backupCount/maxBytes 优先从环境变量读取(与 app.py 中的 configure_log_config 保持一致), | ||
| 161 | + # 避免旧 handler(SDK 默认值 20)覆盖用户配置 | ||
| 162 | + env_backup_count = int(os.getenv("JIUWEN_LOG_BACKUP_COUNT", handler.backupCount)) | ||
| 163 | + env_max_bytes = int(os.getenv("JIUWEN_LOG_MAX_BYTES", handler.maxBytes)) | ||
| 164 | + cleanable_handler = CleanableCompressedRotatingFileHandler( | ||
| 165 | + filename=handler.baseFilename, | ||
| 166 | + max_bytes=env_max_bytes, | ||
| 167 | + backup_count=env_backup_count, | ||
| 168 | + encoding="utf-8", | ||
| 169 | + retention_days=self._retention_days, | ||
| 170 | + max_total_size=self._max_total_size, | ||
| 171 | + ) | ||
| 172 | + cleanable_handler.addFilter(ContextFilter(self.log_type)) | ||
| 173 | + cleanable_handler.setFormatter(handler.formatter) | ||
| 174 | + cleanable_handler.setLevel(handler.level) | ||
| 175 | + self._logger.addHandler(cleanable_handler) | ||
| 176 | + | ||
| 177 | + | ||
| 178 | +# SDK 中所有 log_type 列表(含内置 4 个 + 懒加载的全部) | ||
| 179 | +# 来源:openjiuwen/core/common/logging/__init__.py 中的 LazyLogger 定义 | ||
| 180 | +_ALL_SDK_LOG_TYPES = [ | ||
| 181 | + # 内置(_Builtin_LOG_TYPES,initialize() 时创建) | ||
| 182 | + "common", "interface", "prompt_builder", "performance", | ||
| 183 | + # 懒加载(首次 get_logger() 时创建) | ||
| 184 | + "agent", "multi_agent", "workflow", "session", "controller", "runner", | ||
| 185 | + "sys_operation", "llm", "tool", "prompt", "store", "memory", "retrieval", | ||
| 186 | + "context_engine", "graph", "operator", "mcp", "team", | ||
| 187 | +] | ||
| 188 | + | ||
| 189 | + | ||
| 190 | +def setup_sdk_log_cleaner() -> None: | ||
| 191 | + """在 SDK 初始化完成后,替换 SDK 的 logger 为带清理逻辑的版本。 | ||
| 192 | + | ||
| 193 | + 关键步骤: | ||
| 194 | + 1. 预触发所有懒加载 logger(遍历全部 log_type 调用 get_logger) | ||
| 195 | + 2. 遍历所有已创建的 logger,用 register_logger 替换为 CleanableDefaultLogger | ||
| 196 | + 3. 压缩残留的未压缩归档(.1/.2/... → .1.gz/.2.gz/...) | ||
| 197 | + | ||
| 198 | + 注意:必须在 await initialize() 之后调用。 | ||
| 199 | + | ||
| 200 | + 背景:configure_log_config() 在 initialize() 之前执行,此时 SDK 用原生 | ||
| 201 | + SafeRotatingFileHandler(不压缩)。如果此时活跃文件已超 maxBytes,轮转会 | ||
| 202 | + 生成 .1/.2/...(未压缩)。本函数第 3 步把这些残留文件压缩成 .gz,否则 | ||
| 203 | + 它们不会被清理逻辑扫描到(清理只扫 .gz),会永久残留。 | ||
| 204 | + """ | ||
| 205 | + retention_days = int(os.getenv("JIUWEN_LOG_RETENTION_DAYS", 7)) | ||
| 206 | + max_total_size = int(os.getenv("JIUWEN_LOG_MAX_TOTAL_SIZE", 524288000)) | ||
| 207 | + | ||
| 208 | + # 第 1 步:预触发所有懒加载 logger,确保它们被创建并加入 _loggers dict | ||
| 209 | + for log_type in _ALL_SDK_LOG_TYPES: | ||
| 210 | + try: | ||
| 211 | + LogManager.get_logger(log_type) | ||
| 212 | + except Exception as e: | ||
| 213 | + # 某些 logger 可能因配置缺失创建失败,跳过但不静默 | ||
| 214 | + sys.stderr.write(f"[sdk_log_cleanup] WARN 预触发 logger 失败 log_type={log_type}: {e}\n") | ||
| 215 | + | ||
| 216 | + # 第 2 步:遍历所有已创建的 logger,逐个替换 | ||
| 217 | + all_loggers = LogManager.get_all_loggers() | ||
| 218 | + for log_type, original_logger in all_loggers.items(): | ||
| 219 | + config = original_logger.get_config() | ||
| 220 | + | ||
| 221 | + cleanable_logger = CleanableDefaultLogger( | ||
| 222 | + log_type=log_type, | ||
| 223 | + config=config, | ||
| 224 | + retention_days=retention_days, | ||
| 225 | + max_total_size=max_total_size, | ||
| 226 | + ) | ||
| 227 | + | ||
| 228 | + LogManager.register_logger(log_type, cleanable_logger) | ||
| 229 | + | ||
| 230 | + # 第 3 步:压缩残留的未压缩归档(configure_log_config 到本函数之间产生的 .1/.2/...) | ||
| 231 | + _compress_legacy_archives(all_loggers) | ||
| 232 | + | ||
| 233 | + | ||
| 234 | +def _compress_legacy_archives(all_loggers: dict) -> None: | ||
| 235 | + """把 logger 目录下残留的 .1/.2/...(未压缩)归档压缩成 .gz。 | ||
| 236 | + | ||
| 237 | + 只处理 .1/.2/... 这种纯数字后缀的文件,不处理 .1.gz(已压缩)。 | ||
| 238 | + 压缩时如果目标 .gz 已存在,直接删除未压缩版本(避免覆盖)。 | ||
| 239 | + """ | ||
| 240 | + import gzip | ||
| 241 | + import shutil | ||
| 242 | + | ||
| 243 | + seen_keys = set() # (log_dir, log_name) 组合去重,避免同目录不同文件名被误跳过 | ||
| 244 | + for log_type, logger in all_loggers.items(): | ||
| 245 | + # 从 logger 的 handler 拿 baseFilename | ||
| 246 | + py_logger = getattr(logger, "_logger", None) or getattr(logger, "py_logger", None) | ||
| 247 | + if py_logger is None: | ||
| 248 | + continue | ||
| 249 | + for handler in py_logger.handlers: | ||
| 250 | + base_filename = getattr(handler, "baseFilename", None) | ||
| 251 | + if not base_filename: | ||
| 252 | + continue | ||
| 253 | + log_dir = str(Path(base_filename).parent) | ||
| 254 | + log_name = Path(base_filename).name | ||
| 255 | + key = (log_dir, log_name) | ||
| 256 | + if key in seen_keys: | ||
| 257 | + continue | ||
| 258 | + seen_keys.add(key) | ||
| 259 | + # 扫描 .1/.2/... 这种纯数字后缀(不带 .gz) | ||
| 260 | + for legacy in Path(log_dir).glob(f"{log_name}.[0-9]*"): | ||
| 261 | + if legacy.suffix == ".gz": | ||
| 262 | + continue # 已压缩 | ||
| 263 | + # 检查是否是纯数字后缀(如 jiuwen.log.1) | ||
| 264 | + suffix = legacy.name[len(log_name) + 1:] | ||
| 265 | + if not suffix.isdigit(): | ||
| 266 | + continue | ||
| 267 | + gz_path = legacy.with_suffix(legacy.suffix + ".gz") | ||
| 268 | + if gz_path.exists(): | ||
| 269 | + # 目标 .gz 已存在,直接删除未压缩版本 | ||
| 270 | + legacy.unlink() | ||
| 271 | + continue | ||
| 272 | + try: | ||
| 273 | + with open(legacy, "rb") as src, gzip.open(gz_path, "wb") as dst: | ||
| 274 | + shutil.copyfileobj(src, dst) | ||
| 275 | + legacy.unlink() | ||
| 276 | + except Exception as e: | ||
| 277 | + # 压缩失败保留原文件,避免数据丢失 | ||
| 278 | + sys.stderr.write(f"[sdk_log_cleanup] WARN 压缩残留归档失败 {legacy}: {e}\n") | ||
| @@ -88,6 +88,22 @@ class Settings(BaseSettings): | |||
| 88 | # ── 日志 ──────────────────────────────────────────────────────────────── | 88 | # ── 日志 ──────────────────────────────────────────────────────────────── |
| 89 | log_level: Optional[str] = None | 89 | log_level: Optional[str] = None |
| 90 | log_dir: Optional[str] = None | 90 | log_dir: Optional[str] = None |
| 91 | + # 日志轮转大小(loguru格式,如 "20 MB") | ||
| 92 | + log_rotation_size: str = "20 MB" | ||
| 93 | + # 日志保留天数(0=不按天数清理) | ||
| 94 | + log_retention_days: int = 7 | ||
| 95 | + # 日志总空间上限(字节,0=不按空间清理,默认500MB) | ||
| 96 | + log_max_total_size: int = 524288000 | ||
| 97 | + | ||
| 98 | + # ── SDK 日志(openjiuwen SDK 的日志清理配置)──────────────────────────── | ||
| 99 | + # SDK 日志单文件大小阈值(字节,达到即触发轮转,默认20MB) | ||
| 100 | + jiuwen_log_max_bytes: int = 20971520 | ||
| 101 | + # SDK 日志归档文件保留数量(轮转后最多保留 N 个 .gz,默认20) | ||
| 102 | + jiuwen_log_backup_count: int = 20 | ||
| 103 | + # SDK 日志归档保留天数(0=不按天数清理,默认7天) | ||
| 104 | + jiuwen_log_retention_days: int = 7 | ||
| 105 | + # SDK 日志总空间上限(字节,0=不按空间清理,默认500MB) | ||
| 106 | + jiuwen_log_max_total_size: int = 524288000 | ||
| 91 | 107 | ||
| 92 | @property | 108 | @property |
| 93 | def redis_url(self) -> str: | 109 | def redis_url(self) -> str: |
| @@ -0,0 +1,229 @@ | |||
| 1 | +# coding: utf-8 | ||
| 2 | +# Copyright (c) Huawei Technologies Co., Ltd. 2026-2026. All rights reserved | ||
| 3 | + | ||
| 4 | +"""CleanableCompressedRotatingFileHandler 日志清理逻辑单元测试。 | ||
| 5 | + | ||
| 6 | +覆盖场景: | ||
| 7 | +1. 按天数清理:删除超过 retention_days 的 .gz 归档,保留新归档与活跃文件 | ||
| 8 | +2. 按总空间清理:超限时从最旧 .gz 归档开始逐一删除,活跃文件不删 | ||
| 9 | +3. 异常分支:归档文件 stat/unlink 失败时输出 stderr 警告,不抛出 | ||
| 10 | + | ||
| 11 | +实现说明: | ||
| 12 | +- conftest.py 的 _OpenJiuwenStubFinder 会将 openjiuwen 包替换为 MagicMock, | ||
| 13 | + 但 openjiuwen_runtime(foundation)通过 sys.path.append 加载真实包。 | ||
| 14 | +- CleanableCompressedRotatingFileHandler 的父类 CompressedRotatingFileHandler | ||
| 15 | + 来自真实 foundation,可正常 __init__ 构造。 | ||
| 16 | +- retention_days / max_total_size 通过构造参数传入,cleanup_by_retention / | ||
| 17 | + cleanup_by_total_size 是公开方法,直接调用测试。 | ||
| 18 | +""" | ||
| 19 | + | ||
| 20 | +from __future__ import annotations | ||
| 21 | + | ||
| 22 | +import gzip | ||
| 23 | +import io | ||
| 24 | +import os | ||
| 25 | +import sys | ||
| 26 | +import unittest | ||
| 27 | +from datetime import datetime, timedelta, timezone | ||
| 28 | +from pathlib import Path | ||
| 29 | +from tempfile import TemporaryDirectory | ||
| 30 | +from unittest.mock import patch | ||
| 31 | + | ||
| 32 | +# 显式从源码目录加载真实 foundation 包,避免被 conftest 的 stub 污染。 | ||
| 33 | +_FOUNDATION_ROOT = Path(__file__).resolve().parents[4] / "foundation" | ||
| 34 | +if str(_FOUNDATION_ROOT) not in sys.path: | ||
| 35 | + sys.path.append(str(_FOUNDATION_ROOT)) | ||
| 36 | +# 清除可能被 stub 污染的缓存,强制重新解析真实模块 | ||
| 37 | +for _mod_name in list(sys.modules.keys()): | ||
| 38 | + if _mod_name == "openjiuwen_runtime" or _mod_name.startswith("openjiuwen_runtime."): | ||
| 39 | + _mod = sys.modules.get(_mod_name) | ||
| 40 | + if _mod is not None and not hasattr(_mod, "__file__"): | ||
| 41 | + del sys.modules[_mod_name] | ||
| 42 | + | ||
| 43 | +# 注意:common.sdk_log_cleaner 在模块加载期会 import openjiuwen SDK, | ||
| 44 | +# 在 stub 环境下可正常加载(SDK 被 MagicMock 替换)。 | ||
| 45 | +from common.sdk_log_cleaner import CleanableCompressedRotatingFileHandler # noqa: E402 | ||
| 46 | + | ||
| 47 | + | ||
| 48 | +def _write_bytes(path: Path, size: int) -> None: | ||
| 49 | + """向指定文件写入 size 字节的占位数据。""" | ||
| 50 | + with open(path, "wb") as f: | ||
| 51 | + f.write(b"x" * size) | ||
| 52 | + | ||
| 53 | + | ||
| 54 | +def _make_gz(path: Path, size: int, mtime: float) -> None: | ||
| 55 | + """生成一个 gzip 压缩文件,内部展开后为 size 字节,并设置 mtime。""" | ||
| 56 | + with gzip.open(path, "wb") as f: | ||
| 57 | + f.write(b"x" * size) | ||
| 58 | + os.utime(path, (mtime, mtime)) | ||
| 59 | + | ||
| 60 | + | ||
| 61 | +def _make_handler(log_path: Path, retention_days: int, max_total_size: int) -> CleanableCompressedRotatingFileHandler: | ||
| 62 | + """正式构造 handler 实例。 | ||
| 63 | + | ||
| 64 | + CleanableCompressedRotatingFileHandler 的父类 CompressedRotatingFileHandler | ||
| 65 | + 来自真实 foundation 包(通过 sys.path.append 加载),可以正常 __init__ 构造。 | ||
| 66 | + retention_days 和 max_total_size 通过构造参数传入,cleanup_by_retention / | ||
| 67 | + cleanup_by_total_size 是公开方法,可直接调用测试。 | ||
| 68 | + """ | ||
| 69 | + handler = CleanableCompressedRotatingFileHandler( | ||
| 70 | + filename=str(log_path), | ||
| 71 | + max_bytes=1024, | ||
| 72 | + backup_count=3, | ||
| 73 | + encoding="utf-8", | ||
| 74 | + delay=True, # 延迟打开文件,避免测试清理时文件句柄占用 | ||
| 75 | + retention_days=retention_days, | ||
| 76 | + max_total_size=max_total_size, | ||
| 77 | + ) | ||
| 78 | + return handler | ||
| 79 | + | ||
| 80 | + | ||
| 81 | +class CleanableHandlerRetentionDaysTest(unittest.TestCase): | ||
| 82 | + """按天数清理逻辑测试。""" | ||
| 83 | + | ||
| 84 | + def test_deletes_archives_older_than_retention_days(self) -> None: | ||
| 85 | + """超过 retention_days 的 .gz 被删除,未超过的保留。""" | ||
| 86 | + with TemporaryDirectory() as tmp: | ||
| 87 | + log_path = Path(tmp) / "app.log" | ||
| 88 | + _write_bytes(log_path, 100) | ||
| 89 | + | ||
| 90 | + now_ts = datetime.now(tz=timezone.utc).timestamp() | ||
| 91 | + old_ts = (datetime.now(tz=timezone.utc) - timedelta(days=10)).timestamp() | ||
| 92 | + | ||
| 93 | + _make_gz(log_path.with_suffix(".log.1.gz"), 50, old_ts) | ||
| 94 | + _make_gz(log_path.with_suffix(".log.2.gz"), 50, old_ts) | ||
| 95 | + _make_gz(log_path.with_suffix(".log.3.gz"), 50, now_ts) | ||
| 96 | + | ||
| 97 | + handler = _make_handler(log_path, retention_days=7, max_total_size=0) | ||
| 98 | + handler.cleanup_by_retention() | ||
| 99 | + | ||
| 100 | + self.assertFalse(log_path.with_suffix(".log.1.gz").exists()) | ||
| 101 | + self.assertFalse(log_path.with_suffix(".log.2.gz").exists()) | ||
| 102 | + self.assertTrue(log_path.with_suffix(".log.3.gz").exists()) | ||
| 103 | + self.assertTrue(log_path.exists()) | ||
| 104 | + | ||
| 105 | + def test_retention_days_zero_cleans_all_historical_archives(self) -> None: | ||
| 106 | + """retention_days=0 直接调用方法时,cutoff=now,所有历史归档都会被删。 | ||
| 107 | + | ||
| 108 | + 注意:跳过逻辑在 doRollover 中(if retention_days > 0),不在方法内部。 | ||
| 109 | + 直接调用 _cleanup_by_retention 时,retention_days=0 等价于"清理所有早于 | ||
| 110 | + 当前时间的归档"。 | ||
| 111 | + """ | ||
| 112 | + with TemporaryDirectory() as tmp: | ||
| 113 | + log_path = Path(tmp) / "app.log" | ||
| 114 | + _write_bytes(log_path, 100) | ||
| 115 | + | ||
| 116 | + old_ts = (datetime.now(tz=timezone.utc) - timedelta(days=30)).timestamp() | ||
| 117 | + _make_gz(log_path.with_suffix(".log.1.gz"), 50, old_ts) | ||
| 118 | + | ||
| 119 | + handler = _make_handler(log_path, retention_days=0, max_total_size=0) | ||
| 120 | + handler.cleanup_by_retention() | ||
| 121 | + | ||
| 122 | + # cutoff=now,30天前的文件 mtime < now,被删除 | ||
| 123 | + self.assertFalse(log_path.with_suffix(".log.1.gz").exists()) | ||
| 124 | + | ||
| 125 | + | ||
| 126 | +class CleanableHandlerTotalSizeTest(unittest.TestCase): | ||
| 127 | + """按总空间清理逻辑测试。""" | ||
| 128 | + | ||
| 129 | + def test_deletes_oldest_archives_when_total_exceeds(self) -> None: | ||
| 130 | + """总空间超限时,从最旧 .gz 开始删除直到降到阈值以下。 | ||
| 131 | + | ||
| 132 | + 注意:f.stat().st_size 返回 gzip 文件在磁盘上的实际大小(压缩后), | ||
| 133 | + 非 gzip 内展开后的原始大小。本用例用 max_total_size=1 强制所有归档 | ||
| 134 | + 都超限,验证"从最旧开始删"的行为。 | ||
| 135 | + """ | ||
| 136 | + with TemporaryDirectory() as tmp: | ||
| 137 | + log_path = Path(tmp) / "app.log" | ||
| 138 | + _write_bytes(log_path, 100) | ||
| 139 | + | ||
| 140 | + base_ts = datetime.now(tz=timezone.utc).timestamp() | ||
| 141 | + _make_gz(log_path.with_suffix(".log.1.gz"), 100, base_ts - 300) | ||
| 142 | + _make_gz(log_path.with_suffix(".log.2.gz"), 100, base_ts - 200) | ||
| 143 | + _make_gz(log_path.with_suffix(".log.3.gz"), 100, base_ts - 100) | ||
| 144 | + | ||
| 145 | + # max_total_size=1,活跃文件 100B 已超限,所有归档都会被删到 total_size<=1 | ||
| 146 | + # 由于活跃文件不能删,所有 .gz 归档会被删尽 | ||
| 147 | + handler = _make_handler(log_path, retention_days=0, max_total_size=1) | ||
| 148 | + handler.cleanup_by_total_size() | ||
| 149 | + | ||
| 150 | + # 所有归档被删(从最旧 1.gz 开始,依次 2.gz、3.gz) | ||
| 151 | + self.assertFalse(log_path.with_suffix(".log.1.gz").exists()) | ||
| 152 | + self.assertFalse(log_path.with_suffix(".log.2.gz").exists()) | ||
| 153 | + self.assertFalse(log_path.with_suffix(".log.3.gz").exists()) | ||
| 154 | + self.assertTrue(log_path.exists()) | ||
| 155 | + | ||
| 156 | + def test_total_size_zero_cleans_all_archives(self) -> None: | ||
| 157 | + """max_total_size=0 直接调用方法时,任何非零占用都会触发清理。 | ||
| 158 | + | ||
| 159 | + 注意:跳过逻辑在 doRollover 中(if max_total_size > 0),不在方法内部。 | ||
| 160 | + 直接调用 _cleanup_by_total_size 时,max_total_size=0 等价于"清理所有归档 | ||
| 161 | + 直到 total_size <= 0",由于活跃文件不能删,所有 .gz 归档会被删尽。 | ||
| 162 | + """ | ||
| 163 | + with TemporaryDirectory() as tmp: | ||
| 164 | + log_path = Path(tmp) / "app.log" | ||
| 165 | + _write_bytes(log_path, 100) | ||
| 166 | + | ||
| 167 | + _make_gz(log_path.with_suffix(".log.1.gz"), 100, datetime.now(tz=timezone.utc).timestamp()) | ||
| 168 | + | ||
| 169 | + handler = _make_handler(log_path, retention_days=0, max_total_size=0) | ||
| 170 | + handler.cleanup_by_total_size() | ||
| 171 | + | ||
| 172 | + # max_total_size=0,total_size=活跃100+归档>0,归档被删到 total_size<=0 | ||
| 173 | + # 活跃文件不删,但归档会被删尽 | ||
| 174 | + self.assertFalse(log_path.with_suffix(".log.1.gz").exists()) | ||
| 175 | + self.assertTrue(log_path.exists()) | ||
| 176 | + | ||
| 177 | + def test_total_size_under_limit_keeps_all(self) -> None: | ||
| 178 | + """总空间未超限时,所有归档都保留。""" | ||
| 179 | + with TemporaryDirectory() as tmp: | ||
| 180 | + log_path = Path(tmp) / "app.log" | ||
| 181 | + _write_bytes(log_path, 50) | ||
| 182 | + | ||
| 183 | + _make_gz(log_path.with_suffix(".log.1.gz"), 50, datetime.now(tz=timezone.utc).timestamp()) | ||
| 184 | + | ||
| 185 | + handler = _make_handler(log_path, retention_days=0, max_total_size=1024) | ||
| 186 | + handler.cleanup_by_total_size() | ||
| 187 | + | ||
| 188 | + self.assertTrue(log_path.with_suffix(".log.1.gz").exists()) | ||
| 189 | + | ||
| 190 | + | ||
| 191 | +class CleanableHandlerExceptionTest(unittest.TestCase): | ||
| 192 | + """异常分支测试:确保 print 到 stderr,不抛出。""" | ||
| 193 | + | ||
| 194 | + def test_cleanup_by_retention_handles_oserror(self) -> None: | ||
| 195 | + """_cleanup_by_retention 中 unlink 抛 OSError 时输出 stderr,不传播。""" | ||
| 196 | + with TemporaryDirectory() as tmp: | ||
| 197 | + log_path = Path(tmp) / "app.log" | ||
| 198 | + _write_bytes(log_path, 10) | ||
| 199 | + | ||
| 200 | + old_ts = (datetime.now(tz=timezone.utc) - timedelta(days=10)).timestamp() | ||
| 201 | + gz = log_path.with_suffix(".log.1.gz") | ||
| 202 | + _make_gz(gz, 10, old_ts) | ||
| 203 | + | ||
| 204 | + handler = _make_handler(log_path, retention_days=7, max_total_size=0) | ||
| 205 | + | ||
| 206 | + with patch("sys.stderr", new_callable=io.StringIO) as fake_err: | ||
| 207 | + with patch.object(Path, "unlink", side_effect=OSError("perm denied")): | ||
| 208 | + handler.cleanup_by_retention() | ||
| 209 | + self.assertIn("按天数清理删除失败", fake_err.getvalue()) | ||
| 210 | + | ||
| 211 | + def test_cleanup_by_total_size_handles_oserror(self) -> None: | ||
| 212 | + """_cleanup_by_total_size 中 unlink 抛 OSError 时输出 stderr,不传播。""" | ||
| 213 | + with TemporaryDirectory() as tmp: | ||
| 214 | + log_path = Path(tmp) / "app.log" | ||
| 215 | + _write_bytes(log_path, 100) | ||
| 216 | + | ||
| 217 | + base_ts = datetime.now(tz=timezone.utc).timestamp() | ||
| 218 | + _make_gz(log_path.with_suffix(".log.1.gz"), 100, base_ts - 100) | ||
| 219 | + | ||
| 220 | + handler = _make_handler(log_path, retention_days=0, max_total_size=10) | ||
| 221 | + | ||
| 222 | + with patch("sys.stderr", new_callable=io.StringIO) as fake_err: | ||
| 223 | + with patch.object(Path, "unlink", side_effect=OSError("perm denied")): | ||
| 224 | + handler.cleanup_by_total_size() | ||
| 225 | + self.assertIn("按空间清理删除失败", fake_err.getvalue()) | ||
| 226 | + | ||
| 227 | + | ||
| 228 | +if __name__ == "__main__": | ||
| 229 | + unittest.main() | ||
| @@ -44,6 +44,12 @@ VA_WORKFLOW_ADAPTER_TYPE=a2a_gateway | |||
| 44 | # ── Log ─────────────────────────────────────────────────────────────────────── | 44 | # ── Log ─────────────────────────────────────────────────────────────────────── |
| 45 | ADAPTER_LOG_LEVEL=DEBUG | 45 | ADAPTER_LOG_LEVEL=DEBUG |
| 46 | ADAPTER_LOG_FILE=logs/versatile_adapter.log | 46 | ADAPTER_LOG_FILE=logs/versatile_adapter.log |
| 47 | +# 日志轮转大小(loguru格式,如 "100 MB") | ||
| 48 | +ADAPTER_LOG_ROTATION_SIZE=100 MB | ||
| 49 | +# 日志保留天数(0=不按天数清理,默认7天) | ||
| 50 | +ADAPTER_LOG_RETENTION_DAYS=7 | ||
| 51 | +# 日志总空间上限(字节,0=不按空间清理,默认500MB) | ||
| 52 | +ADAPTER_LOG_MAX_TOTAL_SIZE=524288000 | ||
| 47 | 53 | ||
| 48 | # ── Redis(会话状态)───────────────────────────────────────────────────────── | 54 | # ── Redis(会话状态)───────────────────────────────────────────────────────── |
| 49 | REDIS_HOST=localhost | 55 | REDIS_HOST=localhost |
| @@ -20,6 +20,8 @@ import os | |||
| 20 | import sys | 20 | import sys |
| 21 | import uuid | 21 | import uuid |
| 22 | from contextlib import asynccontextmanager | 22 | from contextlib import asynccontextmanager |
| 23 | +from datetime import datetime, timedelta, timezone | ||
| 24 | +from pathlib import Path | ||
| 23 | 25 | ||
| 24 | from a2a.server.request_handlers import DefaultRequestHandler | 26 | from a2a.server.request_handlers import DefaultRequestHandler |
| 25 | from a2a.server.routes import create_agent_card_routes, create_jsonrpc_routes | 27 | from a2a.server.routes import create_agent_card_routes, create_jsonrpc_routes |
| @@ -61,6 +63,141 @@ def dynamic_format(record) -> str: | |||
| 61 | "<level>{message}</level>\n" | 63 | "<level>{message}</level>\n" |
| 62 | 64 | ||
| 63 | 65 | ||
| 66 | +def _va_cleanup_logs(files): | ||
| 67 | + """ | ||
| 68 | + loguru retention 回调函数:按天数 + 按总空间清理 VA 日志。 | ||
| 69 | + | ||
| 70 | + 【函数身份】loguru 的 retention 回调,VA 每次日志轮转后由 loguru 自动调用 | ||
| 71 | + 【参数 files】loguru 传入的,当前 sink 匹配到的所有日志文件列表(Path 对象列表) | ||
| 72 | + 【返回值】无返回值,通过副作用(删除文件)实现清理 | ||
| 73 | + | ||
| 74 | + 清理逻辑: | ||
| 75 | + 1. 按保留天数清理:删除修改时间超过 retention_days 的 .gz 归档文件 | ||
| 76 | + 2. 按总空间清理:如果总空间超限,从最旧 .gz 归档开始逐一删除 | ||
| 77 | + | ||
| 78 | + 注意:活跃文件(versatile_adapter_{pid}.log,无 .gz 后缀)不参与删除, | ||
| 79 | + 只删 .gz 归档文件。 | ||
| 80 | + """ | ||
| 81 | + # ━━━ 第1步:读取配置 ━━━ | ||
| 82 | + _settings = get_settings() | ||
| 83 | + retention_days = _settings.adapter_log_retention_days | ||
| 84 | + max_total_size = _settings.adapter_log_max_total_size | ||
| 85 | + | ||
| 86 | + # ━━━ 第2步:空列表保护 ━━━ | ||
| 87 | + if not files: | ||
| 88 | + return | ||
| 89 | + # loguru 传入空列表时直接返回,防御性编程 | ||
| 90 | + | ||
| 91 | + # 统一转为 Path 对象(loguru 可能传入字符串) | ||
| 92 | + files = [Path(f) for f in files] | ||
| 93 | + | ||
| 94 | + # ━━━ 第3步:区分活跃文件和归档文件 ━━━ | ||
| 95 | + # 活跃文件:无 .gz 后缀的 .log 文件(如 versatile_adapter_12345.log,正在被 loguru 写入) | ||
| 96 | + # 归档文件:.gz 后缀的压缩文件(如 versatile_adapter_12345.log.2026-07-13_10-30-00.gz) | ||
| 97 | + # 用文件名后缀判断,不用 mtime 判断,避免误删刚压缩的归档(刚压缩的归档 mtime 可能比活跃文件还新) | ||
| 98 | + active_files = [f for f in files if f.exists() and not f.name.endswith(".gz")] | ||
| 99 | + archive_files = [f for f in files if f.exists() and f.name.endswith(".gz")] | ||
| 100 | + | ||
| 101 | + if not archive_files: | ||
| 102 | + return | ||
| 103 | + # 没有归档文件就没有可清理的东西,活跃文件永远不删,直接返回 | ||
| 104 | + | ||
| 105 | + # ━━━ 第4步:按天数清理(只清理归档文件)━━━ | ||
| 106 | + if retention_days > 0: | ||
| 107 | + # retention_days=0 表示不按天数清理,跳过此步骤 | ||
| 108 | + cutoff_time = (datetime.now(tz=timezone.utc) - timedelta(days=retention_days)).timestamp() | ||
| 109 | + # 计算截止时间点:当前时间 - retention_days 天 | ||
| 110 | + # 例如 retention_days=7,cutoff_time 就是7天前的 Unix 时间戳 | ||
| 111 | + for f in archive_files: | ||
| 112 | + # 遍历所有归档文件 | ||
| 113 | + try: | ||
| 114 | + if f.stat().st_mtime < cutoff_time: | ||
| 115 | + # f.stat().st_mtime 是文件最后修改时间(Unix 时间戳) | ||
| 116 | + # 如果文件的 mtime 早于截止时间(即文件超过 retention_days 天没修改过) | ||
| 117 | + f.unlink() | ||
| 118 | + # Path.unlink() 删除文件 | ||
| 119 | + except OSError as e: | ||
| 120 | + # OSError 可能原因:文件被占用、权限不足、文件已被其他进程删除 | ||
| 121 | + # 用 print 到 stderr,不用 loguru(避免在 retention 回调中递归调用 loguru) | ||
| 122 | + sys.stderr.write(f"[va_log_cleanup] WARN 按天数清理删除失败: {f} -> {e}\n") | ||
| 123 | + | ||
| 124 | + # ━━━ 第5步:按总空间清理(活跃文件 + 剩余归档文件的总和)━━━ | ||
| 125 | + if max_total_size > 0: | ||
| 126 | + # max_total_size=0 表示不按空间清理,跳过此步骤 | ||
| 127 | + | ||
| 128 | + remaining_archives = [(f.stat().st_mtime, f) for f in archive_files if f.exists()] | ||
| 129 | + # 重新获取未被第4步删掉的归档文件,返回 [(mtime, Path), ...] 列表 | ||
| 130 | + # f.exists() 过滤掉已删除的文件 | ||
| 131 | + | ||
| 132 | + active_size = sum(f.stat().st_size for f in active_files if f.exists()) | ||
| 133 | + # 计算所有活跃文件的总大小(字节数) | ||
| 134 | + # 活跃文件计入总空间,但不删除 | ||
| 135 | + | ||
| 136 | + archive_total = sum(f.stat().st_size for _, f in remaining_archives) | ||
| 137 | + # 计算所有剩余归档文件的总大小 | ||
| 138 | + | ||
| 139 | + total_size = active_size + archive_total | ||
| 140 | + # 总空间 = 活跃文件大小 + 归档文件大小 | ||
| 141 | + | ||
| 142 | + if total_size > max_total_size: | ||
| 143 | + # 如果总空间超过阈值,需要清理 | ||
| 144 | + remaining_archives.sort(key=lambda x: x[0]) | ||
| 145 | + # 按 mtime 排序(最旧的在前),优先删最旧的归档 | ||
| 146 | + | ||
| 147 | + for _mtime, f in remaining_archives: | ||
| 148 | + # 从最旧开始遍历 | ||
| 149 | + if total_size <= max_total_size: | ||
| 150 | + break | ||
| 151 | + # 每删一个文件后检查,如果总空间已降到阈值以下,停止删除 | ||
| 152 | + | ||
| 153 | + try: | ||
| 154 | + file_size = f.stat().st_size | ||
| 155 | + # 先获取文件大小(删除后就获取不到了) | ||
| 156 | + f.unlink() | ||
| 157 | + # 删除文件 | ||
| 158 | + total_size -= file_size | ||
| 159 | + # 从总空间中减去已删除文件的大小 | ||
| 160 | + except OSError as e: | ||
| 161 | + sys.stderr.write(f"[va_log_cleanup] WARN 按空间清理删除失败: {f} -> {e}\n") | ||
| 162 | + | ||
| 163 | + | ||
| 164 | +def _cleanup_stale_pid_logs(log_dir: str, current_pid: int, retention_days: int) -> None: | ||
| 165 | + """ | ||
| 166 | + 清理旧 PID 的僵尸日志文件。 | ||
| 167 | + | ||
| 168 | + 【背景】VA 日志文件名带 PID:versatile_adapter_{pid}.log | ||
| 169 | + 每次 VA 重启都会用新的 PID,产生新的日志文件。旧 PID 的日志文件不会被 loguru | ||
| 170 | + 的 retention 回调清理(因为 loguru 只管当前 sink 绑定的文件),会无限累积。 | ||
| 171 | + | ||
| 172 | + 【调用时机】VA 启动时 setup_logging() 中调用一次,在 logger.add 之前执行 | ||
| 173 | + | ||
| 174 | + 【清理规则】与需求文档一致,统一按 mtime 清理(不区分 .gz/.log) | ||
| 175 | + 1. 扫描日志目录下所有匹配 versatile_adapter_*.log* 的文件 | ||
| 176 | + 2. 排除当前 PID 的文件(当前 PID 的文件正在使用,不能删) | ||
| 177 | + 3. 统一按 mtime 检查:超过 retention_days 天没修改过则删除(.gz 和 .log 同标准) | ||
| 178 | + """ | ||
| 179 | + if retention_days <= 0: | ||
| 180 | + return | ||
| 181 | + | ||
| 182 | + log_path = Path(log_dir) | ||
| 183 | + if not log_path.exists(): | ||
| 184 | + return | ||
| 185 | + | ||
| 186 | + cutoff_time = (datetime.now(tz=timezone.utc) - timedelta(days=retention_days)).timestamp() | ||
| 187 | + | ||
| 188 | + for f in log_path.glob("versatile_adapter_*.log*"): | ||
| 189 | + if not f.is_file(): | ||
| 190 | + continue | ||
| 191 | + # 不删除当前进程的日志文件 | ||
| 192 | + if f"_{current_pid}." in f.name or f.name.endswith(f"_{current_pid}.log"): | ||
| 193 | + continue | ||
| 194 | + try: | ||
| 195 | + if f.stat().st_mtime < cutoff_time: | ||
| 196 | + f.unlink() | ||
| 197 | + except OSError as e: | ||
| 198 | + sys.stderr.write(f"[va_log_cleanup] WARN 僵尸文件清理失败: {f} -> {e}\n") | ||
| 199 | + | ||
| 200 | + | ||
| 64 | def setup_logging() -> None: | 201 | def setup_logging() -> None: |
| 65 | """配置日志""" | 202 | """配置日志""" |
| 66 | settings = get_settings() | 203 | settings = get_settings() |
| @@ -78,14 +215,22 @@ def setup_logging() -> None: | |||
| 78 | log_dir = os.path.dirname(settings.adapter_log_file) | 215 | log_dir = os.path.dirname(settings.adapter_log_file) |
| 79 | if log_dir: | 216 | if log_dir: |
| 80 | os.makedirs(log_dir, exist_ok=True) | 217 | os.makedirs(log_dir, exist_ok=True) |
| 218 | + | ||
| 219 | + # 启动时清理旧 pid 残留日志(在 logger.add 之前执行,与需求文档一致) | ||
| 220 | + _cleanup_stale_pid_logs( | ||
| 221 | + log_dir, | ||
| 222 | + os.getpid(), | ||
| 223 | + settings.adapter_log_retention_days | ||
| 224 | + ) | ||
| 225 | + | ||
| 81 | log_file_path = settings.adapter_log_file | 226 | log_file_path = settings.adapter_log_file |
| 82 | base, ext = os.path.splitext(log_file_path) | 227 | base, ext = os.path.splitext(log_file_path) |
| 83 | log_file_with_pid = f"{base}_{os.getpid()}{ext}" | 228 | log_file_with_pid = f"{base}_{os.getpid()}{ext}" |
| 84 | logger.add( | 229 | logger.add( |
| 85 | log_file_with_pid, | 230 | log_file_with_pid, |
| 86 | level=settings.adapter_log_level.upper() if settings.adapter_log_level else "INFO", | 231 | level=settings.adapter_log_level.upper() if settings.adapter_log_level else "INFO", |
| 87 | - rotation="100 MB", | 232 | + rotation=settings.adapter_log_rotation_size, |
| 88 | - retention="7 days", | 233 | + retention=_va_cleanup_logs, |
| 89 | compression="gz", | 234 | compression="gz", |
| 90 | format=dynamic_format, | 235 | format=dynamic_format, |
| 91 | filter=lambda record: len(record["extra"]) == 0 or "trace_id" in record["extra"] | 236 | filter=lambda record: len(record["extra"]) == 0 or "trace_id" in record["extra"] |
| @@ -62,6 +62,12 @@ class Settings(BaseSettings): | |||
| 62 | # ── 日志 ──────────────────────────────────────────────────────────────── | 62 | # ── 日志 ──────────────────────────────────────────────────────────────── |
| 63 | adapter_log_level: Optional[str] = None | 63 | adapter_log_level: Optional[str] = None |
| 64 | adapter_log_file: Optional[str] = None | 64 | adapter_log_file: Optional[str] = None |
| 65 | + # 日志轮转大小(loguru格式,如 "100 MB") | ||
| 66 | + adapter_log_rotation_size: str = "100 MB" | ||
| 67 | + # 日志保留天数(0=不按天数清理) | ||
| 68 | + adapter_log_retention_days: int = 7 | ||
| 69 | + # 日志总空间上限(字节,0=不按空间清理,默认500MB) | ||
| 70 | + adapter_log_max_total_size: int = 524288000 | ||
| 65 | 71 | ||
| 66 | # ── Redis ────────────────────────────────────────────────────────────── | 72 | # ── Redis ────────────────────────────────────────────────────────────── |
| 67 | redis_host: Optional[str] = None | 73 | redis_host: Optional[str] = None |


【openlibing.ci】检测到当前PR中存在代码检查告警抑制 2 处,详情见下表,请Committer检视合理性。 / Detected 2 code check alert suppression(s) in this PR, see table below. Committers please review.
common/sdk_log_cleaner.py
def doRollover(self) -> None: # noqa: N802 继承自标准库 RotatingFileHandler,必须保持原名tests/common/test_sdk_log_cleaner.py
from common.sdk_log_cleaner import CleanableCompressedRotatingFileHandler # noqa: E402