Merged
feat(observability): Add observability for intelli router #215
yutaoJustinBieber1223Add on Jun 2
feat(observability): Add observability for intelli router #215
Merged
Total: 51 files changed+9116-403
| @@ -6,3 +6,12 @@ dist/ | |||
| 6 | build/ | 6 | build/ |
| 7 | .venv/ | 7 | .venv/ |
| 8 | *.egg | 8 | *.egg |
| 9 | +.remember/ | ||
| 10 | + | ||
| 11 | +# Docs (local only) | ||
| 12 | +intelli_router/docs/ | ||
| 13 | + | ||
| 14 | +# Nested gitignore | ||
| 15 | +intelli_router/.gitignore | ||
| 16 | + | ||
| 17 | +CLAUDE.md | ||
| @@ -7,9 +7,9 @@ | |||
| 7 | 核心特性: | 7 | 核心特性: |
| 8 | - **API池化**: 单一模型名 → 多部署端点映射 | 8 | - **API池化**: 单一模型名 → 多部署端点映射 |
| 9 | - **策略插拔**: 支持SimpleShuffle、LowestLatency、TagBased、TokenAware、RateLimitAware、Adaptive等路由策略 | 9 | - **策略插拔**: 支持SimpleShuffle、LowestLatency、TagBased、TokenAware、RateLimitAware、Adaptive等路由策略 |
| 10 | -- **状态管理**: 部署状态追踪、Token/RPM实一监控,失败计数,冷却机制 | 10 | +- **状态管理**: 部署状态追踪、Token/RPM实时监控,失败计数,冷却机制 |
| 11 | -- **健庭检查**: 后台健庭检查循环 | 11 | +- **健康检查**: 后台健康检查循环 |
| 12 | -- **重试机制**: 自重试机制 | 12 | +- **重试机制**: 自动重试机制 |
| 13 | 13 | ||
| 14 | ## 代码结构设计 | 14 | ## 代码结构设计 |
| 15 | 15 | ||
| @@ -18,7 +18,7 @@ intelli_router/ | |||
| 18 | ├── __init__.py 公共接口 | 18 | ├── __init__.py 公共接口 |
| 19 | ├── core/ | 19 | ├── core/ |
| 20 | │ ├── __init__.py | 20 | │ ├── __init__.py |
| 21 | -│ ├── deployment.py 部署配罫 | 21 | +│ ├── deployment.py 部署配置 |
| 22 | │ ├── state.py 状态管理 | 22 | │ ├── state.py 状态管理 |
| 23 | │ ├── context.py 路由上下文 | 23 | │ ├── context.py 路由上下文 |
| 24 | │ └── deployment.py 部署状态枚举 | 24 | │ └── deployment.py 部署状态枚举 |
| @@ -26,33 +26,33 @@ intelli_router/ | |||
| 26 | │ ├── __init__.py 策略工厂 | 26 | │ ├── __init__.py 策略工厂 |
| 27 | │ ├── base_strategy.py 抽象策略基类 | 27 | │ ├── base_strategy.py 抽象策略基类 |
| 28 | │ ├── simple_shuffle.py 随机选择策略 | 28 | │ ├── simple_shuffle.py 随机选择策略 |
| 29 | -│ ├── lowest_latency.py 最迪延迟策略 | 29 | +│ ├── lowest_latency.py 最低延迟策略 |
| 30 | │ ├── tag_based.py 标签策略 | 30 | │ ├── tag_based.py 标签策略 |
| 31 | │ ├── token_aware.py Token感知策略 | 31 | │ ├── token_aware.py Token感知策略 |
| 32 | │ ├── rate_limit_aware.py RPM限流策略 | 32 | │ ├── rate_limit_aware.py RPM限流策略 |
| 33 | │ └──── adaptive.py 自适应策略 | 33 | │ └──── adaptive.py 自适应策略 |
| 34 | ├── router/ 路由器 | 34 | ├── router/ 路由器 |
| 35 | │ ├── __init__.py | 35 | │ ├── __init__.py |
| 36 | -│ ├── base_router.py 基础路一器 | 36 | +│ ├── base_router.py 基础路由器 |
| 37 | -│ ├── reliable_router.py 可靠性路一器 | 37 | +│ ├── reliable_router.py 可靠性路由器 |
| 38 | ├── cache/ 缓存 | 38 | ├── cache/ 缓存 |
| 39 | │ ├── __init__.py | 39 | │ ├── __init__.py |
| 40 | │ └──── local_cache.py 本地缓存 (LRU) | 40 | │ └──── local_cache.py 本地缓存 (LRU) |
| 41 | -├── health/ 健庭检查 | 41 | +├── health/ 健康检查 |
| 42 | │ ├── __init__.py | 42 | │ ├── __init__.py |
| 43 | -│ └──── checker.py SDK健庭检查 | 43 | +│ └──── checker.py SDK健康检查 |
| 44 | ├── utils/ 工具函数 | 44 | ├── utils/ 工具函数 |
| 45 | │ └──── exceptions.py 异常层次 | 45 | │ └──── exceptions.py 异常层次 |
| 46 | ├── tests/ 测试 | 46 | ├── tests/ 测试 |
| 47 | └── sdk_proxy_demo/ | 47 | └── sdk_proxy_demo/ |
| 48 | ├── __init__.py | 48 | ├── __init__.py |
| 49 | - ├── conftest.py pyest配置 | 49 | + ├── conftest.py pytest配置 |
| 50 | - ├── test_api_pooling.py API池化测 | 50 | + ├── test_api_pooling.py API池化测试 |
| 51 | - ├── test_routing_strategies.py 路由策略测 | 51 | + ├── test_routing_strategies.py 路由策略测试 |
| 52 | - ├── test_state_management.py 状态管理测 | 52 | + ├── test_state_management.py 状态管理测试 |
| 53 | - ├── test_caching.py 缓存测 | 53 | + ├── test_caching.py 缓存测试 |
| 54 | - ├── test_concurrent_requests.py 并发请测 | 54 | + ├── test_concurrent_requests.py 并发请求测试 |
| 55 | - ├── test_adaptive_strategy.py 自适应策略测 | 55 | + ├── test_adaptive_strategy.py 自适应策略测试 |
| 56 | └── utils/ | 56 | └── utils/ |
| 57 | ├── __init__.py | 57 | ├── __init__.py |
| 58 | ├── helpers.py 测试辅助函数 | 58 | ├── helpers.py 测试辅助函数 |
| @@ -73,7 +73,7 @@ intelli_router/ | |||
| 73 | **策略**: | 73 | **策略**: |
| 74 | 1. 过滤可用部署 | 74 | 1. 过滤可用部署 |
| 75 | 2. 按Token剩余量排序 | 75 | 2. 按Token剩余量排序 |
| 76 | -3. 优先择Token充足的部署 | 76 | +3. 优先选择Token充足的部署 |
| 77 | 4. Token耗尽时无缝切换到下一个 | 77 | 4. Token耗尽时无缝切换到下一个 |
| 78 | 78 | ||
| 79 | **关键参数**: | 79 | **关键参数**: |
| @@ -87,7 +87,7 @@ intelli_router/ | |||
| 87 | **策略**: | 87 | **策略**: |
| 88 | 1. 过滤可用部署 | 88 | 1. 过滤可用部署 |
| 89 | 2. 按RPM剩余配额排序 | 89 | 2. 按RPM剩余配额排序 |
| 90 | -3. 优先择RPM配额充足的部署 | 90 | +3. 优先选择RPM配额充足的部署 |
| 91 | 4. RPM耗尽时无缝切换 | 91 | 4. RPM耗尽时无缝切换 |
| 92 | 92 | ||
| 93 | **关键参数**: | 93 | **关键参数**: |
| @@ -99,7 +99,7 @@ intelli_router/ | |||
| 99 | **场景**: 多级决策树 | 99 | **场景**: 多级决策树 |
| 100 | 100 | ||
| 101 | **策略**: | 101 | **策略**: |
| 102 | -1. 健庭检查 → 咒除不健康部署 | 102 | +1. 健康检查 → 剔除不健康部署 |
| 103 | 2. Token剩余 → 优先Token充足的 | 103 | 2. Token剩余 → 优先Token充足的 |
| 104 | 3. RPM剩余 → 优先RPM配额充足的 | 104 | 3. RPM剩余 → 优先RPM配额充足的 |
| 105 | 4. 延迟收益 → 优先低延迟的 | 105 | 4. 延迟收益 → 优先低延迟的 |
| @@ -114,7 +114,7 @@ score = w_health * health_score | |||
| 114 | ``` | 114 | ``` |
| 115 | 115 | ||
| 116 | **关键参数**: | 116 | **关键参数**: |
| 117 | -- `w_health`: 健庭权重 (默认1.0) | 117 | +- `w_health`: 健康权重 (默认1.0) |
| 118 | - `w_token`: Token权重 (默认0.5) | 118 | - `w_token`: Token权重 (默认0.5) |
| 119 | - `w_rpm`: RPM权重 (默认0.3) | 119 | - `w_rpm`: RPM权重 (默认0.3) |
| 120 | - `w_latency`: 延迟权重 (默认0.2) | 120 | - `w_latency`: 延迟权重 (默认0.2) |
| @@ -198,12 +198,128 @@ python3 -m pytest intelli_router/tests/sdk_proxy_demo/test_adaptive_strategy.py | |||
| 198 | ## 项目结构 | 198 | ## 项目结构 |
| 199 | 199 | ||
| 200 | 详见 `sdk_llm` 目录树: | 200 | 详见 `sdk_llm` 目录树: |
| 201 | - | 201 | + |
| 202 | - `core/`: 核心和状态管理 | 202 | - `core/`: 核心和状态管理 |
| 203 | - `strategy/`: 路由策略实现 | 203 | - `strategy/`: 路由策略实现 |
| 204 | - `router/`: 路由器实现 | 204 | - `router/`: 路由器实现 |
| 205 | - `cache/`: 缓存实现 | 205 | - `cache/`: 缓存实现 |
| 206 | -- `health/`: 健庭检查实现 | 206 | +- `health/`: 健康检查实现 |
| 207 | - `tests/`: 测试用例 | 207 | - `tests/`: 测试用例 |
| 208 | - `demo_*`: 演示脚本 | 208 | - `demo_*`: 演示脚本 |
| 209 | 209 | ||
| 210 | +## 快速上手 | ||
| 211 | + | ||
| 212 | +```python | ||
| 213 | +import asyncio | ||
| 214 | +from intelli_router import ReliableRouter, Deployment | ||
| 215 | + | ||
| 216 | +# 1. 定义部署端点 | ||
| 217 | +dep = Deployment( | ||
| 218 | + id="openai-1", | ||
| 219 | + model_name="gpt-4o-mini", | ||
| 220 | + api_key="sk-...", | ||
| 221 | + api_base="https://api.openai.com", | ||
| 222 | + provider="openai", | ||
| 223 | +) | ||
| 224 | + | ||
| 225 | +# 2. 创建路由器 | ||
| 226 | +router = ReliableRouter(deployments=[dep], strategy="adaptive") | ||
| 227 | + | ||
| 228 | +# 3. 调用 | ||
| 229 | +async def main(): | ||
| 230 | + resp = await router.completion( | ||
| 231 | + "gpt-4o-mini", | ||
| 232 | + [{"role": "user", "content": "你好"}], | ||
| 233 | + ) | ||
| 234 | + print(resp["choices"][0]["message"]["content"]) | ||
| 235 | + await router.close() | ||
| 236 | + | ||
| 237 | +asyncio.run(main()) | ||
| 238 | +``` | ||
| 239 | + | ||
| 240 | +Streaming 用法: | ||
| 241 | + | ||
| 242 | +```python | ||
| 243 | +async for chunk in router.stream("gpt-4o-mini", messages): | ||
| 244 | + print(chunk["choices"][0]["delta"].get("content", ""), end="") | ||
| 245 | +``` | ||
| 246 | + | ||
| 247 | +多 Provider failover: | ||
| 248 | + | ||
| 249 | +```python | ||
| 250 | +deps = [ | ||
| 251 | + Deployment(id="main", model_name="gpt-4o-mini", api_key="sk-...", | ||
| 252 | + api_base="https://api.openai.com", provider="openai"), | ||
| 253 | + Deployment(id="backup", model_name="gpt-4o-mini", api_key="sk-...", | ||
| 254 | + api_base="https://api.openai.com", provider="openai"), | ||
| 255 | +] | ||
| 256 | +router = ReliableRouter(deployments=deps, num_retries=1, strategy="adaptive") | ||
| 257 | +# 主端点失败时自动切换到备用端点 | ||
| 258 | +resp = await router.completion("gpt-4o-mini", messages) | ||
| 259 | +``` | ||
| 260 | + | ||
| 261 | +## 多 Provider 适配 | ||
| 262 | + | ||
| 263 | +IntelliRouter 以 OpenAI 格式为内部标准格式,通过 Provider Adapter 自动完成与各厂商 API 的协议转换。调用者只需在 `Deployment` 中指定 `provider` 字段,无需关心底层协议差异。 | ||
| 264 | + | ||
| 265 | +支持的 Provider: | ||
| 266 | + | ||
| 267 | +| Provider | `provider` 标识 | 说明 | | ||
| 268 | +|----------|----------------|------| | ||
| 269 | +| OpenAI | `openai` | 基准实现,直接透传 | | ||
| 270 | +| Anthropic | `anthropic` | 完整协议转换(Messages API) | | ||
| 271 | +| Google Gemini | `google-gemini` | 完整协议转换(generateContent) | | ||
| 272 | +| DeepSeek | `deepseek` | OpenAI 兼容 + reasoning_content 字段注入 | | ||
| 273 | +| DashScope (阿里云) | `dashscope` | OpenAI 兼容,端点路径不同 | | ||
| 274 | +| SiliconFlow (硅基流动) | `siliconflow` | OpenAI 兼容 + tool_calls 字段清洗 | | ||
| 275 | +| Zhipu (智谱) | `zhipu` | OpenAI 兼容,端点路径不同 | | ||
| 276 | +| InferenceAffinity (vLLM) | `inference-affinity` | OpenAI 兼容 + KV cache 共享 | | ||
| 277 | +| AWS Bedrock | `aws-bedrock` | 待实现(需 SigV4 签名) | | ||
| 278 | + | ||
| 279 | +各 Provider 协议差异详见 `docs/PROVIDER_API_DIFFERENCES.md`。 | ||
| 280 | + | ||
| 281 | +## 输出解析器 (Parser) | ||
| 282 | + | ||
| 283 | +内置两种输出解析器,用于将 LLM 响应解析为结构化数据: | ||
| 284 | + | ||
| 285 | +- **JsonOutputParser**:从 LLM 回复中提取 JSON,支持裸 JSON 和 ` ```json ` 代码块包裹格式,支持流式增量解析 | ||
| 286 | +- **MarkdownOutputParser**:将 Markdown 文本解析为结构化 `MarkdownContent`,包含 headers、code_blocks、links、images、tables、lists 等元素 | ||
| 287 | + | ||
| 288 | +```python | ||
| 289 | +from intelli_router import JsonOutputParser, MarkdownOutputParser | ||
| 290 | + | ||
| 291 | +# JSON 解析 | ||
| 292 | +parser = JsonOutputParser() | ||
| 293 | +result = parser.parse(assistant_message) # -> dict/list | ||
| 294 | + | ||
| 295 | +# Markdown 解析 | ||
| 296 | +md_parser = MarkdownOutputParser() | ||
| 297 | +content = md_parser.parse(assistant_message) # -> MarkdownContent | ||
| 298 | +print(content.headers) # 所有标题 | ||
| 299 | +print(content.code_blocks) # 所有代码块 | ||
| 300 | +``` | ||
| 301 | + | ||
| 302 | +## 类型定义 (Types) | ||
| 303 | + | ||
| 304 | +`intelli_router.types` 提供统一的类型化响应对象: | ||
| 305 | + | ||
| 306 | +- **`ToolCall`** — 工具调用信息(id、name、arguments) | ||
| 307 | +- **`UsageMetadata`** — Token 用量统计(input_tokens、output_tokens、total_tokens) | ||
| 308 | +- **`AssistantMessage`** — 完整的助手响应(content、tool_calls、usage) | ||
| 309 | +- **`AssistantMessageChunk`** — 流式响应片段,支持 `+` 运算符累加合并 | ||
| 310 | + | ||
| 311 | +```python | ||
| 312 | +from intelli_router import AssistantMessage, AssistantMessageChunk | ||
| 313 | + | ||
| 314 | +# chunk 累加 | ||
| 315 | +full = AssistantMessageChunk() | ||
| 316 | +async for chunk in router.stream(...): | ||
| 317 | + full = full + chunk # 自动合并 content、tool_calls、usage | ||
| 318 | +``` | ||
| 319 | + | ||
| 320 | +## 更多文档 | ||
| 321 | + | ||
| 322 | +- [API.md](API.md) — 完整 API 参考 | ||
| 323 | +- [examples/demo_e2e.py](examples/demo_e2e.py) — 端到端演示(OpenAI / Anthropic / Gemini / Failover) | ||
| 324 | +- [docs/](docs/) — 设计文档(Provider 差异对比、可观测性设计等) | ||
| 325 | + | ||
| @@ -0,0 +1,236 @@ | |||
| 1 | +""" | ||
| 2 | +intelli_router 端到端 Demo | ||
| 3 | + | ||
| 4 | +演示场景: | ||
| 5 | + 1. OpenAI 真实调用 | ||
| 6 | + 2. Anthropic 真实调用 | ||
| 7 | + 3. Gemini 真实调用 | ||
| 8 | + 4. Failover — dead endpoint 自动切换到 healthy endpoint | ||
| 9 | + 5. All dead — 所有 endpoint 不可用,抛出 RouterError | ||
| 10 | + | ||
| 11 | +使用方式: | ||
| 12 | + export OPENAI_API_KEY="sk-..." | ||
| 13 | + export ANTHROPIC_API_KEY="sk-ant-..." | ||
| 14 | + export GEMINI_API_KEY="AIza..." | ||
| 15 | + | ||
| 16 | + python examples/demo_e2e.py | ||
| 17 | + | ||
| 18 | +没有配置 key 的场景会自动 skip。 | ||
| 19 | +""" | ||
| 20 | + | ||
| 21 | +import os | ||
| 22 | +import asyncio | ||
| 23 | +import time | ||
| 24 | + | ||
| 25 | +from intelli_router.core.deployment import Deployment | ||
| 26 | +from intelli_router.router.reliable_router import ReliableRouter | ||
| 27 | +from intelli_router.utils.exceptions import RouterError | ||
| 28 | + | ||
| 29 | +# --------------------------------------------------------------------------- | ||
| 30 | +# 配置 | ||
| 31 | +# --------------------------------------------------------------------------- | ||
| 32 | + | ||
| 33 | +MESSAGE = [{"role": "user", "content": "回复三个中文字符:你好世界"}] | ||
| 34 | + | ||
| 35 | +OPENAI_KEY = os.environ.get("OPENAI_API_KEY", "") | ||
| 36 | +ANTHROPIC_KEY = os.environ.get("ANTHROPIC_API_KEY", "") | ||
| 37 | +GEMINI_KEY = os.environ.get("GEMINI_API_KEY", "") | ||
| 38 | + | ||
| 39 | +DEAD_BASE = "http://localhost:19999" # 没有服务监听的端口 | ||
| 40 | + | ||
| 41 | +MODELS = { | ||
| 42 | + "openai": "gpt-4o-mini", | ||
| 43 | + "anthropic": "claude-3-haiku-20240307", | ||
| 44 | + "gemini": "gemini-1.5-flash", | ||
| 45 | +} | ||
| 46 | + | ||
| 47 | + | ||
| 48 | +# --------------------------------------------------------------------------- | ||
| 49 | +# 场景 | ||
| 50 | +# --------------------------------------------------------------------------- | ||
| 51 | + | ||
| 52 | + | ||
| 53 | +async def scene_openai(): | ||
| 54 | + """场景 1:OpenAI 真实调用""" | ||
| 55 | + print("\n=== scene: OpenAI real ===") | ||
| 56 | + dep = Deployment( | ||
| 57 | + id="openai-1", | ||
| 58 | + model_name=MODELS["openai"], | ||
| 59 | + api_key=OPENAI_KEY, | ||
| 60 | + api_base="https://api.openai.com", | ||
| 61 | + provider="openai", | ||
| 62 | + ) | ||
| 63 | + router = ReliableRouter( | ||
| 64 | + deployments=[dep], | ||
| 65 | + num_retries=0, | ||
| 66 | + strategy="simple-shuffle", | ||
| 67 | + ) | ||
| 68 | + t0 = time.time() | ||
| 69 | + resp = await router.completion(MODELS["openai"], MESSAGE) | ||
| 70 | + cost = time.time() - t0 | ||
| 71 | + print(f" response: {resp['choices'][0]['message']['content']}") | ||
| 72 | + print(f" latency: {cost:.2f}s") | ||
| 73 | + print(f" usage: {resp['usage']}") | ||
| 74 | + await router.close() | ||
| 75 | + | ||
| 76 | + | ||
| 77 | +async def scene_anthropic(): | ||
| 78 | + """场景 2:Anthropic 真实调用""" | ||
| 79 | + print("\n=== scene: Anthropic real ===") | ||
| 80 | + dep = Deployment( | ||
| 81 | + id="anthropic-1", | ||
| 82 | + model_name=MODELS["anthropic"], | ||
| 83 | + api_key=ANTHROPIC_KEY, | ||
| 84 | + api_base="https://api.anthropic.com", | ||
| 85 | + provider="anthropic", | ||
| 86 | + ) | ||
| 87 | + router = ReliableRouter( | ||
| 88 | + deployments=[dep], | ||
| 89 | + num_retries=0, | ||
| 90 | + strategy="simple-shuffle", | ||
| 91 | + ) | ||
| 92 | + t0 = time.time() | ||
| 93 | + resp = await router.completion(MODELS["anthropic"], MESSAGE) | ||
| 94 | + cost = time.time() - t0 | ||
| 95 | + print(f" response: {resp['choices'][0]['message']['content']}") | ||
| 96 | + print(f" latency: {cost:.2f}s") | ||
| 97 | + print(f" usage: {resp['usage']}") | ||
| 98 | + await router.close() | ||
| 99 | + | ||
| 100 | + | ||
| 101 | +async def scene_gemini(): | ||
| 102 | + """场景 3:Gemini 真实调用""" | ||
| 103 | + print("\n=== scene: Gemini real ===") | ||
| 104 | + dep = Deployment( | ||
| 105 | + id="gemini-1", | ||
| 106 | + model_name=MODELS["gemini"], | ||
| 107 | + api_key=GEMINI_KEY, | ||
| 108 | + api_base="https://generativelanguage.googleapis.com", | ||
| 109 | + provider="google-gemini", | ||
| 110 | + ) | ||
| 111 | + router = ReliableRouter( | ||
| 112 | + deployments=[dep], | ||
| 113 | + num_retries=0, | ||
| 114 | + strategy="simple-shuffle", | ||
| 115 | + ) | ||
| 116 | + t0 = time.time() | ||
| 117 | + resp = await router.completion(MODELS["gemini"], MESSAGE) | ||
| 118 | + cost = time.time() - t0 | ||
| 119 | + print(f" response: {resp['choices'][0]['message']['content']}") | ||
| 120 | + print(f" latency: {cost:.2f}s") | ||
| 121 | + print(f" usage: {resp['usage']}") | ||
| 122 | + await router.close() | ||
| 123 | + | ||
| 124 | + | ||
| 125 | +async def scene_failover(): | ||
| 126 | + """场景 4:Failover — dead endpoint 自动切换到 healthy | ||
| 127 | + | ||
| 128 | + 两个 deployment 指向同一模型 "gpt-4o-mini": | ||
| 129 | + - openai-dead: api_base 指向不存在的 localhost:19999 | ||
| 130 | + - openai-real: 指向真实 OpenAI | ||
| 131 | + | ||
| 132 | + ReliableRouter 首次选到 dead 后 AdaptiveStrategy 标记 cooldown, | ||
| 133 | + 重试选到 real,验证 fallback 成功。 | ||
| 134 | + """ | ||
| 135 | + print("\n=== scene: Failover ===") | ||
| 136 | + dead = Deployment( | ||
| 137 | + id="openai-dead", | ||
| 138 | + model_name=MODELS["openai"], | ||
| 139 | + api_key="fake-key", | ||
| 140 | + api_base=DEAD_BASE, | ||
| 141 | + provider="openai", | ||
| 142 | + ) | ||
| 143 | + real = Deployment( | ||
| 144 | + id="openai-real", | ||
| 145 | + model_name=MODELS["openai"], | ||
| 146 | + api_key=OPENAI_KEY, | ||
| 147 | + api_base="https://api.openai.com", | ||
| 148 | + provider="openai", | ||
| 149 | + ) | ||
| 150 | + router = ReliableRouter( | ||
| 151 | + deployments=[dead, real], | ||
| 152 | + num_retries=1, | ||
| 153 | + strategy="adaptive", # AdaptiveStrategy 在 on_failure 时标记 cooldown | ||
| 154 | + ) | ||
| 155 | + t0 = time.time() | ||
| 156 | + resp = await router.completion(MODELS["openai"], MESSAGE) | ||
| 157 | + cost = time.time() - t0 | ||
| 158 | + | ||
| 159 | + print(f" primary deployment (dead): {dead.id} → {DEAD_BASE}") | ||
| 160 | + print(f" fallback deployment (healthy): {real.id} → https://api.openai.com") | ||
| 161 | + print(f" response: {resp['choices'][0]['message']['content']}") | ||
| 162 | + print(f" latency: {cost:.2f}s") | ||
| 163 | + | ||
| 164 | + stats = router.get_stats() | ||
| 165 | + print(f" after call status:") | ||
| 166 | + for dep_id, status in stats["deployment_status"].items(): | ||
| 167 | + print(f" {dep_id}: {status}") | ||
| 168 | + | ||
| 169 | + await router.close() | ||
| 170 | + | ||
| 171 | + | ||
| 172 | +async def scene_all_dead(): | ||
| 173 | + """场景 5:All dead — 所有 endpoint 不可用 | ||
| 174 | + | ||
| 175 | + 两个 deployment 都指向 localhost:19999,预期抛出 RouterError。 | ||
| 176 | + """ | ||
| 177 | + print("\n=== scene: All dead ===") | ||
| 178 | + dead1 = Deployment( | ||
| 179 | + id="dead-1", | ||
| 180 | + model_name=MODELS["openai"], | ||
| 181 | + api_key="fake-key", | ||
| 182 | + api_base=DEAD_BASE, | ||
| 183 | + provider="openai", | ||
| 184 | + ) | ||
| 185 | + dead2 = Deployment( | ||
| 186 | + id="dead-2", | ||
| 187 | + model_name=MODELS["openai"], | ||
| 188 | + api_key="fake-key", | ||
| 189 | + api_base=DEAD_BASE, | ||
| 190 | + provider="openai", | ||
| 191 | + ) | ||
| 192 | + router = ReliableRouter( | ||
| 193 | + deployments=[dead1, dead2], | ||
| 194 | + num_retries=1, | ||
| 195 | + strategy="adaptive", | ||
| 196 | + ) | ||
| 197 | + try: | ||
| 198 | + await router.completion(MODELS["openai"], MESSAGE) | ||
| 199 | + print(" ERROR: expected RouterError but got success") | ||
| 200 | + except RouterError as e: | ||
| 201 | + print(f" caught expected RouterError ✓") | ||
| 202 | + print(f" error: {e}") | ||
| 203 | + await router.close() | ||
| 204 | + | ||
| 205 | + | ||
| 206 | +# --------------------------------------------------------------------------- | ||
| 207 | +# main | ||
| 208 | +# --------------------------------------------------------------------------- | ||
| 209 | + | ||
| 210 | + | ||
| 211 | +async def main(): | ||
| 212 | + print("intelli_router 端到端 Demo") | ||
| 213 | + print("=" * 50) | ||
| 214 | + | ||
| 215 | + if OPENAI_KEY: | ||
| 216 | + await scene_openai() | ||
| 217 | + await scene_failover() | ||
| 218 | + await scene_all_dead() | ||
| 219 | + else: | ||
| 220 | + print("\n[skip] OPENAI_API_KEY 未设置,跳过 OpenAI / failover / all-dead 场景") | ||
| 221 | + | ||
| 222 | + if ANTHROPIC_KEY: | ||
| 223 | + await scene_anthropic() | ||
| 224 | + else: | ||
| 225 | + print("\n[skip] ANTHROPIC_API_KEY 未设置,跳过 Anthropic 场景") | ||
| 226 | + | ||
| 227 | + if GEMINI_KEY: | ||
| 228 | + await scene_gemini() | ||
| 229 | + else: | ||
| 230 | + print("\n[skip] GEMINI_API_KEY 未设置,跳过 Gemini 场景") | ||
| 231 | + | ||
| 232 | + print("\n=== done ===") | ||
| 233 | + | ||
| 234 | + | ||
| 235 | +if __name__ == "__main__": | ||
| 236 | + asyncio.run(main()) | ||
| @@ -0,0 +1,193 @@ | |||
| 1 | +""" | ||
| 2 | +IntelliRouter Prometheus 指标导出 Demo | ||
| 3 | + | ||
| 4 | +演示 Prometheus 指标暴露。使用模拟请求,无需真实 API Key。 | ||
| 5 | +启动后可通过 curl http://localhost:9090/metrics 查看指标。 | ||
| 6 | + | ||
| 7 | +使用方式: | ||
| 8 | + pip install prometheus-client | ||
| 9 | + python examples/demo_prometheus.py | ||
| 10 | + | ||
| 11 | +验证(默认端口 9091,可通过 METRICS_PORT 环境变量修改): | ||
| 12 | + curl http://localhost:9091/metrics | grep intelli_router | ||
| 13 | +""" | ||
| 14 | + | ||
| 15 | +import asyncio | ||
| 16 | +import os | ||
| 17 | +import random | ||
| 18 | +import signal | ||
| 19 | +import sys | ||
| 20 | + | ||
| 21 | +from intelli_router import ( | ||
| 22 | + ReliableRouter, | ||
| 23 | + Deployment, | ||
| 24 | + EventBus, | ||
| 25 | + MetricsCollector, | ||
| 26 | + LoggingHook, | ||
| 27 | +) | ||
| 28 | +from intelli_router.router.base_router import BaseRouter | ||
| 29 | + | ||
| 30 | + | ||
| 31 | +# --------------------------------------------------------------------------- | ||
| 32 | +# 模拟 Deployments | ||
| 33 | +# --------------------------------------------------------------------------- | ||
| 34 | +DEPLOYMENTS = [ | ||
| 35 | + Deployment( | ||
| 36 | + id="azure-east", | ||
| 37 | + model_name="gpt-4", | ||
| 38 | + provider="openai", | ||
| 39 | + api_key="fake-key-1", | ||
| 40 | + api_base="http://localhost:19999", | ||
| 41 | + ), | ||
| 42 | + Deployment( | ||
| 43 | + id="openai-main", | ||
| 44 | + model_name="gpt-4", | ||
| 45 | + provider="openai", | ||
| 46 | + api_key="fake-key-2", | ||
| 47 | + api_base="http://localhost:19998", | ||
| 48 | + ), | ||
| 49 | + Deployment( | ||
| 50 | + id="anthropic-1", | ||
| 51 | + model_name="claude-3", | ||
| 52 | + provider="anthropic", | ||
| 53 | + api_key="fake-key-3", | ||
| 54 | + api_base="http://localhost:19997", | ||
| 55 | + ), | ||
| 56 | +] | ||
| 57 | + | ||
| 58 | + | ||
| 59 | +# --------------------------------------------------------------------------- | ||
| 60 | +# Mock:模拟不同 deployment 的延迟和失败 | ||
| 61 | +# --------------------------------------------------------------------------- | ||
| 62 | +async def mock_completion(self, model, messages, deployment=None, **kwargs): | ||
| 63 | + if deployment is None: | ||
| 64 | + raise RuntimeError("No deployment") | ||
| 65 | + | ||
| 66 | + # 不同 deployment 模拟不同特性 | ||
| 67 | + profiles = { | ||
| 68 | + "azure-east": {"latency": (0.3, 0.8), "fail_rate": 0.05}, | ||
| 69 | + "openai-main": {"latency": (0.5, 1.5), "fail_rate": 0.10}, | ||
| 70 | + "anthropic-1": {"latency": (0.8, 2.0), "fail_rate": 0.15}, | ||
| 71 | + } | ||
| 72 | + profile = profiles.get(deployment.id, {"latency": (0.5, 1.0), "fail_rate": 0.1}) | ||
| 73 | + | ||
| 74 | + latency = random.uniform(*profile["latency"]) | ||
| 75 | + await asyncio.sleep(latency) | ||
| 76 | + | ||
| 77 | + if random.random() < profile["fail_rate"]: | ||
| 78 | + error_types = [ | ||
| 79 | + RuntimeError("Connection timeout"), | ||
| 80 | + RuntimeError("Rate limit exceeded"), | ||
| 81 | + RuntimeError("Internal server error"), | ||
| 82 | + ] | ||
| 83 | + raise random.choice(error_types) | ||
| 84 | + | ||
| 85 | + tokens_prompt = random.randint(50, 300) | ||
| 86 | + tokens_completion = random.randint(20, 200) | ||
| 87 | + return { | ||
| 88 | + "choices": [{"message": {"content": "Hello!"}, "finish_reason": "stop"}], | ||
| 89 | + "usage": { | ||
| 90 | + "prompt_tokens": tokens_prompt, | ||
| 91 | + "completion_tokens": tokens_completion, | ||
| 92 | + "total_tokens": tokens_prompt + tokens_completion, | ||
| 93 | + }, | ||
| 94 | + } | ||
| 95 | + | ||
| 96 | + | ||
| 97 | +async def main(): | ||
| 98 | + print("=" * 60) | ||
| 99 | + print(" IntelliRouter Prometheus Demo") | ||
| 100 | + print("=" * 60) | ||
| 101 | + print() | ||
| 102 | + port = int(os.environ.get("METRICS_PORT", "9091")) | ||
| 103 | + print(f" 指标端口: http://localhost:{port}/metrics") | ||
| 104 | + print(" 模拟 3 个 deployment (gpt-4 x2, claude-3 x1)") | ||
| 105 | + print(" 按 Ctrl+C 退出") | ||
| 106 | + print() | ||
| 107 | + | ||
| 108 | + # 设置 EventBus | ||
| 109 | + bus = EventBus() | ||
| 110 | + | ||
| 111 | + # Prometheus 指标采集 | ||
| 112 | + metrics = MetricsCollector(enable_prometheus=True) | ||
| 113 | + metrics.expose_prometheus(port=port) | ||
| 114 | + bus.register(metrics) | ||
| 115 | + | ||
| 116 | + # 同时开启文本日志(方便观察) | ||
| 117 | + bus.register(LoggingHook(format="text")) | ||
| 118 | + | ||
| 119 | + # 创建 Router | ||
| 120 | + router = ReliableRouter( | ||
| 121 | + deployments=DEPLOYMENTS, | ||
| 122 | + strategy="simple-shuffle", | ||
| 123 | + num_retries=2, | ||
| 124 | + event_bus=bus, | ||
| 125 | + ) | ||
| 126 | + | ||
| 127 | + # Monkey-patch | ||
| 128 | + original_completion = BaseRouter.completion | ||
| 129 | + BaseRouter.completion = mock_completion | ||
| 130 | + | ||
| 131 | + print(" Prometheus HTTP server 已启动") | ||
| 132 | + print(" 验证: curl http://localhost:9090/metrics | grep intelli_router") | ||
| 133 | + print() | ||
| 134 | + print(" 开始模拟请求...\n") | ||
| 135 | + | ||
| 136 | + try: | ||
| 137 | + request_num = 0 | ||
| 138 | + while True: | ||
| 139 | + request_num += 1 | ||
| 140 | + | ||
| 141 | + # 随机选择模型 | ||
| 142 | + model = random.choice(["gpt-4", "gpt-4", "gpt-4", "claude-3"]) | ||
| 143 | + | ||
| 144 | + try: | ||
| 145 | + await router.completion(model, [{"role": "user", "content": f"Request #{request_num}"}]) | ||
| 146 | + except Exception: | ||
| 147 | + pass | ||
| 148 | + | ||
| 149 | + # 模拟流量间隔 | ||
| 150 | + await asyncio.sleep(random.uniform(0.2, 0.8)) | ||
| 151 | + | ||
| 152 | + # 每 20 个请求打印一次汇总 | ||
| 153 | + if request_num % 20 == 0: | ||
| 154 | + stats = metrics.get_stats() | ||
| 155 | + print(f"\n --- 已完成 {request_num} 个请求 ---") | ||
| 156 | + print(f" 成功: {stats['successful']} " | ||
| 157 | + f"失败: {stats['failed']} " | ||
| 158 | + f"重试: {stats['retries']} " | ||
| 159 | + f"Token: {stats['tokens']['total']}") | ||
| 160 | + if stats['latency']['count'] > 0: | ||
| 161 | + print(f" 延迟: avg={stats['latency']['avg']:.2f}s " | ||
| 162 | + f"min={stats['latency']['min']:.2f}s " | ||
| 163 | + f"max={stats['latency']['max']:.2f}s") | ||
| 164 | + print() | ||
| 165 | + | ||
| 166 | + except KeyboardInterrupt: | ||
| 167 | + print("\n\n 正在停止...") | ||
| 168 | + finally: | ||
| 169 | + BaseRouter.completion = original_completion | ||
| 170 | + | ||
| 171 | + # 打印最终统计 | ||
| 172 | + stats = metrics.get_stats() | ||
| 173 | + print("\n ========== 最终统计 ==========") | ||
| 174 | + print(f" 总请求: {stats['total_requests']}") | ||
| 175 | + print(f" 成功: {stats['successful']}") | ||
| 176 | + print(f" 失败: {stats['failed']}") | ||
| 177 | + print(f" 重试: {stats['retries']}") | ||
| 178 | + print(f" 耗尽: {stats['exhausted']}") | ||
| 179 | + if stats['latency']['count'] > 0: | ||
| 180 | + print(f" 平均延迟: {stats['latency']['avg']:.2f}s") | ||
| 181 | + print(f" 总 Token: {stats['tokens']['total']}") | ||
| 182 | + if stats['errors_by_type']: | ||
| 183 | + print(f" 错误分布: {dict(stats['errors_by_type'])}") | ||
| 184 | + print() | ||
| 185 | + print(" Prometheus 指标示例 (最终值):") | ||
| 186 | + print(" intelli_router_requests_total{status='success'} =", stats['successful']) | ||
| 187 | + print(" intelli_router_retries_total =", stats['retries']) | ||
| 188 | + print(" intelli_router_tokens_total =", stats['tokens']['total']) | ||
| 189 | + print() | ||
| 190 | + | ||
| 191 | + | ||
| 192 | +if __name__ == "__main__": | ||
| 193 | + asyncio.run(main()) | ||
| @@ -0,0 +1,158 @@ | |||
| 1 | +""" | ||
| 2 | +IntelliRouter Web 指标看板 Demo | ||
| 3 | + | ||
| 4 | +启动一个轻量 Web 服务,在浏览器中实时展示路由指标。 | ||
| 5 | +使用模拟请求,无需真实 API Key。 | ||
| 6 | + | ||
| 7 | +使用方式: | ||
| 8 | + python examples/demo_web_dashboard.py | ||
| 9 | + | ||
| 10 | + 然后浏览器打开 http://localhost:8080 | ||
| 11 | +""" | ||
| 12 | + | ||
| 13 | +import asyncio | ||
| 14 | +import os | ||
| 15 | +import random | ||
| 16 | + | ||
| 17 | +from intelli_router import ( | ||
| 18 | + ReliableRouter, | ||
| 19 | + Deployment, | ||
| 20 | + EventBus, | ||
| 21 | + MetricsCollector, | ||
| 22 | + MetricsWebServer, | ||
| 23 | +) | ||
| 24 | +from intelli_router.router.base_router import BaseRouter | ||
| 25 | + | ||
| 26 | + | ||
| 27 | +# --------------------------------------------------------------------------- | ||
| 28 | +# 模拟 Deployments | ||
| 29 | +# --------------------------------------------------------------------------- | ||
| 30 | +DEPLOYMENTS = [ | ||
| 31 | + Deployment( | ||
| 32 | + id="azure-east", | ||
| 33 | + model_name="gpt-4", | ||
| 34 | + provider="openai", | ||
| 35 | + api_key="fake-key-1", | ||
| 36 | + api_base="http://localhost:19999", | ||
| 37 | + ), | ||
| 38 | + Deployment( | ||
| 39 | + id="openai-main", | ||
| 40 | + model_name="gpt-4", | ||
| 41 | + provider="openai", | ||
| 42 | + api_key="fake-key-2", | ||
| 43 | + api_base="http://localhost:19998", | ||
| 44 | + ), | ||
| 45 | + Deployment( | ||
| 46 | + id="anthropic-1", | ||
| 47 | + model_name="claude-3", | ||
| 48 | + provider="anthropic", | ||
| 49 | + api_key="fake-key-3", | ||
| 50 | + api_base="http://localhost:19997", | ||
| 51 | + ), | ||
| 52 | + Deployment( | ||
| 53 | + id="deepseek-1", | ||
| 54 | + model_name="deepseek-chat", | ||
| 55 | + provider="deepseek", | ||
| 56 | + api_key="fake-key-4", | ||
| 57 | + api_base="http://localhost:19996", | ||
| 58 | + ), | ||
| 59 | +] | ||
| 60 | + | ||
| 61 | + | ||
| 62 | +# --------------------------------------------------------------------------- | ||
| 63 | +# Mock | ||
| 64 | +# --------------------------------------------------------------------------- | ||
| 65 | +async def mock_completion(self, model, messages, deployment=None, **kwargs): | ||
| 66 | + if deployment is None: | ||
| 67 | + raise RuntimeError("No deployment") | ||
| 68 | + | ||
| 69 | + profiles = { | ||
| 70 | + "azure-east": {"latency": (0.2, 0.6), "fail_rate": 0.03}, | ||
| 71 | + "openai-main": {"latency": (0.4, 1.2), "fail_rate": 0.08}, | ||
| 72 | + "anthropic-1": {"latency": (0.5, 1.5), "fail_rate": 0.12}, | ||
| 73 | + "deepseek-1": {"latency": (0.3, 0.9), "fail_rate": 0.06}, | ||
| 74 | + } | ||
| 75 | + profile = profiles.get(deployment.id, {"latency": (0.5, 1.0), "fail_rate": 0.1}) | ||
| 76 | + | ||
| 77 | + latency = random.uniform(*profile["latency"]) | ||
| 78 | + await asyncio.sleep(latency) | ||
| 79 | + | ||
| 80 | + if random.random() < profile["fail_rate"]: | ||
| 81 | + errors = ["TimeoutError", "RateLimitError", "ConnectionError", "ServerError"] | ||
| 82 | + raise RuntimeError(random.choice(errors)) | ||
| 83 | + | ||
| 84 | + tokens_prompt = random.randint(50, 300) | ||
| 85 | + tokens_completion = random.randint(20, 200) | ||
| 86 | + return { | ||
| 87 | + "choices": [{"message": {"content": "OK"}, "finish_reason": "stop"}], | ||
| 88 | + "usage": { | ||
| 89 | + "prompt_tokens": tokens_prompt, | ||
| 90 | + "completion_tokens": tokens_completion, | ||
| 91 | + "total_tokens": tokens_prompt + tokens_completion, | ||
| 92 | + }, | ||
| 93 | + } | ||
| 94 | + | ||
| 95 | + | ||
| 96 | +async def main(): | ||
| 97 | + port = int(os.environ.get("WEB_PORT", "8080")) | ||
| 98 | + | ||
| 99 | + print("=" * 60) | ||
| 100 | + print(" IntelliRouter Web Dashboard Demo") | ||
| 101 | + print("=" * 60) | ||
| 102 | + print() | ||
| 103 | + print(f" Web 看板: http://localhost:{port}") | ||
| 104 | + print(" 模拟 4 个 deployment,持续发送请求") | ||
| 105 | + print(" 按 Ctrl+C 退出") | ||
| 106 | + print() | ||
| 107 | + | ||
| 108 | + # 设置 | ||
| 109 | + bus = EventBus() | ||
| 110 | + metrics = MetricsCollector() | ||
| 111 | + bus.register(metrics) | ||
| 112 | + | ||
| 113 | + server = MetricsWebServer(metrics, port=port) | ||
| 114 | + server.start() | ||
| 115 | + | ||
| 116 | + router = ReliableRouter( | ||
| 117 | + deployments=DEPLOYMENTS, | ||
| 118 | + strategy="simple-shuffle", | ||
| 119 | + num_retries=2, | ||
| 120 | + event_bus=bus, | ||
| 121 | + ) | ||
| 122 | + | ||
| 123 | + # Monkey-patch | ||
| 124 | + original = BaseRouter.completion | ||
| 125 | + BaseRouter.completion = mock_completion | ||
| 126 | + | ||
| 127 | + print(f" 服务已启动,请在浏览器中打开: http://localhost:{port}") | ||
| 128 | + print() | ||
| 129 | + | ||
| 130 | + try: | ||
| 131 | + request_num = 0 | ||
| 132 | + while True: | ||
| 133 | + request_num += 1 | ||
| 134 | + model = random.choice(["gpt-4", "gpt-4", "claude-3", "deepseek-chat"]) | ||
| 135 | + try: | ||
| 136 | + await router.completion(model, [{"role": "user", "content": f"req-{request_num}"}]) | ||
| 137 | + except Exception: | ||
| 138 | + pass | ||
| 139 | + await asyncio.sleep(random.uniform(0.1, 0.4)) | ||
| 140 | + | ||
| 141 | + if request_num % 50 == 0: | ||
| 142 | + stats = metrics.get_stats() | ||
| 143 | + print(f" [{request_num} reqs] " | ||
| 144 | + f"success={stats['successful']} " | ||
| 145 | + f"failed={stats['failed']} " | ||
| 146 | + f"retries={stats['retries']} " | ||
| 147 | + f"tokens={stats['tokens']['total']}") | ||
| 148 | + | ||
| 149 | + except KeyboardInterrupt: | ||
| 150 | + print("\n 停止中...") | ||
| 151 | + finally: | ||
| 152 | + server.stop() | ||
| 153 | + BaseRouter.completion = original | ||
| 154 | + print(" 已退出。") | ||
| 155 | + | ||
| 156 | + | ||
| 157 | +if __name__ == "__main__": | ||
| 158 | + asyncio.run(main()) | ||
| @@ -1,19 +1,65 @@ | |||
| 1 | """ | 1 | """ |
| 2 | IntelliRouter - 公开接口 | 2 | IntelliRouter - 公开接口 |
| 3 | """ | 3 | """ |
| 4 | -from .router.reliable_router import ReliableRouter | 4 | +from .router.reliable_router import ReliableRouter, MODEL_WILDCARD |
| 5 | from .router.base_router import BaseRouter | 5 | from .router.base_router import BaseRouter |
| 6 | from .core.deployment import Deployment, DeploymentStatus | 6 | from .core.deployment import Deployment, DeploymentStatus |
| 7 | from .strategy.base_strategy import RoutingStrategy | 7 | from .strategy.base_strategy import RoutingStrategy |
| 8 | from .strategy import StrategyType | 8 | from .strategy import StrategyType |
| 9 | from .core.state import TokenUsage | 9 | from .core.state import TokenUsage |
| 10 | +from .provider.base_provider import BaseProviderAdapter | ||
| 11 | +from .types import ( | ||
| 12 | + ToolCall, | ||
| 13 | + UsageMetadata, | ||
| 14 | + AssistantMessage, | ||
| 15 | + AssistantMessageChunk, | ||
| 16 | + ImageGenerationResponse, | ||
| 17 | + AudioGenerationResponse, | ||
| 18 | + VideoGenerationResponse, | ||
| 19 | +) | ||
| 20 | + | ||
| 21 | +from .parser import ( | ||
| 22 | + BaseOutputParser, | ||
| 23 | + JsonOutputParser, | ||
| 24 | + MarkdownOutputParser, | ||
| 25 | +) | ||
| 26 | + | ||
| 27 | +from .observability import ( | ||
| 28 | + EventBus, | ||
| 29 | + EventHandler, | ||
| 30 | + RoutingEvent, | ||
| 31 | + RoutingEventType, | ||
| 32 | + LoggingHook, | ||
| 33 | + MetricsCollector, | ||
| 34 | + MetricsWebServer, | ||
| 35 | +) | ||
| 10 | 36 | ||
| 11 | __all__ = [ | 37 | __all__ = [ |
| 12 | "ReliableRouter", | 38 | "ReliableRouter", |
| 39 | + "MODEL_WILDCARD", | ||
| 13 | "BaseRouter", | 40 | "BaseRouter", |
| 14 | "Deployment", | 41 | "Deployment", |
| 15 | "DeploymentStatus", | 42 | "DeploymentStatus", |
| 16 | "RoutingStrategy", | 43 | "RoutingStrategy", |
| 17 | "StrategyType", | 44 | "StrategyType", |
| 18 | - "TokenUsage" | 45 | + "TokenUsage", |
| 46 | + "BaseProviderAdapter", | ||
| 47 | + "ToolCall", | ||
| 48 | + "UsageMetadata", | ||
| 49 | + "AssistantMessage", | ||
| 50 | + "AssistantMessageChunk", | ||
| 51 | + "ImageGenerationResponse", | ||
| 52 | + "AudioGenerationResponse", | ||
| 53 | + "VideoGenerationResponse", | ||
| 54 | + "BaseOutputParser", | ||
| 55 | + "JsonOutputParser", | ||
| 56 | + "MarkdownOutputParser", | ||
| 57 | + # Observability | ||
| 58 | + "EventBus", | ||
| 59 | + "EventHandler", | ||
| 60 | + "RoutingEvent", | ||
| 61 | + "RoutingEventType", | ||
| 62 | + "LoggingHook", | ||
| 63 | + "MetricsCollector", | ||
| 64 | + "MetricsWebServer", | ||
| 19 | ] | 65 | ] |
| @@ -1,5 +1,5 @@ | |||
| 1 | """ | 1 | """ |
| 2 | -SDK LLM Core - 部署配罫 | 2 | +SDK LLM Core - 部署配置 |
| 3 | 3 | ||
| 4 | Deployment: 一个API端点 | 4 | Deployment: 一个API端点 |
| 5 | DeploymentStatus: 状态枚举 | 5 | DeploymentStatus: 状态枚举 |
| @@ -47,8 +47,8 @@ class Deployment: | |||
| 47 | rpm: Optional[int] = None | 47 | rpm: Optional[int] = None |
| 48 | timeout: Optional[float] = None | 48 | timeout: Optional[float] = None |
| 49 | 49 | ||
| 50 | + provider: str = "openai" | ||
| 50 | verify_ssl: bool = True | 51 | verify_ssl: bool = True |
| 51 | - litellm_params: Dict[str, Any] = field(default_factory=dict) | ||
| 52 | 52 | ||
| 53 | 53 | ||
| 54 | def to_dict(self) -> Dict[str, Any]: | 54 | def to_dict(self) -> Dict[str, Any]: |
| @@ -65,8 +65,8 @@ class Deployment: | |||
| 65 | "tpm": self.tpm, | 65 | "tpm": self.tpm, |
| 66 | "rpm": self.rpm, | 66 | "rpm": self.rpm, |
| 67 | "timeout": self.timeout, | 67 | "timeout": self.timeout, |
| 68 | + "provider": self.provider, | ||
| 68 | "verify_ssl": self.verify_ssl, | 69 | "verify_ssl": self.verify_ssl, |
| 69 | - "litellm_params": self.litellm_params, | ||
| 70 | } | 70 | } |
| 71 | 71 | ||
| 72 | @classmethod | 72 | @classmethod |
| @@ -1,8 +1,5 @@ | |||
| 1 | """ | 1 | """ |
| 2 | SDK LLM Health - 健康检查 | 2 | SDK LLM Health - 健康检查 |
| 3 | - | ||
| 4 | -SDKHealthChecker: 检查部署可用性 | ||
| 5 | -HealthCheckResult: 检查结果 | ||
| 6 | """ | 3 | """ |
| 7 | from typing import Dict, List, Optional, Set | 4 | from typing import Dict, List, Optional, Set |
| 8 | from dataclasses import dataclass, field | 5 | from dataclasses import dataclass, field |
| @@ -11,20 +8,20 @@ import time | |||
| 11 | import httpx | 8 | import httpx |
| 12 | from ..core.deployment import Deployment | 9 | from ..core.deployment import Deployment |
| 13 | from ..core.state import LocalRouterState | 10 | from ..core.state import LocalRouterState |
| 14 | -from ..utils.exceptions import HealthCheckError | 11 | +from ..provider.registry import get_provider_adapter |
| 15 | 12 | ||
| 16 | # 默认检查间隔 (秒) | 13 | # 默认检查间隔 (秒) |
| 17 | DEFAULT_CHECK_INTERVAL = 300 | 14 | DEFAULT_CHECK_INTERVAL = 300 |
| 18 | -# 黙认检查超时 (秒) | 15 | +# 默认检查超时 (秒) |
| 19 | DEFAULT_CHECK_TIMEOUT = 5 | 16 | DEFAULT_CHECK_TIMEOUT = 5 |
| 20 | -# 黙认检查消息 | 17 | +# 默认检查消息 |
| 21 | DEFAULT_CHECK_MESSAGE = [{"role": "user", "content": "ping"}] | 18 | DEFAULT_CHECK_MESSAGE = [{"role": "user", "content": "ping"}] |
| 22 | -# 黙认最大 tokens | 19 | +# 默认最大 tokens |
| 23 | DEFAULT_CHECK_MAX_TOKENS = 10 | 20 | DEFAULT_CHECK_MAX_TOKENS = 10 |
| 24 | 21 | ||
| 25 | @dataclass | 22 | @dataclass |
| 26 | class HealthCheckResult: | 23 | class HealthCheckResult: |
| 27 | - """健庭检查结果""" | 24 | + """健康检查结果""" |
| 28 | deployment_id: str | 25 | deployment_id: str |
| 29 | is_healthy: bool | 26 | is_healthy: bool |
| 30 | latency: Optional[float] = None | 27 | latency: Optional[float] = None |
| @@ -33,7 +30,7 @@ class HealthCheckResult: | |||
| 33 | 30 | ||
| 34 | class SDKHealthChecker: | 31 | class SDKHealthChecker: |
| 35 | """ | 32 | """ |
| 36 | - SDK健庭检查器 - 检查部署可用性 | 33 | + SDK健康检查器 - 检查部署可用性 |
| 37 | 34 | ||
| 38 | 按需检查 + 后台可选 | 35 | 按需检查 + 后台可选 |
| 39 | """ | 36 | """ |
| @@ -53,48 +50,63 @@ class SDKHealthChecker: | |||
| 53 | self.check_timeout = check_timeout | 50 | self.check_timeout = check_timeout |
| 54 | self.check_message = check_message | 51 | self.check_message = check_message |
| 55 | self.check_max_tokens = check_max_tokens | 52 | self.check_max_tokens = check_max_tokens |
| 56 | - # 运行状态 | ||
| 57 | self._running = False | 53 | self._running = False |
| 58 | self._task: Optional[asyncio.Task] = None | 54 | self._task: Optional[asyncio.Task] = None |
| 55 | + self._client: Optional[httpx.AsyncClient] = None | ||
| 56 | + self._adapter_cache: dict = {} | ||
| 57 | + | ||
| 58 | + def _ensure_client(self) -> httpx.AsyncClient: | ||
| 59 | + if self._client is None: | ||
| 60 | + self._client = httpx.AsyncClient(timeout=self.check_timeout) | ||
| 61 | + return self._client | ||
| 62 | + | ||
| 63 | + def _get_cached_adapter(self, provider: str): | ||
| 64 | + if provider not in self._adapter_cache: | ||
| 65 | + self._adapter_cache[provider] = get_provider_adapter(provider) | ||
| 66 | + return self._adapter_cache[provider] | ||
| 67 | + | ||
| 68 | + async def close(self) -> None: | ||
| 69 | + if self._client is not None: | ||
| 70 | + await self._client.aclose() | ||
| 71 | + self._client = None | ||
| 59 | 72 | ||
| 60 | async def check_deployment( | 73 | async def check_deployment( |
| 61 | self, | 74 | self, |
| 62 | deployment: Deployment | 75 | deployment: Deployment |
| 63 | ) -> HealthCheckResult: | 76 | ) -> HealthCheckResult: |
| 64 | """ | 77 | """ |
| 65 | - 检查单个部署健庭状态 | 78 | + 检查单个部署健康状态 |
| 66 | 策略: 发送一个最小化completion请求 | 79 | 策略: 发送一个最小化completion请求 |
| 67 | """ | 80 | """ |
| 68 | start = time.time() | 81 | start = time.time() |
| 69 | try: | 82 | try: |
| 70 | - # 构造请求 | 83 | + adapter = self._get_cached_adapter(deployment.provider) |
| 71 | - async with httpx.AsyncClient(timeout=self.check_timeout) as client: | 84 | + request_body = adapter.transform_request( |
| 72 | - response = await client.post( | 85 | + model=deployment.model_name, |
| 73 | - f"{deployment.api_base}/chat/completions", | 86 | + messages=self.check_message, |
| 74 | - headers={ | 87 | + deployment=deployment, |
| 75 | - "Authorization": f"Bearer {deployment.api_key}", | 88 | + max_tokens=self.check_max_tokens, |
| 76 | - "Content-Type": "application/json" | 89 | + ) |
| 77 | - }, | 90 | + url = adapter.get_api_url(deployment) |
| 78 | - json={ | 91 | + headers = adapter.get_headers(deployment) |
| 79 | - "model": deployment.model_name, | 92 | + |
| 80 | - "messages": self.check_message, | 93 | + client = self._ensure_client() |
| 81 | - "max_tokens": self.check_max_tokens | 94 | + response = await client.post( |
| 82 | - } | 95 | + url, headers=headers, json=request_body |
| 96 | + ) | ||
| 97 | + if response.status_code == 200: | ||
| 98 | + latency = time.time() - start | ||
| 99 | + return HealthCheckResult( | ||
| 100 | + deployment_id=deployment.id, | ||
| 101 | + is_healthy=True, | ||
| 102 | + latency=latency | ||
| 103 | + ) | ||
| 104 | + else: | ||
| 105 | + return HealthCheckResult( | ||
| 106 | + deployment_id=deployment.id, | ||
| 107 | + is_healthy=False, | ||
| 108 | + error=f"HTTP {response.status_code}" | ||
| 83 | ) | 109 | ) |
| 84 | - # 检查响应 | ||
| 85 | - if response.status_code == 200: | ||
| 86 | - latency = time.time() - start | ||
| 87 | - return HealthCheckResult( | ||
| 88 | - deployment_id=deployment.id, | ||
| 89 | - is_healthy=True, | ||
| 90 | - latency=latency | ||
| 91 | - ) | ||
| 92 | - else: | ||
| 93 | - return HealthCheckResult( | ||
| 94 | - deployment_id=deployment.id, | ||
| 95 | - is_healthy=False, | ||
| 96 | - error=f"HTTP {response.status_code}" | ||
| 97 | - ) | ||
| 98 | except Exception as e: | 110 | except Exception as e: |
| 99 | return HealthCheckResult( | 111 | return HealthCheckResult( |
| 100 | deployment_id=deployment.id, | 112 | deployment_id=deployment.id, |
| @@ -124,7 +136,7 @@ class SDKHealthChecker: | |||
| 124 | return results | 136 | return results |
| 125 | 137 | ||
| 126 | def get_healthy_deployments(self, now: float) -> List[Deployment]: | 138 | def get_healthy_deployments(self, now: float) -> List[Deployment]: |
| 127 | - """获取健庭部署列表""" | 139 | + """获取健康部署列表""" |
| 128 | healthy = [] | 140 | healthy = [] |
| 129 | for dep in self.deployments: | 141 | for dep in self.deployments: |
| 130 | if self.state.health_state.get(dep.id, True): | 142 | if self.state.health_state.get(dep.id, True): |
| @@ -133,7 +145,7 @@ class SDKHealthChecker: | |||
| 133 | return healthy | 145 | return healthy |
| 134 | 146 | ||
| 135 | def get_unhealthy_ids(self) -> Set[str]: | 147 | def get_unhealthy_ids(self) -> Set[str]: |
| 136 | - """获取不健庭部署ID集合""" | 148 | + """获取不健康部署ID集合""" |
| 137 | return { | 149 | return { |
| 138 | dep_id | 150 | dep_id |
| 139 | for dep_id, is_healthy in self.state.health_state.items() | 151 | for dep_id, is_healthy in self.state.health_state.items() |
| @@ -141,14 +153,14 @@ class SDKHealthChecker: | |||
| 141 | } | 153 | } |
| 142 | 154 | ||
| 143 | async def start_background_check(self) -> None: | 155 | async def start_background_check(self) -> None: |
| 144 | - """启动后台健庭检查""" | 156 | + """启动后台健康检查""" |
| 145 | if self._running: | 157 | if self._running: |
| 146 | return | 158 | return |
| 147 | self._running = True | 159 | self._running = True |
| 148 | self._task = asyncio.create_task(self._background_loop()) | 160 | self._task = asyncio.create_task(self._background_loop()) |
| 149 | 161 | ||
| 150 | async def stop_background_check(self) -> None: | 162 | async def stop_background_check(self) -> None: |
| 151 | - """停止后台健庭检查""" | 163 | + """停止后台健康检查""" |
| 152 | self._running = False | 164 | self._running = False |
| 153 | if self._task: | 165 | if self._task: |
| 154 | self._task.cancel() | 166 | self._task.cancel() |
| @@ -157,13 +169,13 @@ class SDKHealthChecker: | |||
| 157 | except asyncio.CancelledError: | 169 | except asyncio.CancelledError: |
| 158 | pass | 170 | pass |
| 159 | self._task = None | 171 | self._task = None |
| 172 | + await self.close() | ||
| 160 | 173 | ||
| 161 | async def _background_loop(self) -> None: | 174 | async def _background_loop(self) -> None: |
| 162 | """后台检查循环""" | 175 | """后台检查循环""" |
| 163 | while self._running: | 176 | while self._running: |
| 164 | try: | 177 | try: |
| 165 | await self.check_all_deployments() | 178 | await self.check_all_deployments() |
| 166 | - except Exception as e: | 179 | + except Exception: |
| 167 | - # 记录错误但继续运行 | ||
| 168 | pass | 180 | pass |
| 169 | await asyncio.sleep(self.check_interval) | 181 | await asyncio.sleep(self.check_interval) |
| @@ -0,0 +1,17 @@ | |||
| 1 | +"""IntelliRouter 可观测性子系统""" | ||
| 2 | +from .events import RoutingEvent, RoutingEventType | ||
| 3 | +from .handler import EventHandler | ||
| 4 | +from .bus import EventBus | ||
| 5 | +from .logging_hook import LoggingHook | ||
| 6 | +from .metrics import MetricsCollector | ||
| 7 | +from .web_dashboard import MetricsWebServer | ||
| 8 | + | ||
| 9 | +__all__ = [ | ||
| 10 | + "RoutingEvent", | ||
| 11 | + "RoutingEventType", | ||
| 12 | + "EventHandler", | ||
| 13 | + "EventBus", | ||
| 14 | + "LoggingHook", | ||
| 15 | + "MetricsCollector", | ||
| 16 | + "MetricsWebServer", | ||
| 17 | +] | ||
| @@ -0,0 +1,51 @@ | |||
| 1 | +"""事件总线 - 管理 EventHandler 注册和事件分发""" | ||
| 2 | +import logging | ||
| 3 | +from typing import List | ||
| 4 | + | ||
| 5 | +from .events import RoutingEvent | ||
| 6 | +from .handler import EventHandler | ||
| 7 | + | ||
| 8 | +logger = logging.getLogger(__name__) | ||
| 9 | + | ||
| 10 | + | ||
| 11 | +class EventBus: | ||
| 12 | + """事件总线 | ||
| 13 | + | ||
| 14 | + 管理 EventHandler 的注册和事件分发。 | ||
| 15 | + 无 handler 注册时 emit() 几乎零开销。 | ||
| 16 | + 事件通过直接 await 分发,单个 handler 异常不影响其他 handler 和路由主路径。 | ||
| 17 | + """ | ||
| 18 | + | ||
| 19 | + def __init__(self): | ||
| 20 | + self._handlers: List[EventHandler] = [] | ||
| 21 | + | ||
| 22 | + @property | ||
| 23 | + def handlers(self) -> List[EventHandler]: | ||
| 24 | + """已注册的 handler 列表(只读副本)""" | ||
| 25 | + return list(self._handlers) | ||
| 26 | + | ||
| 27 | + def register(self, handler: EventHandler) -> None: | ||
| 28 | + """注册一个事件处理器""" | ||
| 29 | + self._handlers.append(handler) | ||
| 30 | + | ||
| 31 | + def unregister(self, handler: EventHandler) -> None: | ||
| 32 | + """注销一个事件处理器""" | ||
| 33 | + self._handlers.remove(handler) | ||
| 34 | + | ||
| 35 | + async def emit(self, event: RoutingEvent) -> None: | ||
| 36 | + """分发事件到所有已注册的处理器 | ||
| 37 | + | ||
| 38 | + 每个 handler 依次 await。如果某个 handler 抛异常, | ||
| 39 | + 异常被捕获并记录日志,不会影响后续 handler 或请求路径。 | ||
| 40 | + """ | ||
| 41 | + if not self._handlers: | ||
| 42 | + return | ||
| 43 | + for handler in self._handlers: | ||
| 44 | + try: | ||
| 45 | + await handler.handle_event(event) | ||
| 46 | + except Exception: | ||
| 47 | + logger.debug( | ||
| 48 | + "EventHandler %s failed for event %s", | ||
| 49 | + handler.name, event.event_type.value, | ||
| 50 | + exc_info=True, | ||
| 51 | + ) | ||
| @@ -0,0 +1,88 @@ | |||
| 1 | +"""路由事件定义""" | ||
| 2 | +from dataclasses import dataclass, field | ||
| 3 | +from enum import Enum | ||
| 4 | +from typing import Optional, Dict, Any | ||
| 5 | +import time | ||
| 6 | +import uuid | ||
| 7 | + | ||
| 8 | + | ||
| 9 | +class RoutingEventType(str, Enum): | ||
| 10 | + """路由事件类型""" | ||
| 11 | + # 请求生命周期 | ||
| 12 | + REQUEST_STARTED = "request_started" | ||
| 13 | + REQUEST_SUCCEEDED = "request_succeeded" | ||
| 14 | + REQUEST_RETRIED = "request_retried" | ||
| 15 | + ALL_DEPLOYMENTS_EXHAUSTED = "all_deployments_exhausted" | ||
| 16 | + | ||
| 17 | + # 流式请求 | ||
| 18 | + STREAM_STARTED = "stream_started" | ||
| 19 | + STREAM_SUCCEEDED = "stream_succeeded" | ||
| 20 | + | ||
| 21 | + | ||
| 22 | +@dataclass | ||
| 23 | +class RoutingEvent: | ||
| 24 | + """一次路由事件的完整信息 | ||
| 25 | + | ||
| 26 | + 不携带请求 body (messages) 以避免敏感数据泄露和内存开销。 | ||
| 27 | + """ | ||
| 28 | + # 必填字段 | ||
| 29 | + event_type: RoutingEventType | ||
| 30 | + request_id: str | ||
| 31 | + model: str | ||
| 32 | + | ||
| 33 | + # 时间戳 | ||
| 34 | + timestamp: float = field(default_factory=time.time) | ||
| 35 | + | ||
| 36 | + # 部署信息 | ||
| 37 | + deployment_id: Optional[str] = None | ||
| 38 | + provider: Optional[str] = None | ||
| 39 | + | ||
| 40 | + # 性能数据 | ||
| 41 | + latency: Optional[float] = None | ||
| 42 | + prompt_tokens: Optional[int] = None | ||
| 43 | + completion_tokens: Optional[int] = None | ||
| 44 | + total_tokens: Optional[int] = None | ||
| 45 | + | ||
| 46 | + # 重试信息 | ||
| 47 | + attempt: int = 0 | ||
| 48 | + total_attempts: int = 0 | ||
| 49 | + | ||
| 50 | + # 错误信息 | ||
| 51 | + error_type: Optional[str] = None | ||
| 52 | + error_message: Optional[str] = None | ||
| 53 | + | ||
| 54 | + # 流式特有 | ||
| 55 | + chunk_count: Optional[int] = None | ||
| 56 | + | ||
| 57 | + # 扩展字段 | ||
| 58 | + extra: Dict[str, Any] = field(default_factory=dict) | ||
| 59 | + | ||
| 60 | + @staticmethod | ||
| 61 | + def new_request_id() -> str: | ||
| 62 | + """生成短唯一请求 ID""" | ||
| 63 | + return uuid.uuid4().hex[:12] | ||
| 64 | + | ||
| 65 | + def to_dict(self) -> Dict[str, Any]: | ||
| 66 | + """序列化为字典,省略 None 字段""" | ||
| 67 | + result: Dict[str, Any] = { | ||
| 68 | + "event_type": self.event_type.value, | ||
| 69 | + "request_id": self.request_id, | ||
| 70 | + "model": self.model, | ||
| 71 | + "timestamp": self.timestamp, | ||
| 72 | + } | ||
| 73 | + optional_fields = [ | ||
| 74 | + "deployment_id", "provider", "latency", | ||
| 75 | + "prompt_tokens", "completion_tokens", "total_tokens", | ||
| 76 | + "error_type", "error_message", "chunk_count", | ||
| 77 | + ] | ||
| 78 | + for attr in optional_fields: | ||
| 79 | + val = getattr(self, attr) | ||
| 80 | + if val is not None: | ||
| 81 | + result[attr] = val | ||
| 82 | + if self.attempt > 0: | ||
| 83 | + result["attempt"] = self.attempt | ||
| 84 | + if self.total_attempts > 0: | ||
| 85 | + result["total_attempts"] = self.total_attempts | ||
| 86 | + if self.extra: | ||
| 87 | + result["extra"] = self.extra | ||
| 88 | + return result | ||
| @@ -0,0 +1,22 @@ | |||
| 1 | +"""事件处理器抽象基类""" | ||
| 2 | +from abc import ABC, abstractmethod | ||
| 3 | + | ||
| 4 | +from .events import RoutingEvent | ||
| 5 | + | ||
| 6 | + | ||
| 7 | +class EventHandler(ABC): | ||
| 8 | + """事件处理器基类 | ||
| 9 | + | ||
| 10 | + 用户/集成方实现此接口来处理路由事件。 | ||
| 11 | + 实现应避免抛出异常,内部处理错误。 | ||
| 12 | + """ | ||
| 13 | + | ||
| 14 | + @abstractmethod | ||
| 15 | + async def handle_event(self, event: RoutingEvent) -> None: | ||
| 16 | + """处理一个路由事件""" | ||
| 17 | + ... | ||
| 18 | + | ||
| 19 | + @property | ||
| 20 | + def name(self) -> str: | ||
| 21 | + """处理器名称(用于日志标识)""" | ||
| 22 | + return self.__class__.__name__ | ||
| @@ -0,0 +1,71 @@ | |||
| 1 | +"""结构化日志事件处理器""" | ||
| 2 | +import json | ||
| 3 | +import logging | ||
| 4 | +from typing import Optional | ||
| 5 | + | ||
| 6 | +from .events import RoutingEvent | ||
| 7 | +from .handler import EventHandler | ||
| 8 | + | ||
| 9 | + | ||
| 10 | +class LoggingHook(EventHandler): | ||
| 11 | + """标准日志 Hook | ||
| 12 | + | ||
| 13 | + 将路由事件输出到 Python logging 系统。 | ||
| 14 | + 支持 JSON 和人类可读文本两种格式。 | ||
| 15 | + | ||
| 16 | + 用法: | ||
| 17 | + hook = LoggingHook() # JSON 格式,INFO 级别 | ||
| 18 | + hook = LoggingHook(format="text") # 人类可读文本 | ||
| 19 | + hook = LoggingHook(logger_name="my.logger") # 自定义 logger | ||
| 20 | + """ | ||
| 21 | + | ||
| 22 | + def __init__( | ||
| 23 | + self, | ||
| 24 | + logger_name: str = "intelli_router", | ||
| 25 | + level: int = logging.INFO, | ||
| 26 | + format: str = "json", | ||
| 27 | + ): | ||
| 28 | + """ | ||
| 29 | + Args: | ||
| 30 | + logger_name: Python logger 名称 | ||
| 31 | + level: 日志级别 | ||
| 32 | + format: 输出格式 - "json" 或 "text" | ||
| 33 | + """ | ||
| 34 | + self._logger = logging.getLogger(logger_name) | ||
| 35 | + self._level = level | ||
| 36 | + self._format = format | ||
| 37 | + | ||
| 38 | + async def handle_event(self, event: RoutingEvent) -> None: | ||
| 39 | + """将事件格式化并输出到 logger""" | ||
| 40 | + message = self._format_event(event) | ||
| 41 | + self._logger.log(self._level, message) | ||
| 42 | + | ||
| 43 | + def _format_event(self, event: RoutingEvent) -> str: | ||
| 44 | + if self._format == "json": | ||
| 45 | + return self._format_json(event) | ||
| 46 | + return self._format_text(event) | ||
| 47 | + | ||
| 48 | + def _format_json(self, event: RoutingEvent) -> str: | ||
| 49 | + return json.dumps(event.to_dict(), ensure_ascii=False) | ||
| 50 | + | ||
| 51 | + def _format_text(self, event: RoutingEvent) -> str: | ||
| 52 | + parts = [ | ||
| 53 | + f"[{event.event_type.value}]", | ||
| 54 | + f"req={event.request_id[:8]}", | ||
| 55 | + f"model={event.model}", | ||
| 56 | + ] | ||
| 57 | + if event.deployment_id: | ||
| 58 | + parts.append(f"dep={event.deployment_id}") | ||
| 59 | + if event.provider: | ||
| 60 | + parts.append(f"provider={event.provider}") | ||
| 61 | + if event.latency is not None: | ||
| 62 | + parts.append(f"lat={event.latency:.3f}s") | ||
| 63 | + if event.total_tokens is not None: | ||
| 64 | + parts.append(f"tokens={event.total_tokens}") | ||
| 65 | + if event.attempt > 0: | ||
| 66 | + parts.append(f"attempt={event.attempt}/{event.total_attempts}") | ||
| 67 | + if event.error_message: | ||
| 68 | + parts.append(f"err={event.error_message}") | ||
| 69 | + if event.chunk_count is not None: | ||
| 70 | + parts.append(f"chunks={event.chunk_count}") | ||
| 71 | + return " ".join(parts) | ||
| @@ -0,0 +1,417 @@ | |||
| 1 | +"""指标采集事件处理器""" | ||
| 2 | +import time | ||
| 3 | +from collections import defaultdict, deque | ||
| 4 | +from dataclasses import dataclass, field | ||
| 5 | +from typing import Dict, Any, List, Deque, Optional | ||
| 6 | + | ||
| 7 | +from .events import RoutingEvent, RoutingEventType | ||
| 8 | +from .handler import EventHandler | ||
| 9 | + | ||
| 10 | + | ||
| 11 | +@dataclass | ||
| 12 | +class LatencyStats: | ||
| 13 | + """简易延迟统计(含分位数)""" | ||
| 14 | + | ||
| 15 | + _SAMPLE_LIMIT: int = field(default=1000, repr=False) | ||
| 16 | + | ||
| 17 | + count: int = 0 | ||
| 18 | + total: float = 0.0 | ||
| 19 | + min_val: float = float("inf") | ||
| 20 | + max_val: float = 0.0 | ||
| 21 | + _samples: List[float] = field(default_factory=list, repr=False) | ||
| 22 | + | ||
| 23 | + def observe(self, value: float) -> None: | ||
| 24 | + self.count += 1 | ||
| 25 | + self.total += value | ||
| 26 | + if value < self.min_val: | ||
| 27 | + self.min_val = value | ||
| 28 | + if value > self.max_val: | ||
| 29 | + self.max_val = value | ||
| 30 | + self._samples.append(value) | ||
| 31 | + if len(self._samples) > self._SAMPLE_LIMIT: | ||
| 32 | + self._samples = self._samples[-self._SAMPLE_LIMIT:] | ||
| 33 | + | ||
| 34 | + def percentile(self, p: float) -> Optional[float]: | ||
| 35 | + """计算分位数 (0-100)""" | ||
| 36 | + if not self._samples: | ||
| 37 | + return None | ||
| 38 | + sorted_s = sorted(self._samples) | ||
| 39 | + idx = int(len(sorted_s) * p / 100) | ||
| 40 | + return sorted_s[min(idx, len(sorted_s) - 1)] | ||
| 41 | + | ||
| 42 | + @property | ||
| 43 | + def avg(self) -> float: | ||
| 44 | + return self.total / self.count if self.count > 0 else 0.0 | ||
| 45 | + | ||
| 46 | + def to_dict(self) -> Dict[str, Any]: | ||
| 47 | + if self.count == 0: | ||
| 48 | + return {"count": 0, "avg": 0.0, "min": None, "max": None, | ||
| 49 | + "p50": None, "p95": None, "p99": None} | ||
| 50 | + return { | ||
| 51 | + "count": self.count, | ||
| 52 | + "avg": round(self.avg, 4), | ||
| 53 | + "min": round(self.min_val, 4), | ||
| 54 | + "max": round(self.max_val, 4), | ||
| 55 | + "p50": round(self.percentile(50), 4), | ||
| 56 | + "p95": round(self.percentile(95), 4), | ||
| 57 | + "p99": round(self.percentile(99), 4), | ||
| 58 | + } | ||
| 59 | + | ||
| 60 | + | ||
| 61 | +class MetricsCollector(EventHandler): | ||
| 62 | + """指标采集器 | ||
| 63 | + | ||
| 64 | + 维护内存中的请求统计数据,可通过 get_stats() 获取快照。 | ||
| 65 | + 可选开启 Prometheus 指标导出(需安装 prometheus-client)。 | ||
| 66 | + | ||
| 67 | + 用法: | ||
| 68 | + collector = MetricsCollector() | ||
| 69 | + bus.register(collector) | ||
| 70 | + # ... 执行请求 ... | ||
| 71 | + print(collector.get_stats()) | ||
| 72 | + | ||
| 73 | + # 可选 Prometheus: | ||
| 74 | + collector = MetricsCollector(enable_prometheus=True) | ||
| 75 | + """ | ||
| 76 | + | ||
| 77 | + def __init__( | ||
| 78 | + self, | ||
| 79 | + enable_prometheus: bool = False, | ||
| 80 | + prometheus_prefix: str = "intelli_router", | ||
| 81 | + ): | ||
| 82 | + self._enable_prometheus = enable_prometheus | ||
| 83 | + self._prefix = prometheus_prefix | ||
| 84 | + | ||
| 85 | + # 内存计数器 | ||
| 86 | + self._request_count: int = 0 | ||
| 87 | + self._success_count: int = 0 | ||
| 88 | + self._failure_count: int = 0 | ||
| 89 | + self._retry_count: int = 0 | ||
| 90 | + self._exhausted_count: int = 0 | ||
| 91 | + | ||
| 92 | + # 按 model 统计 | ||
| 93 | + self._model_requests: Dict[str, int] = defaultdict(int) | ||
| 94 | + self._model_successes: Dict[str, int] = defaultdict(int) | ||
| 95 | + self._model_failures: Dict[str, int] = defaultdict(int) | ||
| 96 | + | ||
| 97 | + # 按 deployment 细分统计 | ||
| 98 | + self._deployment_requests: Dict[str, int] = defaultdict(int) | ||
| 99 | + self._deployment_successes: Dict[str, int] = defaultdict(int) | ||
| 100 | + self._deployment_failures: Dict[str, int] = defaultdict(int) | ||
| 101 | + self._deployment_latency: Dict[str, LatencyStats] = defaultdict(LatencyStats) | ||
| 102 | + self._deployment_tokens: Dict[str, int] = defaultdict(int) | ||
| 103 | + self._deployment_provider: Dict[str, str] = {} | ||
| 104 | + | ||
| 105 | + # 按错误类型统计 | ||
| 106 | + self._errors_by_type: Dict[str, int] = defaultdict(int) | ||
| 107 | + | ||
| 108 | + # 延迟统计 | ||
| 109 | + self._latency: LatencyStats = LatencyStats() | ||
| 110 | + self._latency_by_model: Dict[str, LatencyStats] = defaultdict(LatencyStats) | ||
| 111 | + | ||
| 112 | + # Token 统计 | ||
| 113 | + self._total_prompt_tokens: int = 0 | ||
| 114 | + self._total_completion_tokens: int = 0 | ||
| 115 | + | ||
| 116 | + # Stream 统计 | ||
| 117 | + self._stream_count: int = 0 | ||
| 118 | + self._total_chunks: int = 0 | ||
| 119 | + | ||
| 120 | + # TTFT (Time To First Token) 统计 | ||
| 121 | + self._ttft: LatencyStats = LatencyStats() | ||
| 122 | + self._ttft_by_model: Dict[str, LatencyStats] = defaultdict(LatencyStats) | ||
| 123 | + | ||
| 124 | + # 吐字速率 (tokens/s) | ||
| 125 | + self._tokens_per_sec: LatencyStats = LatencyStats() | ||
| 126 | + | ||
| 127 | + # QPS 追踪(最近 60s 请求时间戳) | ||
| 128 | + self._request_timestamps: Deque[float] = deque() | ||
| 129 | + | ||
| 130 | + # 请求时间线(最近 N 条记录) | ||
| 131 | + self._timeline: Deque[Dict[str, Any]] = deque(maxlen=100) | ||
| 132 | + | ||
| 133 | + # Prometheus (延迟初始化) | ||
| 134 | + self._prom = None | ||
| 135 | + if enable_prometheus: | ||
| 136 | + self._init_prometheus() | ||
| 137 | + | ||
| 138 | + def _init_prometheus(self) -> None: | ||
| 139 | + try: | ||
| 140 | + from prometheus_client import Counter, Histogram | ||
| 141 | + except ImportError: | ||
| 142 | + raise ImportError( | ||
| 143 | + "prometheus-client is required for Prometheus metrics. " | ||
| 144 | + "Install with: pip install intelli-router[metrics]" | ||
| 145 | + ) | ||
| 146 | + | ||
| 147 | + p = self._prefix | ||
| 148 | + self._prom = { | ||
| 149 | + "requests_total": Counter( | ||
| 150 | + f"{p}_requests_total", | ||
| 151 | + "Total routing requests", | ||
| 152 | + ["model", "status"], | ||
| 153 | + ), | ||
| 154 | + "latency": Histogram( | ||
| 155 | + f"{p}_request_latency_seconds", | ||
| 156 | + "Request latency in seconds", | ||
| 157 | + ["model"], | ||
| 158 | + buckets=(0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 30.0, 60.0), | ||
| 159 | + ), | ||
| 160 | + "tokens_total": Counter( | ||
| 161 | + f"{p}_tokens_total", | ||
| 162 | + "Total tokens consumed", | ||
| 163 | + ["model", "type"], | ||
| 164 | + ), | ||
| 165 | + "retries_total": Counter( | ||
| 166 | + f"{p}_retries_total", | ||
| 167 | + "Total retry attempts", | ||
| 168 | + ["model"], | ||
| 169 | + ), | ||
| 170 | + "errors_total": Counter( | ||
| 171 | + f"{p}_errors_total", | ||
| 172 | + "Total errors by type", | ||
| 173 | + ["model", "error_type"], | ||
| 174 | + ), | ||
| 175 | + } | ||
| 176 | + | ||
| 177 | + async def handle_event(self, event: RoutingEvent) -> None: | ||
| 178 | + """处理事件并更新指标""" | ||
| 179 | + et = event.event_type | ||
| 180 | + | ||
| 181 | + if et == RoutingEventType.REQUEST_STARTED: | ||
| 182 | + self._on_request_started(event) | ||
| 183 | + elif et == RoutingEventType.REQUEST_SUCCEEDED: | ||
| 184 | + self._on_request_succeeded(event) | ||
| 185 | + elif et == RoutingEventType.REQUEST_RETRIED: | ||
| 186 | + self._on_request_retried(event) | ||
| 187 | + elif et == RoutingEventType.ALL_DEPLOYMENTS_EXHAUSTED: | ||
| 188 | + self._on_all_exhausted(event) | ||
| 189 | + elif et == RoutingEventType.STREAM_STARTED: | ||
| 190 | + self._on_stream_started(event) | ||
| 191 | + elif et == RoutingEventType.STREAM_SUCCEEDED: | ||
| 192 | + self._on_stream_succeeded(event) | ||
| 193 | + | ||
| 194 | + def _on_request_started(self, event: RoutingEvent) -> None: | ||
| 195 | + self._request_count += 1 | ||
| 196 | + self._model_requests[event.model] += 1 | ||
| 197 | + self._request_timestamps.append(event.timestamp) | ||
| 198 | + | ||
| 199 | + def _on_request_succeeded(self, event: RoutingEvent) -> None: | ||
| 200 | + self._success_count += 1 | ||
| 201 | + self._model_successes[event.model] += 1 | ||
| 202 | + if event.deployment_id: | ||
| 203 | + self._deployment_requests[event.deployment_id] += 1 | ||
| 204 | + self._deployment_successes[event.deployment_id] += 1 | ||
| 205 | + if event.provider: | ||
| 206 | + self._deployment_provider[event.deployment_id] = event.provider | ||
| 207 | + if event.latency is not None: | ||
| 208 | + self._deployment_latency[event.deployment_id].observe(event.latency) | ||
| 209 | + total_tok = (event.prompt_tokens or 0) + (event.completion_tokens or 0) | ||
| 210 | + if total_tok: | ||
| 211 | + self._deployment_tokens[event.deployment_id] += total_tok | ||
| 212 | + if event.latency is not None: | ||
| 213 | + self._latency.observe(event.latency) | ||
| 214 | + self._latency_by_model[event.model].observe(event.latency) | ||
| 215 | + if event.prompt_tokens: | ||
| 216 | + self._total_prompt_tokens += event.prompt_tokens | ||
| 217 | + if event.completion_tokens: | ||
| 218 | + self._total_completion_tokens += event.completion_tokens | ||
| 219 | + # 吐字速率 | ||
| 220 | + if event.completion_tokens and event.latency and event.latency > 0: | ||
| 221 | + self._tokens_per_sec.observe(event.completion_tokens / event.latency) | ||
| 222 | + | ||
| 223 | + # 时间线 | ||
| 224 | + self._timeline.append({ | ||
| 225 | + "ts": event.timestamp, | ||
| 226 | + "type": "success", | ||
| 227 | + "model": event.model, | ||
| 228 | + "deployment": event.deployment_id, | ||
| 229 | + "latency": event.latency, | ||
| 230 | + "tokens": (event.prompt_tokens or 0) + (event.completion_tokens or 0), | ||
| 231 | + }) | ||
| 232 | + | ||
| 233 | + if self._prom: | ||
| 234 | + self._prom["requests_total"].labels(model=event.model, status="success").inc() | ||
| 235 | + if event.latency is not None: | ||
| 236 | + self._prom["latency"].labels(model=event.model).observe(event.latency) | ||
| 237 | + if event.prompt_tokens: | ||
| 238 | + self._prom["tokens_total"].labels(model=event.model, type="prompt").inc(event.prompt_tokens) | ||
| 239 | + if event.completion_tokens: | ||
| 240 | + self._prom["tokens_total"].labels(model=event.model, type="completion").inc(event.completion_tokens) | ||
| 241 | + | ||
| 242 | + def _on_request_retried(self, event: RoutingEvent) -> None: | ||
| 243 | + self._retry_count += 1 | ||
| 244 | + if event.deployment_id: | ||
| 245 | + self._deployment_failures[event.deployment_id] += 1 | ||
| 246 | + if event.error_type: | ||
| 247 | + self._errors_by_type[event.error_type] += 1 | ||
| 248 | + self._timeline.append({ | ||
| 249 | + "ts": event.timestamp, | ||
| 250 | + "type": "retry", | ||
| 251 | + "model": event.model, | ||
| 252 | + "deployment": event.deployment_id, | ||
| 253 | + "latency": None, | ||
| 254 | + "error": event.error_type, | ||
| 255 | + }) | ||
| 256 | + if self._prom: | ||
| 257 | + self._prom["retries_total"].labels(model=event.model).inc() | ||
| 258 | + if event.error_type: | ||
| 259 | + self._prom["errors_total"].labels(model=event.model, error_type=event.error_type).inc() | ||
| 260 | + | ||
| 261 | + def _on_all_exhausted(self, event: RoutingEvent) -> None: | ||
| 262 | + self._exhausted_count += 1 | ||
| 263 | + if self._prom: | ||
| 264 | + self._prom["requests_total"].labels(model=event.model, status="exhausted").inc() | ||
| 265 | + | ||
| 266 | + def _on_stream_started(self, event: RoutingEvent) -> None: | ||
| 267 | + self._stream_count += 1 | ||
| 268 | + self._request_count += 1 | ||
| 269 | + self._model_requests[event.model] += 1 | ||
| 270 | + self._request_timestamps.append(event.timestamp) | ||
| 271 | + | ||
| 272 | + def _on_stream_succeeded(self, event: RoutingEvent) -> None: | ||
| 273 | + self._success_count += 1 | ||
| 274 | + self._model_successes[event.model] += 1 | ||
| 275 | + if event.deployment_id: | ||
| 276 | + self._deployment_requests[event.deployment_id] += 1 | ||
| 277 | + self._deployment_successes[event.deployment_id] += 1 | ||
| 278 | + if event.provider: | ||
| 279 | + self._deployment_provider[event.deployment_id] = event.provider | ||
| 280 | + if event.latency is not None: | ||
| 281 | + self._deployment_latency[event.deployment_id].observe(event.latency) | ||
| 282 | + if event.latency is not None: | ||
| 283 | + self._latency.observe(event.latency) | ||
| 284 | + self._latency_by_model[event.model].observe(event.latency) | ||
| 285 | + if event.chunk_count: | ||
| 286 | + self._total_chunks += event.chunk_count | ||
| 287 | + # 吐字速率 | ||
| 288 | + if event.chunk_count and event.latency and event.latency > 0: | ||
| 289 | + self._tokens_per_sec.observe(event.chunk_count / event.latency) | ||
| 290 | + # TTFT | ||
| 291 | + ttft = event.extra.get("ttft") | ||
| 292 | + if ttft is not None: | ||
| 293 | + self._ttft.observe(ttft) | ||
| 294 | + self._ttft_by_model[event.model].observe(ttft) | ||
| 295 | + # 时间线 | ||
| 296 | + self._timeline.append({ | ||
| 297 | + "ts": event.timestamp, | ||
| 298 | + "type": "stream_success", | ||
| 299 | + "model": event.model, | ||
| 300 | + "deployment": event.deployment_id, | ||
| 301 | + "latency": event.latency, | ||
| 302 | + "chunks": event.chunk_count, | ||
| 303 | + "ttft": ttft, | ||
| 304 | + }) | ||
| 305 | + if self._prom: | ||
| 306 | + self._prom["requests_total"].labels(model=event.model, status="success").inc() | ||
| 307 | + if event.latency is not None: | ||
| 308 | + self._prom["latency"].labels(model=event.model).observe(event.latency) | ||
| 309 | + | ||
| 310 | + def get_stats(self) -> Dict[str, Any]: | ||
| 311 | + """返回当前所有指标快照""" | ||
| 312 | + # 计算 QPS(最近 60s) | ||
| 313 | + now = time.time() | ||
| 314 | + cutoff = now - 60 | ||
| 315 | + while self._request_timestamps and self._request_timestamps[0] < cutoff: | ||
| 316 | + self._request_timestamps.popleft() | ||
| 317 | + qps = len(self._request_timestamps) / 60.0 | ||
| 318 | + | ||
| 319 | + return { | ||
| 320 | + "total_requests": self._request_count, | ||
| 321 | + "successful": self._success_count, | ||
| 322 | + "failed": self._failure_count, | ||
| 323 | + "retries": self._retry_count, | ||
| 324 | + "exhausted": self._exhausted_count, | ||
| 325 | + "streams": self._stream_count, | ||
| 326 | + "total_chunks": self._total_chunks, | ||
| 327 | + "qps": round(qps, 2), | ||
| 328 | + "tokens_per_sec": self._tokens_per_sec.to_dict(), | ||
| 329 | + "latency": self._latency.to_dict(), | ||
| 330 | + "ttft": self._ttft.to_dict(), | ||
| 331 | + "tokens": { | ||
| 332 | + "prompt": self._total_prompt_tokens, | ||
| 333 | + "completion": self._total_completion_tokens, | ||
| 334 | + "total": self._total_prompt_tokens + self._total_completion_tokens, | ||
| 335 | + }, | ||
| 336 | + "by_model": { | ||
| 337 | + model: { | ||
| 338 | + "requests": self._model_requests[model], | ||
| 339 | + "successes": self._model_successes.get(model, 0), | ||
| 340 | + "failures": self._model_failures.get(model, 0), | ||
| 341 | + "latency": self._latency_by_model[model].to_dict() | ||
| 342 | + if model in self._latency_by_model else None, | ||
| 343 | + "ttft": self._ttft_by_model[model].to_dict() | ||
| 344 | + if model in self._ttft_by_model else None, | ||
| 345 | + } | ||
| 346 | + for model in self._model_requests | ||
| 347 | + }, | ||
| 348 | + "by_deployment": { | ||
| 349 | + dep_id: { | ||
| 350 | + "provider": self._deployment_provider.get(dep_id, ""), | ||
| 351 | + "requests": self._deployment_requests[dep_id], | ||
| 352 | + "successes": self._deployment_successes.get(dep_id, 0), | ||
| 353 | + "failures": self._deployment_failures.get(dep_id, 0), | ||
| 354 | + "latency": self._deployment_latency[dep_id].to_dict() | ||
| 355 | + if dep_id in self._deployment_latency else None, | ||
| 356 | + "tokens": self._deployment_tokens.get(dep_id, 0), | ||
| 357 | + } | ||
| 358 | + for dep_id in self._deployment_requests | ||
| 359 | + }, | ||
| 360 | + "errors_by_type": dict(self._errors_by_type), | ||
| 361 | + "timeline": list(self._timeline), | ||
| 362 | + } | ||
| 363 | + | ||
| 364 | + def expose_prometheus(self, port: int = 9090, addr: str = "0.0.0.0") -> None: | ||
| 365 | + """启动 Prometheus HTTP 服务,暴露指标供 Grafana 等 scrape | ||
| 366 | + | ||
| 367 | + Args: | ||
| 368 | + port: HTTP 服务端口,默认 9090 | ||
| 369 | + addr: 绑定地址,默认 0.0.0.0 | ||
| 370 | + | ||
| 371 | + 用法: | ||
| 372 | + collector = MetricsCollector(enable_prometheus=True) | ||
| 373 | + collector.expose_prometheus(port=9090) | ||
| 374 | + # 然后 curl http://localhost:9090/metrics | ||
| 375 | + """ | ||
| 376 | + if not self._enable_prometheus: | ||
| 377 | + self._init_prometheus() | ||
| 378 | + self._enable_prometheus = True | ||
| 379 | + | ||
| 380 | + try: | ||
| 381 | + from prometheus_client import start_http_server | ||
| 382 | + except ImportError: | ||
| 383 | + raise ImportError( | ||
| 384 | + "prometheus-client is required for Prometheus metrics. " | ||
| 385 | + "Install with: pip install intelli-router[metrics]" | ||
| 386 | + ) | ||
| 387 | + | ||
| 388 | + start_http_server(port, addr=addr) | ||
| 389 | + | ||
| 390 | + def reset(self) -> None: | ||
| 391 | + """重置所有内存指标""" | ||
| 392 | + self._request_count = 0 | ||
| 393 | + self._success_count = 0 | ||
| 394 | + self._failure_count = 0 | ||
| 395 | + self._retry_count = 0 | ||
| 396 | + self._exhausted_count = 0 | ||
| 397 | + self._model_requests.clear() | ||
| 398 | + self._model_successes.clear() | ||
| 399 | + self._model_failures.clear() | ||
| 400 | + self._deployment_requests.clear() | ||
| 401 | + self._deployment_successes.clear() | ||
| 402 | + self._deployment_failures.clear() | ||
| 403 | + self._deployment_latency.clear() | ||
| 404 | + self._deployment_tokens.clear() | ||
| 405 | + self._deployment_provider.clear() | ||
| 406 | + self._errors_by_type.clear() | ||
| 407 | + self._latency = LatencyStats() | ||
| 408 | + self._latency_by_model.clear() | ||
| 409 | + self._total_prompt_tokens = 0 | ||
| 410 | + self._total_completion_tokens = 0 | ||
| 411 | + self._stream_count = 0 | ||
| 412 | + self._total_chunks = 0 | ||
| 413 | + self._ttft = LatencyStats() | ||
| 414 | + self._ttft_by_model.clear() | ||
| 415 | + self._tokens_per_sec = LatencyStats() | ||
| 416 | + self._request_timestamps.clear() | ||
| 417 | + self._timeline.clear() | ||
| @@ -0,0 +1,363 @@ | |||
| 1 | +"""轻量 Web 指标看板 - 基于 stdlib http.server""" | ||
| 2 | +import json | ||
| 3 | +import threading | ||
| 4 | +from http.server import HTTPServer, BaseHTTPRequestHandler | ||
| 5 | +from typing import TYPE_CHECKING | ||
| 6 | + | ||
| 7 | +if TYPE_CHECKING: | ||
| 8 | + from .metrics import MetricsCollector | ||
| 9 | + | ||
| 10 | +_DASHBOARD_HTML = """\ | ||
| 11 | +<!DOCTYPE html> | ||
| 12 | +<html lang="zh"> | ||
| 13 | +<head> | ||
| 14 | +<meta charset="UTF-8"> | ||
| 15 | +<meta name="viewport" content="width=device-width, initial-scale=1.0"> | ||
| 16 | +<title>IntelliRouter Metrics</title> | ||
| 17 | +<script src="https://cdn.jsdelivr.net/npm/chart.js@4"></script> | ||
| 18 | +<style> | ||
| 19 | +* { margin: 0; padding: 0; box-sizing: border-box; } | ||
| 20 | +body { | ||
| 21 | + font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Inter, sans-serif; | ||
| 22 | + background: #f8f9fa; color: #495057; padding: 32px 40px; | ||
| 23 | + line-height: 1.5; -webkit-font-smoothing: antialiased; | ||
| 24 | +} | ||
| 25 | +.header { | ||
| 26 | + display: flex; align-items: center; justify-content: space-between; | ||
| 27 | + margin-bottom: 32px; | ||
| 28 | +} | ||
| 29 | +.header h1 { | ||
| 30 | + font-size: 20px; font-weight: 600; color: #1a1a1a; | ||
| 31 | + letter-spacing: -0.3px; | ||
| 32 | +} | ||
| 33 | +.status { | ||
| 34 | + display: flex; align-items: center; gap: 8px; | ||
| 35 | + font-size: 12px; color: #868e96; | ||
| 36 | +} | ||
| 37 | +.status .dot { | ||
| 38 | + width: 7px; height: 7px; border-radius: 50%; | ||
| 39 | + background: #22c55e; box-shadow: 0 0 6px rgba(34,197,94,0.3); | ||
| 40 | + animation: pulse 2s infinite; | ||
| 41 | +} | ||
| 42 | +@keyframes pulse { | ||
| 43 | + 0%, 100% { opacity: 1; } | ||
| 44 | + 50% { opacity: 0.5; } | ||
| 45 | +} | ||
| 46 | +.cards { | ||
| 47 | + display: grid; grid-template-columns: repeat(auto-fit, minmax(170px, 1fr)); | ||
| 48 | + gap: 16px; margin-bottom: 32px; | ||
| 49 | +} | ||
| 50 | +.card { | ||
| 51 | + background: #ffffff; border: 1px solid #e9ecef; border-radius: 12px; | ||
| 52 | + padding: 20px; | ||
| 53 | +} | ||
| 54 | +.card .value { | ||
| 55 | + font-size: 26px; font-weight: 700; color: #1a1a1a; | ||
| 56 | + letter-spacing: -0.5px; | ||
| 57 | +} | ||
| 58 | +.card .label { | ||
| 59 | + font-size: 11px; color: #868e96; margin-top: 6px; | ||
| 60 | + text-transform: uppercase; letter-spacing: 0.5px; | ||
| 61 | +} | ||
| 62 | +.card .sub { | ||
| 63 | + font-size: 11px; color: #868e96; margin-top: 4px; | ||
| 64 | +} | ||
| 65 | +.charts { | ||
| 66 | + display: grid; grid-template-columns: 2fr 1fr; gap: 16px; margin-bottom: 32px; | ||
| 67 | +} | ||
| 68 | +.chart-box { | ||
| 69 | + background: #ffffff; border: 1px solid #e9ecef; | ||
| 70 | + border-radius: 12px; padding: 20px; | ||
| 71 | +} | ||
| 72 | +.section { | ||
| 73 | + background: #ffffff; border: 1px solid #e9ecef; | ||
| 74 | + border-radius: 12px; padding: 20px; margin-bottom: 24px; | ||
| 75 | +} | ||
| 76 | +.section h2 { | ||
| 77 | + font-size: 13px; font-weight: 500; color: #1a1a1a; | ||
| 78 | + margin-bottom: 16px; letter-spacing: -0.2px; | ||
| 79 | +} | ||
| 80 | +table { width: 100%; border-collapse: collapse; } | ||
| 81 | +th, td { | ||
| 82 | + padding: 10px 12px; text-align: left; font-size: 13px; | ||
| 83 | + border-bottom: 1px solid #e9ecef; | ||
| 84 | +} | ||
| 85 | +th { color: #495057; font-weight: 500; font-size: 11px; | ||
| 86 | + text-transform: uppercase; letter-spacing: 0.3px; } | ||
| 87 | +td { color: #495057; } | ||
| 88 | +tbody tr { transition: background 0.15s; } | ||
| 89 | +tbody tr:hover { background: #f1f3f5; } | ||
| 90 | +tbody tr:last-child td { border-bottom: none; } | ||
| 91 | +.empty { color: #adb5bd; font-style: italic; } | ||
| 92 | +@media (max-width: 768px) { | ||
| 93 | + body { padding: 20px 16px; } | ||
| 94 | + .charts { grid-template-columns: 1fr; } | ||
| 95 | +} | ||
| 96 | +</style> | ||
| 97 | +</head> | ||
| 98 | +<body> | ||
| 99 | +<div class="header"> | ||
| 100 | + <h1>IntelliRouter Metrics</h1> | ||
| 101 | + <div class="status"><span class="dot"></span><span id="status-text">Connecting...</span></div> | ||
| 102 | +</div> | ||
| 103 | +<div class="cards"> | ||
| 104 | + <div class="card"><div class="value" id="total-req">-</div><div class="label">Requests</div></div> | ||
| 105 | + <div class="card"><div class="value" id="qps">-</div><div class="label">QPS (60s)</div></div> | ||
| 106 | + <div class="card"><div class="value" id="success-rate">-</div><div class="label">Success Rate</div></div> | ||
| 107 | + <div class="card"><div class="value" id="avg-latency">-</div><div class="label">Avg Latency</div><div class="sub" id="latency-pct"></div></div> | ||
| 108 | + <div class="card"><div class="value" id="tokens-per-sec">-</div><div class="label">Tokens/s</div></div> | ||
| 109 | + <div class="card"><div class="value" id="total-tokens">-</div><div class="label">Tokens</div></div> | ||
| 110 | + <div class="card"><div class="value" id="ttft">-</div><div class="label">Avg TTFT</div></div> | ||
| 111 | +</div> | ||
| 112 | +<div class="charts"> | ||
| 113 | + <div class="chart-box"><canvas id="deployChart"></canvas></div> | ||
| 114 | + <div class="chart-box"><canvas id="outcomeChart"></canvas></div> | ||
| 115 | +</div> | ||
| 116 | +<div class="section"> | ||
| 117 | + <h2>Latency Timeline</h2> | ||
| 118 | + <div style="height:200px"><canvas id="timelineChart"></canvas></div> | ||
| 119 | +</div> | ||
| 120 | +<div class="section"> | ||
| 121 | + <h2>Deployments</h2> | ||
| 122 | + <table><thead><tr><th>Deployment</th><th>Provider</th><th>Requests</th><th>Success</th><th>Failures</th><th>Avg Latency</th><th>Tokens</th></tr></thead> | ||
| 123 | + <tbody id="deploy-table"></tbody></table> | ||
| 124 | +</div> | ||
| 125 | +<div class="section"> | ||
| 126 | + <h2>Models</h2> | ||
| 127 | + <table><thead><tr><th>Model</th><th>Requests</th><th>Success</th><th>Failures</th><th>Avg Latency</th><th>TTFT</th></tr></thead> | ||
| 128 | + <tbody id="model-table"></tbody></table> | ||
| 129 | +</div> | ||
| 130 | +<div class="section"> | ||
| 131 | + <h2>Errors</h2> | ||
| 132 | + <table><thead><tr><th>Type</th><th>Count</th></tr></thead> | ||
| 133 | + <tbody id="error-table"></tbody></table> | ||
| 134 | +</div> | ||
| 135 | + | ||
| 136 | +<script> | ||
| 137 | +let deployChart, outcomeChart, timelineChart; | ||
| 138 | +const ACCENT = '#6366f1'; | ||
| 139 | +const ACCENT_BG = 'rgba(99,102,241,0.08)'; | ||
| 140 | +const GRID_COLOR = '#e9ecef'; | ||
| 141 | +const TICK_COLOR = '#868e96'; | ||
| 142 | + | ||
| 143 | +function initCharts() { | ||
| 144 | + Chart.defaults.color = TICK_COLOR; | ||
| 145 | + Chart.defaults.borderColor = GRID_COLOR; | ||
| 146 | + | ||
| 147 | + const dCtx = document.getElementById('deployChart').getContext('2d'); | ||
| 148 | + deployChart = new Chart(dCtx, { | ||
| 149 | + type: 'bar', | ||
| 150 | + data: { labels: [], datasets: [{ label: 'Requests', data: [], backgroundColor: ACCENT, borderRadius: 4 }] }, | ||
| 151 | + options: { responsive: true, plugins: { title: { display: true, text: 'Requests by Deployment', color: '#1a1a1a', font: { size: 13, weight: '500' } }, | ||
| 152 | + legend: { display: false } }, scales: { x: { grid: { display: false } }, y: { beginAtZero: true, grid: { color: GRID_COLOR } } } } | ||
| 153 | + }); | ||
| 154 | + const oCtx = document.getElementById('outcomeChart').getContext('2d'); | ||
| 155 | + outcomeChart = new Chart(oCtx, { | ||
| 156 | + type: 'doughnut', | ||
| 157 | + data: { labels: ['Success', 'Failed', 'Retries'], datasets: [{ data: [0,0,0], | ||
| 158 | + backgroundColor: ['#22c55e', '#ef4444', '#f59e0b'], borderWidth: 0 }] }, | ||
| 159 | + options: { responsive: true, cutout: '70%', plugins: { title: { display: true, text: 'Outcomes', color: '#1a1a1a', font: { size: 13, weight: '500' } } } } | ||
| 160 | + }); | ||
| 161 | + const tCtx = document.getElementById('timelineChart').getContext('2d'); | ||
| 162 | + timelineChart = new Chart(tCtx, { | ||
| 163 | + type: 'line', | ||
| 164 | + data: { labels: [], datasets: [{ label: 'Latency (s)', data: [], borderColor: ACCENT, | ||
| 165 | + backgroundColor: ACCENT_BG, fill: true, tension: 0.4, pointRadius: 0, borderWidth: 2 }] }, | ||
| 166 | + options: { responsive: true, maintainAspectRatio: false, | ||
| 167 | + interaction: { intersect: false, mode: 'index' }, | ||
| 168 | + plugins: { legend: { display: false } }, | ||
| 169 | + scales: { x: { grid: { display: false }, ticks: { maxTicksLimit: 10 } }, | ||
| 170 | + y: { beginAtZero: true, grid: { color: GRID_COLOR }, | ||
| 171 | + title: { display: true, text: 'seconds', color: '#52525b', font: { size: 11 } } } } } | ||
| 172 | + }); | ||
| 173 | +} | ||
| 174 | + | ||
| 175 | +function formatTokens(n) { | ||
| 176 | + if (n >= 1000000) return (n/1000000).toFixed(1) + 'M'; | ||
| 177 | + if (n >= 1000) return (n/1000).toFixed(1) + 'K'; | ||
| 178 | + return n.toString(); | ||
| 179 | +} | ||
| 180 | + | ||
| 181 | +function update(data) { | ||
| 182 | + const total = data.total_requests || 0; | ||
| 183 | + const successRate = total > 0 ? ((data.successful / total) * 100).toFixed(1) : '0.0'; | ||
| 184 | + document.getElementById('total-req').textContent = total; | ||
| 185 | + document.getElementById('qps').textContent = data.qps !== undefined ? data.qps.toFixed(1) : '-'; | ||
| 186 | + document.getElementById('success-rate').textContent = successRate + '%'; | ||
| 187 | + document.getElementById('avg-latency').textContent = data.latency.count > 0 ? data.latency.avg.toFixed(3) + 's' : '-'; | ||
| 188 | + const lp = data.latency; | ||
| 189 | + document.getElementById('latency-pct').textContent = lp.p50 != null ? `P50 ${lp.p50.toFixed(3)}s · P95 ${lp.p95.toFixed(3)}s · P99 ${lp.p99.toFixed(3)}s` : ''; | ||
| 190 | + const tps = data.tokens_per_sec; | ||
| 191 | + document.getElementById('tokens-per-sec').textContent = tps && tps.count > 0 ? tps.avg.toFixed(0) : '-'; | ||
| 192 | + document.getElementById('total-tokens').textContent = formatTokens(data.tokens.total); | ||
| 193 | + document.getElementById('ttft').textContent = data.ttft && data.ttft.count > 0 ? data.ttft.avg.toFixed(3) + 's' : '-'; | ||
| 194 | + document.getElementById('status-text').textContent = 'Live · ' + new Date().toLocaleTimeString(); | ||
| 195 | + | ||
| 196 | + const depLabels = Object.keys(data.by_deployment); | ||
| 197 | + const depValues = depLabels.map(k => data.by_deployment[k].requests || 0); | ||
| 198 | + deployChart.data.labels = depLabels; | ||
| 199 | + deployChart.data.datasets[0].data = depValues; | ||
| 200 | + deployChart.update(); | ||
| 201 | + | ||
| 202 | + outcomeChart.data.datasets[0].data = [data.successful || 0, data.failed || 0, data.retries || 0]; | ||
| 203 | + outcomeChart.update(); | ||
| 204 | + | ||
| 205 | + const timeline = data.timeline || []; | ||
| 206 | + const recentWithLatency = timeline.filter(e => e.latency != null).slice(-50); | ||
| 207 | + timelineChart.data.labels = recentWithLatency.map(e => { | ||
| 208 | + const d = new Date(e.ts * 1000); return d.toLocaleTimeString(); | ||
| 209 | + }); | ||
| 210 | + timelineChart.data.datasets[0].data = recentWithLatency.map(e => e.latency); | ||
| 211 | + timelineChart.update(); | ||
| 212 | + | ||
| 213 | + const dt = document.getElementById('deploy-table'); | ||
| 214 | + dt.innerHTML = ''; | ||
| 215 | + for (const [dep, stats] of Object.entries(data.by_deployment || {})) { | ||
| 216 | + const lat = stats.latency && stats.latency.count > 0 ? stats.latency.avg.toFixed(3) + 's' : '-'; | ||
| 217 | + dt.innerHTML += `<tr><td>${dep}</td><td>${stats.provider || '-'}</td><td>${stats.requests}</td><td>${stats.successes}</td><td>${stats.failures}</td><td>${lat}</td><td>${formatTokens(stats.tokens || 0)}</td></tr>`; | ||
| 218 | + } | ||
| 219 | + if (!depLabels.length) dt.innerHTML = '<tr><td colspan="7" class="empty">No data yet</td></tr>'; | ||
| 220 | + | ||
| 221 | + const mt = document.getElementById('model-table'); | ||
| 222 | + mt.innerHTML = ''; | ||
| 223 | + for (const [model, stats] of Object.entries(data.by_model || {})) { | ||
| 224 | + const lat = stats.latency && stats.latency.count > 0 ? stats.latency.avg.toFixed(3) + 's' : '-'; | ||
| 225 | + const ttft = stats.ttft && stats.ttft.count > 0 ? stats.ttft.avg.toFixed(3) + 's' : '-'; | ||
| 226 | + mt.innerHTML += `<tr><td>${model}</td><td>${stats.requests}</td><td>${stats.successes}</td><td>${stats.failures}</td><td>${lat}</td><td>${ttft}</td></tr>`; | ||
| 227 | + } | ||
| 228 | + if (!Object.keys(data.by_model || {}).length) mt.innerHTML = '<tr><td colspan="6" class="empty">No data yet</td></tr>'; | ||
| 229 | + | ||
| 230 | + const et = document.getElementById('error-table'); | ||
| 231 | + et.innerHTML = ''; | ||
| 232 | + for (const [err, count] of Object.entries(data.errors_by_type || {})) { | ||
| 233 | + et.innerHTML += `<tr><td>${err}</td><td>${count}</td></tr>`; | ||
| 234 | + } | ||
| 235 | + if (!Object.keys(data.errors_by_type || {}).length) et.innerHTML = '<tr><td colspan="2" class="empty">No errors</td></tr>'; | ||
| 236 | +} | ||
| 237 | + | ||
| 238 | +async function fetchStats() { | ||
| 239 | + try { | ||
| 240 | + const resp = await fetch('/api/stats'); | ||
| 241 | + const data = await resp.json(); | ||
| 242 | + update(data); | ||
| 243 | + } catch (e) { console.error('Fetch failed:', e); } | ||
| 244 | +} | ||
| 245 | + | ||
| 246 | +initCharts(); | ||
| 247 | +fetchStats(); | ||
| 248 | +setInterval(fetchStats, 2000); | ||
| 249 | +</script> | ||
| 250 | +</body> | ||
| 251 | +</html> | ||
| 252 | +""" | ||
| 253 | + | ||
| 254 | + | ||
| 255 | +_DASHBOARD_HTML_BYTES = _DASHBOARD_HTML.encode("utf-8") | ||
| 256 | + | ||
| 257 | + | ||
| 258 | +class MetricsWebServer: | ||
| 259 | + """轻量 Web 指标看板 | ||
| 260 | + | ||
| 261 | + 基于 Python stdlib http.server,提供一个实时刷新的 HTML 看板页面。 | ||
| 262 | + 零额外依赖,图表使用浏览器端 Chart.js CDN。 | ||
| 263 | + | ||
| 264 | + 用法: | ||
| 265 | + from intelli_router.observability import MetricsCollector, MetricsWebServer | ||
| 266 | + | ||
| 267 | + metrics = MetricsCollector() | ||
| 268 | + bus.register(metrics) | ||
| 269 | + | ||
| 270 | + server = MetricsWebServer(metrics, port=8080) | ||
| 271 | + server.start() | ||
| 272 | + # 浏览器打开 http://localhost:8080 | ||
| 273 | + server.stop() | ||
| 274 | + """ | ||
| 275 | + | ||
| 276 | + def __init__( | ||
| 277 | + self, | ||
| 278 | + metrics: "MetricsCollector", | ||
| 279 | + port: int = 8080, | ||
| 280 | + addr: str = "0.0.0.0", | ||
| 281 | + ): | ||
| 282 | + self._metrics = metrics | ||
| 283 | + self._port = port | ||
| 284 | + self._addr = addr | ||
| 285 | + self._thread = None | ||
| 286 | + self._httpd = None | ||
| 287 | + | ||
| 288 | + @property | ||
| 289 | + def url(self) -> str: | ||
| 290 | + """看板访问地址""" | ||
| 291 | + host = "localhost" if self._addr == "0.0.0.0" else self._addr | ||
| 292 | + return f"http://{host}:{self._port}" | ||
| 293 | + | ||
| 294 | + def start(self) -> None: | ||
| 295 | + """启动 HTTP 服务(后台 daemon 线程)""" | ||
| 296 | + metrics = self._metrics | ||
| 297 | + | ||
| 298 | + class Handler(BaseHTTPRequestHandler): | ||
| 299 | + def do_GET(self): | ||
| 300 | + if self.path == "/" or self.path == "/index.html": | ||
| 301 | + self._serve_html() | ||
| 302 | + elif self.path == "/api/stats": | ||
| 303 | + self._serve_json() | ||
| 304 | + elif self.path == "/metrics": | ||
| 305 | + self._serve_metrics() | ||
| 306 | + else: | ||
| 307 | + self.send_error(404) | ||
| 308 | + | ||
| 309 | + def _serve_html(self): | ||
| 310 | + self.send_response(200) | ||
| 311 | + self.send_header("Content-Type", "text/html; charset=utf-8") | ||
| 312 | + self.end_headers() | ||
| 313 | + self.wfile.write(_DASHBOARD_HTML_BYTES) | ||
| 314 | + | ||
| 315 | + def _serve_json(self): | ||
| 316 | + data = json.dumps(metrics.get_stats(), ensure_ascii=False) | ||
| 317 | + self.send_response(200) | ||
| 318 | + self.send_header("Content-Type", "application/json; charset=utf-8") | ||
| 319 | + self.send_header("Access-Control-Allow-Origin", "*") | ||
| 320 | + self.end_headers() | ||
| 321 | + self.wfile.write(data.encode("utf-8")) | ||
| 322 | + | ||
| 323 | + def _serve_metrics(self): | ||
| 324 | + try: | ||
| 325 | + from prometheus_client import generate_latest, CONTENT_TYPE_LATEST | ||
| 326 | + output = generate_latest() | ||
| 327 | + self.send_response(200) | ||
| 328 | + self.send_header("Content-Type", CONTENT_TYPE_LATEST) | ||
| 329 | + self.end_headers() | ||
| 330 | + self.wfile.write(output) | ||
| 331 | + except ImportError: | ||
| 332 | + self.send_response(200) | ||
| 333 | + self.send_header("Content-Type", "text/plain; charset=utf-8") | ||
| 334 | + self.end_headers() | ||
| 335 | + # 简单文本格式输出 | ||
| 336 | + stats = metrics.get_stats() | ||
| 337 | + lines = [ | ||
| 338 | + f"# IntelliRouter Metrics (plain text, install prometheus-client for Prometheus format)", | ||
| 339 | + f"total_requests {stats['total_requests']}", | ||
| 340 | + f"successful {stats['successful']}", | ||
| 341 | + f"failed {stats['failed']}", | ||
| 342 | + f"retries {stats['retries']}", | ||
| 343 | + f"exhausted {stats['exhausted']}", | ||
| 344 | + f"avg_latency {stats['latency']['avg']}", | ||
| 345 | + f"tokens_total {stats['tokens']['total']}", | ||
| 346 | + ] | ||
| 347 | + self.wfile.write("\n".join(lines).encode("utf-8")) | ||
| 348 | + | ||
| 349 | + def log_message(self, format, *args): | ||
| 350 | + pass # 静默,不输出到 stderr | ||
| 351 | + | ||
| 352 | + self._httpd = HTTPServer((self._addr, self._port), Handler) | ||
| 353 | + self._thread = threading.Thread(target=self._httpd.serve_forever, daemon=True) | ||
| 354 | + self._thread.start() | ||
| 355 | + | ||
| 356 | + def stop(self) -> None: | ||
| 357 | + """停止 HTTP 服务""" | ||
| 358 | + if self._httpd: | ||
| 359 | + self._httpd.shutdown() | ||
| 360 | + self._httpd = None | ||
| 361 | + if self._thread: | ||
| 362 | + self._thread.join(timeout=5) | ||
| 363 | + self._thread = None | ||
| @@ -0,0 +1,10 @@ | |||
| 1 | +"""输出解析器。""" | ||
| 2 | +from .base import BaseOutputParser | ||
| 3 | +from .json_parser import JsonOutputParser | ||
| 4 | +from .markdown_parser import MarkdownOutputParser | ||
| 5 | + | ||
| 6 | +__all__ = [ | ||
| 7 | + "BaseOutputParser", | ||
| 8 | + "JsonOutputParser", | ||
| 9 | + "MarkdownOutputParser", | ||
| 10 | +] | ||
| @@ -0,0 +1,48 @@ | |||
| 1 | +"""输出解析器基类。""" | ||
| 2 | +from typing import Any, AsyncIterator, Optional, Union | ||
| 3 | +from abc import ABC, abstractmethod | ||
| 4 | + | ||
| 5 | +from ..types import AssistantMessage, AssistantMessageChunk | ||
| 6 | + | ||
| 7 | + | ||
| 8 | +class BaseOutputParser(ABC): | ||
| 9 | + """LLM 输出解析器基类。 | ||
| 10 | + | ||
| 11 | + parse() — 解析非流式响应(AssistantMessage 或 content 字符串) | ||
| 12 | + stream_parse() — 解析流式响应(逐 chunk 增量解析) | ||
| 13 | + """ | ||
| 14 | + | ||
| 15 | + @staticmethod | ||
| 16 | + def _extract_content(llm_output: Union[str, AssistantMessage]) -> Optional[str]: | ||
| 17 | + """从 AssistantMessage 或字符串中提取纯文本内容。""" | ||
| 18 | + if isinstance(llm_output, AssistantMessage): | ||
| 19 | + return llm_output.content or None | ||
| 20 | + if isinstance(llm_output, str): | ||
| 21 | + return llm_output | ||
| 22 | + return None | ||
| 23 | + | ||
| 24 | + @abstractmethod | ||
| 25 | + async def parse(self, inputs: Any) -> Any: | ||
| 26 | + """解析 LLM 输出。 | ||
| 27 | + | ||
| 28 | + Args: | ||
| 29 | + inputs: AssistantMessage 或 content 字符串 | ||
| 30 | + | ||
| 31 | + Returns: | ||
| 32 | + 解析后的结构化数据 | ||
| 33 | + """ | ||
| 34 | + raise NotImplementedError() | ||
| 35 | + | ||
| 36 | + @abstractmethod | ||
| 37 | + async def stream_parse( | ||
| 38 | + self, streaming_inputs: AsyncIterator | ||
| 39 | + ) -> AsyncIterator[Any]: | ||
| 40 | + """解析流式 LLM 输出。 | ||
| 41 | + | ||
| 42 | + Args: | ||
| 43 | + streaming_inputs: AsyncIterator[AssistantMessageChunk] | ||
| 44 | + | ||
| 45 | + Yields: | ||
| 46 | + 逐段解析的结构化数据 | ||
| 47 | + """ | ||
| 48 | + raise NotImplementedError() | ||
| @@ -0,0 +1,100 @@ | |||
| 1 | +"""JSON 输出解析器。 | ||
| 2 | + | ||
| 3 | +从 LLM 响应中提取 JSON 数据,支持: | ||
| 4 | +- 裸 JSON(如 {"key": "value"}) | ||
| 5 | +- ```json 代码块包裹的 JSON | ||
| 6 | +- 流式增量解析 | ||
| 7 | +""" | ||
| 8 | +import json | ||
| 9 | +import re | ||
| 10 | +from typing import Any, AsyncIterator, Optional, Union | ||
| 11 | + | ||
| 12 | +from .base import BaseOutputParser | ||
| 13 | +from ..types import AssistantMessage, AssistantMessageChunk | ||
| 14 | + | ||
| 15 | + | ||
| 16 | +# Regex to extract content from ```json ... ``` code blocks | ||
| 17 | +_JSON_CODE_BLOCK_RE = re.compile(r"```json\n(.*?)```", re.DOTALL) | ||
| 18 | + | ||
| 19 | + | ||
| 20 | +class JsonOutputParser(BaseOutputParser): | ||
| 21 | + """JSON 输出解析器。""" | ||
| 22 | + | ||
| 23 | + async def parse(self, llm_output: Union[str, AssistantMessage]) -> Any: | ||
| 24 | + """解析 LLM 输出中的 JSON。 | ||
| 25 | + | ||
| 26 | + Args: | ||
| 27 | + llm_output: AssistantMessage 或 content 字符串 | ||
| 28 | + | ||
| 29 | + Returns: | ||
| 30 | + 解析后的 JSON 数据,解析失败返回 None | ||
| 31 | + """ | ||
| 32 | + text = self._extract_content(llm_output) | ||
| 33 | + if not text: | ||
| 34 | + return None | ||
| 35 | + | ||
| 36 | + # 尝试提取 ```json 代码块 | ||
| 37 | + match = _JSON_CODE_BLOCK_RE.search(text) | ||
| 38 | + if match: | ||
| 39 | + json_str = match.group(1).strip() | ||
| 40 | + else: | ||
| 41 | + json_str = text.strip() | ||
| 42 | + | ||
| 43 | + try: | ||
| 44 | + return json.loads(json_str) | ||
| 45 | + except json.JSONDecodeError: | ||
| 46 | + return None | ||
| 47 | + | ||
| 48 | + async def stream_parse( | ||
| 49 | + self, streaming_inputs: AsyncIterator | ||
| 50 | + ) -> AsyncIterator[Optional[dict]]: | ||
| 51 | + """流式解析 JSON。 | ||
| 52 | + | ||
| 53 | + 维护一个 buffer,每收到一个 chunk 尝试解析。 | ||
| 54 | + 解析成功后 yield 结果并清空 buffer。 | ||
| 55 | + | ||
| 56 | + Args: | ||
| 57 | + streaming_inputs: AsyncIterator[AssistantMessageChunk] | ||
| 58 | + | ||
| 59 | + Yields: | ||
| 60 | + 解析出的 dict,或 None | ||
| 61 | + """ | ||
| 62 | + buffer = "" | ||
| 63 | + | ||
| 64 | + async for chunk in streaming_inputs: | ||
| 65 | + if isinstance(chunk, AssistantMessageChunk) and chunk.content: | ||
| 66 | + buffer += chunk.content | ||
| 67 | + | ||
| 68 | + # 尝试解析当前 buffer | ||
| 69 | + match = _JSON_CODE_BLOCK_RE.search(buffer) | ||
| 70 | + if match: | ||
| 71 | + json_str = match.group(1).strip() | ||
| 72 | + try: | ||
| 73 | + parsed = json.loads(json_str) | ||
| 74 | + yield parsed | ||
| 75 | + buffer = buffer[match.end():].strip() | ||
| 76 | + continue | ||
| 77 | + except json.JSONDecodeError: | ||
| 78 | + pass | ||
| 79 | + | ||
| 80 | + # 也尝试解析裸 JSON | ||
| 81 | + stripped = buffer.strip() | ||
| 82 | + if stripped.startswith("{") and stripped.endswith("}"): | ||
| 83 | + try: | ||
| 84 | + parsed = json.loads(stripped) | ||
| 85 | + yield parsed | ||
| 86 | + buffer = "" | ||
| 87 | + except json.JSONDecodeError: | ||
| 88 | + pass | ||
| 89 | + | ||
| 90 | + # 处理剩余的 buffer | ||
| 91 | + if buffer.strip(): | ||
| 92 | + match = _JSON_CODE_BLOCK_RE.search(buffer) | ||
| 93 | + if match: | ||
| 94 | + json_str = match.group(1).strip() | ||
| 95 | + else: | ||
| 96 | + json_str = buffer.strip() | ||
| 97 | + try: | ||
| 98 | + yield json.loads(json_str) | ||
| 99 | + except json.JSONDecodeError: | ||
| 100 | + pass | ||
| @@ -0,0 +1,253 @@ | |||
| 1 | +"""Markdown 输出解析器。 | ||
| 2 | + | ||
| 3 | +将 LLM 输出的 Markdown 文本解析为结构化 MarkdownContent, | ||
| 4 | +包含 headers、code_blocks、links、images、tables、lists 等元素。 | ||
| 5 | +""" | ||
| 6 | +import re | ||
| 7 | +from dataclasses import dataclass, field | ||
| 8 | +from typing import Any, AsyncIterator, Dict, List, Optional, Union | ||
| 9 | + | ||
| 10 | +from .base import BaseOutputParser | ||
| 11 | +from ..types import AssistantMessage, AssistantMessageChunk | ||
| 12 | + | ||
| 13 | + | ||
| 14 | +class MarkdownElementType: | ||
| 15 | + """Markdown 元素类型常量。""" | ||
| 16 | + HEADER = "header" | ||
| 17 | + CODE_BLOCK = "code_block" | ||
| 18 | + INLINE_CODE = "inline_code" | ||
| 19 | + LINK = "link" | ||
| 20 | + IMAGE = "image" | ||
| 21 | + TABLE = "table" | ||
| 22 | + LIST = "list" | ||
| 23 | + TEXT = "text" | ||
| 24 | + | ||
| 25 | + | ||
| 26 | +@dataclass | ||
| 27 | +class MarkdownElement: | ||
| 28 | + """单个 Markdown 元素。""" | ||
| 29 | + type: str | ||
| 30 | + content: Dict[str, Any] | ||
| 31 | + start_pos: int | ||
| 32 | + end_pos: int | ||
| 33 | + raw: str | ||
| 34 | + | ||
| 35 | + | ||
| 36 | +@dataclass | ||
| 37 | +class MarkdownContent: | ||
| 38 | + """结构化 Markdown 内容。""" | ||
| 39 | + raw_content: str = "" | ||
| 40 | + elements: List[MarkdownElement] = field(default_factory=list) | ||
| 41 | + headers: List[Dict[str, str]] = field(default_factory=list) | ||
| 42 | + code_blocks: List[Dict[str, str]] = field(default_factory=list) | ||
| 43 | + links: List[Dict[str, str]] = field(default_factory=list) | ||
| 44 | + images: List[Dict[str, str]] = field(default_factory=list) | ||
| 45 | + tables: List[str] = field(default_factory=list) | ||
| 46 | + lists: List[str] = field(default_factory=list) | ||
| 47 | + | ||
| 48 | + | ||
| 49 | +class MarkdownOutputParser(BaseOutputParser): | ||
| 50 | + """Markdown 输出解析器。""" | ||
| 51 | + | ||
| 52 | + async def parse(self, llm_output: Union[str, AssistantMessage]) -> Optional[MarkdownContent]: | ||
| 53 | + """解析 Markdown 文本为结构化 MarkdownContent。 | ||
| 54 | + | ||
| 55 | + Args: | ||
| 56 | + llm_output: AssistantMessage 或 content 字符串 | ||
| 57 | + | ||
| 58 | + Returns: | ||
| 59 | + MarkdownContent 或 None(解析失败时) | ||
| 60 | + """ | ||
| 61 | + text = self._extract_content(llm_output) | ||
| 62 | + if not text: | ||
| 63 | + return None | ||
| 64 | + | ||
| 65 | + try: | ||
| 66 | + md = MarkdownContent(raw_content=text) | ||
| 67 | + self._extract_all_elements(text, md) | ||
| 68 | + self._populate_categorized_lists(md) | ||
| 69 | + return md | ||
| 70 | + except Exception: | ||
| 71 | + return None | ||
| 72 | + | ||
| 73 | + async def stream_parse( | ||
| 74 | + self, streaming_inputs: AsyncIterator | ||
| 75 | + ) -> AsyncIterator[Optional[MarkdownContent]]: | ||
| 76 | + """流式解析 Markdown。 | ||
| 77 | + | ||
| 78 | + Batches chunks and only reparses at intervals to avoid O(n²) behavior. | ||
| 79 | + | ||
| 80 | + Args: | ||
| 81 | + streaming_inputs: AsyncIterator[AssistantMessageChunk] | ||
| 82 | + | ||
| 83 | + Yields: | ||
| 84 | + MarkdownContent(增量更新) | ||
| 85 | + """ | ||
| 86 | + buffer = "" | ||
| 87 | + last_parsed_length = 0 | ||
| 88 | + chunk_count = 0 | ||
| 89 | + # Reparse every N chunks to amortize cost | ||
| 90 | + REPARSE_INTERVAL = 10 | ||
| 91 | + | ||
| 92 | + async for chunk in streaming_inputs: | ||
| 93 | + if isinstance(chunk, AssistantMessageChunk) and chunk.content: | ||
| 94 | + buffer += chunk.content | ||
| 95 | + chunk_count += 1 | ||
| 96 | + | ||
| 97 | + if len(buffer) > last_parsed_length and chunk_count >= REPARSE_INTERVAL: | ||
| 98 | + try: | ||
| 99 | + md = MarkdownContent(raw_content=buffer) | ||
| 100 | + self._extract_all_elements(buffer, md) | ||
| 101 | + self._populate_categorized_lists(md) | ||
| 102 | + yield md | ||
| 103 | + last_parsed_length = len(buffer) | ||
| 104 | + chunk_count = 0 | ||
| 105 | + except Exception: | ||
| 106 | + continue | ||
| 107 | + | ||
| 108 | + # Final parse of complete buffer | ||
| 109 | + if len(buffer) > last_parsed_length and buffer.strip(): | ||
| 110 | + try: | ||
| 111 | + md = MarkdownContent(raw_content=buffer) | ||
| 112 | + self._extract_all_elements(buffer, md) | ||
| 113 | + self._populate_categorized_lists(md) | ||
| 114 | + yield md | ||
| 115 | + except Exception: | ||
| 116 | + pass | ||
| 117 | + | ||
| 118 | + # ------------------------------------------------------------------ | ||
| 119 | + # 内部解析方法 | ||
| 120 | + # ------------------------------------------------------------------ | ||
| 121 | + | ||
| 122 | + def _extract_all_elements(self, text: str, md: MarkdownContent) -> None: | ||
| 123 | + """提取所有 Markdown 元素并排序。""" | ||
| 124 | + elements = [] | ||
| 125 | + | ||
| 126 | + # Headers | ||
| 127 | + for m in re.finditer(r'^(#{1,6})\s+(.+)$', text, re.MULTILINE): | ||
| 128 | + elements.append(MarkdownElement( | ||
| 129 | + type=MarkdownElementType.HEADER, | ||
| 130 | + content={"level": str(len(m.group(1))), "title": m.group(2).strip()}, | ||
| 131 | + start_pos=m.start(), end_pos=m.end(), raw=m.group(0), | ||
| 132 | + )) | ||
| 133 | + | ||
| 134 | + # Code blocks | ||
| 135 | + for m in re.finditer(r'```(\w*)\n(.*?)\n```', text, re.DOTALL): | ||
| 136 | + elements.append(MarkdownElement( | ||
| 137 | + type=MarkdownElementType.CODE_BLOCK, | ||
| 138 | + content={"language": m.group(1) or "text", "code": m.group(2)}, | ||
| 139 | + start_pos=m.start(), end_pos=m.end(), raw=m.group(0), | ||
| 140 | + )) | ||
| 141 | + | ||
| 142 | + # Inline code | ||
| 143 | + for m in re.finditer(r'`([^`\n]+)`', text): | ||
| 144 | + elements.append(MarkdownElement( | ||
| 145 | + type=MarkdownElementType.INLINE_CODE, | ||
| 146 | + content={"code": m.group(1)}, | ||
| 147 | + start_pos=m.start(), end_pos=m.end(), raw=m.group(0), | ||
| 148 | + )) | ||
| 149 | + | ||
| 150 | + # Images | ||
| 151 | + for m in re.finditer(r'!\[([^\]]*)\]\(([^)]+)\)', text): | ||
| 152 | + elements.append(MarkdownElement( | ||
| 153 | + type=MarkdownElementType.IMAGE, | ||
| 154 | + content={"alt": m.group(1), "url": m.group(2)}, | ||
| 155 | + start_pos=m.start(), end_pos=m.end(), raw=m.group(0), | ||
| 156 | + )) | ||
| 157 | + | ||
| 158 | + # Links (not images) | ||
| 159 | + for m in re.finditer(r'(?<!\!)\[([^\]]+)\]\(([^)]+)\)', text): | ||
| 160 | + elements.append(MarkdownElement( | ||
| 161 | + type=MarkdownElementType.LINK, | ||
| 162 | + content={"text": m.group(1), "url": m.group(2)}, | ||
| 163 | + start_pos=m.start(), end_pos=m.end(), raw=m.group(0), | ||
| 164 | + )) | ||
| 165 | + | ||
| 166 | + # Tables and lists (multiline) | ||
| 167 | + self._extract_multiline_elements(text, elements) | ||
| 168 | + | ||
| 169 | + elements.sort(key=lambda x: x.start_pos) | ||
| 170 | + md.elements = elements | ||
| 171 | + | ||
| 172 | + def _extract_multiline_elements(self, text: str, elements: List) -> None: | ||
| 173 | + """提取多行元素(表格、列表)。""" | ||
| 174 | + lines = text.split("\n") | ||
| 175 | + current_pos = 0 | ||
| 176 | + | ||
| 177 | + table_lines = [] | ||
| 178 | + list_lines = [] | ||
| 179 | + table_start = -1 | ||
| 180 | + list_start = -1 | ||
| 181 | + | ||
| 182 | + for i, line in enumerate(lines): | ||
| 183 | + line_start = current_pos | ||
| 184 | + line_end = current_pos + len(line) | ||
| 185 | + current_pos = line_end + 1 | ||
| 186 | + | ||
| 187 | + # Table detection | ||
| 188 | + if "|" in line.strip() and line.strip(): | ||
| 189 | + if not table_lines: | ||
| 190 | + table_start = line_start | ||
| 191 | + table_lines.append(line) | ||
| 192 | + else: | ||
| 193 | + if table_lines: | ||
| 194 | + content = "\n".join(table_lines) | ||
| 195 | + elements.append(MarkdownElement( | ||
| 196 | + type=MarkdownElementType.TABLE, | ||
| 197 | + content={"table": content}, | ||
| 198 | + start_pos=table_start, end_pos=line_start - 1, raw=content, | ||
| 199 | + )) | ||
| 200 | + table_lines = [] | ||
| 201 | + | ||
| 202 | + # List detection | ||
| 203 | + if re.match(r'^\s*[-*+]\s+', line) or re.match(r'^\s*\d+\.\s+', line): | ||
| 204 | + if not list_lines: | ||
| 205 | + list_start = line_start | ||
| 206 | + list_lines.append(line) | ||
| 207 | + elif re.match(r'^\s*$', line) and list_lines: | ||
| 208 | + list_lines.append(line) | ||
| 209 | + else: | ||
| 210 | + if list_lines: | ||
| 211 | + content = "\n".join(list_lines).strip() | ||
| 212 | + if content: | ||
| 213 | + elements.append(MarkdownElement( | ||
| 214 | + type=MarkdownElementType.LIST, | ||
| 215 | + content={"list": content}, | ||
| 216 | + start_pos=list_start, end_pos=line_start - 1, raw=content, | ||
| 217 | + )) | ||
| 218 | + list_lines = [] | ||
| 219 | + | ||
| 220 | + # Flush remaining | ||
| 221 | + if table_lines: | ||
| 222 | + content = "\n".join(table_lines) | ||
| 223 | + elements.append(MarkdownElement( | ||
| 224 | + type=MarkdownElementType.TABLE, | ||
| 225 | + content={"table": content}, | ||
| 226 | + start_pos=table_start, end_pos=len(text), raw=content, | ||
| 227 | + )) | ||
| 228 | + if list_lines: | ||
| 229 | + content = "\n".join(list_lines).strip() | ||
| 230 | + if content: | ||
| 231 | + elements.append(MarkdownElement( | ||
| 232 | + type=MarkdownElementType.LIST, | ||
| 233 | + content={"list": content}, | ||
| 234 | + start_pos=list_start, end_pos=len(text), raw=content, | ||
| 235 | + )) | ||
| 236 | + | ||
| 237 | + def _populate_categorized_lists(self, md: MarkdownContent) -> None: | ||
| 238 | + """按类型填充分类列表。""" | ||
| 239 | + for el in md.elements: | ||
| 240 | + if el.type == MarkdownElementType.HEADER: | ||
| 241 | + md.headers.append({"level": el.content["level"], "title": el.content["title"], "raw": el.raw}) | ||
| 242 | + elif el.type == MarkdownElementType.CODE_BLOCK: | ||
| 243 | + md.code_blocks.append({"language": el.content["language"], "code": el.content["code"], "raw": el.raw}) | ||
| 244 | + elif el.type == MarkdownElementType.INLINE_CODE: | ||
| 245 | + md.code_blocks.append({"language": "inline", "code": el.content["code"], "raw": el.raw}) | ||
| 246 | + elif el.type == MarkdownElementType.LINK: | ||
| 247 | + md.links.append({"text": el.content["text"], "url": el.content["url"], "raw": el.raw}) | ||
| 248 | + elif el.type == MarkdownElementType.IMAGE: | ||
| 249 | + md.images.append({"alt": el.content["alt"], "url": el.content["url"], "raw": el.raw}) | ||
| 250 | + elif el.type == MarkdownElementType.TABLE: | ||
| 251 | + md.tables.append(el.content["table"]) | ||
| 252 | + elif el.type == MarkdownElementType.LIST: | ||
| 253 | + md.lists.append(el.content["list"]) | ||
| @@ -0,0 +1,4 @@ | |||
| 1 | +from .base_provider import BaseProviderAdapter | ||
| 2 | +from .registry import get_provider_adapter | ||
| 3 | + | ||
| 4 | +__all__ = ["BaseProviderAdapter", "get_provider_adapter"] | ||
| @@ -0,0 +1,325 @@ | |||
| 1 | +"""Anthropic provider adapter — 消息/工具/Streaming 格式转换。""" | ||
| 2 | +import json | ||
| 3 | +import logging | ||
| 4 | +from typing import Dict, List, Optional, Any | ||
| 5 | + | ||
| 6 | +from ..core.deployment import Deployment | ||
| 7 | +from ..utils.media import is_data_uri, parse_data_uri | ||
| 8 | +from .base_provider import BaseProviderAdapter | ||
| 9 | + | ||
| 10 | +logger = logging.getLogger(__name__) | ||
| 11 | + | ||
| 12 | + | ||
| 13 | +class AnthropicProviderAdapter(BaseProviderAdapter): | ||
| 14 | + | ||
| 15 | + ANTHROPIC_VERSION = "2023-06-01" | ||
| 16 | + DEFAULT_MAX_TOKENS = 4096 | ||
| 17 | + | ||
| 18 | + def get_api_url(self, deployment: Deployment, stream: bool = False) -> str: | ||
| 19 | + base = deployment.api_base.rstrip("/") | ||
| 20 | + return f"{base}/v1/messages" | ||
| 21 | + | ||
| 22 | + def get_headers(self, deployment: Deployment) -> Dict[str, str]: | ||
| 23 | + return { | ||
| 24 | + "x-api-key": deployment.api_key, | ||
| 25 | + "anthropic-version": self.ANTHROPIC_VERSION, | ||
| 26 | + "Content-Type": "application/json", | ||
| 27 | + } | ||
| 28 | + | ||
| 29 | + def transform_request( | ||
| 30 | + self, | ||
| 31 | + model: str, | ||
| 32 | + messages: List[Dict[str, Any]], | ||
| 33 | + deployment: Deployment, | ||
| 34 | + **kwargs, | ||
| 35 | + ) -> Dict[str, Any]: | ||
| 36 | + stream = kwargs.pop("stream", False) | ||
| 37 | + system_blocks, non_system = self._extract_system(messages) | ||
| 38 | + anthropic_messages = self._convert_messages(non_system) | ||
| 39 | + | ||
| 40 | + body: Dict[str, Any] = { | ||
| 41 | + "model": model, | ||
| 42 | + "messages": anthropic_messages, | ||
| 43 | + } | ||
| 44 | + | ||
| 45 | + if stream: | ||
| 46 | + body["stream"] = True | ||
| 47 | + | ||
| 48 | + body["max_tokens"] = kwargs.pop("max_tokens", self.DEFAULT_MAX_TOKENS) | ||
| 49 | + | ||
| 50 | + tools = kwargs.pop("tools", None) | ||
| 51 | + if tools: | ||
| 52 | + body["tools"] = self._convert_tools(tools) | ||
| 53 | + tool_choice = kwargs.pop("tool_choice", None) | ||
| 54 | + if tool_choice: | ||
| 55 | + body["tool_choice"] = self._convert_tool_choice(tool_choice) | ||
| 56 | + | ||
| 57 | + for k, v in kwargs.items(): | ||
| 58 | + if k in ("temperature", "top_p", "top_k"): | ||
| 59 | + body[k] = v | ||
| 60 | + elif k == "stop": | ||
| 61 | + body["stop_sequences"] = [v] if isinstance(v, str) else v | ||
| 62 | + elif k == "metadata": | ||
| 63 | + body["metadata"] = v | ||
| 64 | + | ||
| 65 | + if system_blocks: | ||
| 66 | + body["system"] = system_blocks | ||
| 67 | + | ||
| 68 | + return body | ||
| 69 | + | ||
| 70 | + def transform_response( | ||
| 71 | + self, | ||
| 72 | + raw_response: Dict[str, Any], | ||
| 73 | + model: str, | ||
| 74 | + deployment: Deployment, | ||
| 75 | + ) -> Dict[str, Any]: | ||
| 76 | + content = [] | ||
| 77 | + tool_calls = [] | ||
| 78 | + for block in raw_response.get("content", []): | ||
| 79 | + if block.get("type") == "text": | ||
| 80 | + content.append(block.get("text", "")) | ||
| 81 | + elif block.get("type") == "tool_use": | ||
| 82 | + tool_calls.append({ | ||
| 83 | + "id": block["id"], | ||
| 84 | + "type": "function", | ||
| 85 | + "function": { | ||
| 86 | + "name": block["name"], | ||
| 87 | + "arguments": json.dumps(block.get("input", {})), | ||
| 88 | + }, | ||
| 89 | + }) | ||
| 90 | + | ||
| 91 | + message: Dict[str, Any] = { | ||
| 92 | + "role": "assistant", | ||
| 93 | + "content": "".join(content) if content else None, | ||
| 94 | + } | ||
| 95 | + if tool_calls: | ||
| 96 | + message["tool_calls"] = tool_calls | ||
| 97 | + | ||
| 98 | + stop_reason = raw_response.get("stop_reason") | ||
| 99 | + finish_reason = self.ANTHROPIC_STOP_REASON_MAP.get(stop_reason, "stop") | ||
| 100 | + | ||
| 101 | + usage = raw_response.get("usage", {}) | ||
| 102 | + pi = usage.get("input_tokens", 0) | ||
| 103 | + co = usage.get("output_tokens", 0) | ||
| 104 | + | ||
| 105 | + return self._build_completion_response( | ||
| 106 | + id=raw_response.get("id", ""), | ||
| 107 | + model=model, | ||
| 108 | + message=message, | ||
| 109 | + finish_reason=finish_reason, | ||
| 110 | + prompt_tokens=pi, | ||
| 111 | + completion_tokens=co, | ||
| 112 | + ) | ||
| 113 | + | ||
| 114 | + def transform_stream_chunk( | ||
| 115 | + self, | ||
| 116 | + chunk: Dict[str, Any], | ||
| 117 | + model: str, | ||
| 118 | + deployment: Deployment, | ||
| 119 | + ) -> Optional[Dict[str, Any]]: | ||
| 120 | + event_type = chunk.get("type") | ||
| 121 | + | ||
| 122 | + if event_type == "message_start": | ||
| 123 | + msg = chunk.get("message", {}) | ||
| 124 | + return self._build_chunk_response( | ||
| 125 | + model=model, delta={"role": "assistant"}, id=msg.get("id", ""), | ||
| 126 | + ) | ||
| 127 | + elif event_type == "content_block_delta": | ||
| 128 | + block_index = chunk.get("index", 0) | ||
| 129 | + delta = chunk.get("delta", {}) | ||
| 130 | + if delta.get("type") == "text_delta": | ||
| 131 | + return self._build_chunk_response( | ||
| 132 | + model=model, delta={"content": delta.get("text", "")}, | ||
| 133 | + ) | ||
| 134 | + elif delta.get("type") == "input_json_delta": | ||
| 135 | + return self._build_chunk_response( | ||
| 136 | + model=model, | ||
| 137 | + delta={"tool_calls": [{"index": block_index, "function": {"arguments": delta.get("partial_json", "")}}]}, | ||
| 138 | + ) | ||
| 139 | + elif event_type == "content_block_start": | ||
| 140 | + block_index = chunk.get("index", 0) | ||
| 141 | + block = chunk.get("content_block", {}) | ||
| 142 | + if block.get("type") == "tool_use": | ||
| 143 | + return self._build_chunk_response( | ||
| 144 | + model=model, | ||
| 145 | + delta={"tool_calls": [{ | ||
| 146 | + "index": block_index, "id": block.get("id", ""), | ||
| 147 | + "type": "function", | ||
| 148 | + "function": {"name": block.get("name", ""), "arguments": ""}, | ||
| 149 | + }]}, | ||
| 150 | + ) | ||
| 151 | + elif event_type == "message_delta": | ||
| 152 | + delta = chunk.get("delta", {}) | ||
| 153 | + stop_reason = delta.get("stop_reason") | ||
| 154 | + usage = chunk.get("usage", {}) | ||
| 155 | + co = usage.get("output_tokens", 0) | ||
| 156 | + return self._build_chunk_response( | ||
| 157 | + model=model, | ||
| 158 | + delta={}, | ||
| 159 | + finish_reason=self.ANTHROPIC_STOP_REASON_MAP.get(stop_reason, "stop"), | ||
| 160 | + usage={"completion_tokens": co}, | ||
| 161 | + ) | ||
| 162 | + elif event_type == "ping": | ||
| 163 | + return None | ||
| 164 | + | ||
| 165 | + return None | ||
| 166 | + | ||
| 167 | + def _extract_system( | ||
| 168 | + self, messages: List[Dict[str, Any]] | ||
| 169 | + ) -> tuple[List[Dict[str, str]], List[Dict[str, Any]]]: | ||
| 170 | + """提取 system 消息,返回 (system_blocks, non_system_messages)。""" | ||
| 171 | + system_msgs, non_system = self._extract_system_messages(messages) | ||
| 172 | + system_blocks: List[Dict[str, str]] = [] | ||
| 173 | + for msg in system_msgs: | ||
| 174 | + content = msg.get("content", "") | ||
| 175 | + if content: | ||
| 176 | + system_blocks.append({"type": "text", "text": content}) | ||
| 177 | + return system_blocks, non_system | ||
| 178 | + | ||
| 179 | + @staticmethod | ||
| 180 | + def _convert_messages( | ||
| 181 | + messages: List[Dict[str, Any]], | ||
| 182 | + ) -> List[Dict[str, Any]]: | ||
| 183 | + """将 OpenAI 消息转换为 Anthropic 消息格式。 | ||
| 184 | + | ||
| 185 | + 关键点: | ||
| 186 | + - content 字段必须为 list(不能是纯字符串) | ||
| 187 | + - tool 结果消息映射为 tool_result content block | ||
| 188 | + """ | ||
| 189 | + result: List[Dict[str, Any]] = [] | ||
| 190 | + for msg in messages: | ||
| 191 | + role = msg.get("role", "user") | ||
| 192 | + content = msg.get("content", "") | ||
| 193 | + | ||
| 194 | + if role == "tool": | ||
| 195 | + result.append({ | ||
| 196 | + "role": "user", | ||
| 197 | + "content": [{ | ||
| 198 | + "type": "tool_result", | ||
| 199 | + "tool_use_id": msg.get("tool_call_id", ""), | ||
| 200 | + "content": content if isinstance(content, str) else json.dumps(content), | ||
| 201 | + }], | ||
| 202 | + }) | ||
| 203 | + elif role == "assistant": | ||
| 204 | + blocks: List[Dict[str, Any]] = [] | ||
| 205 | + if content: | ||
| 206 | + blocks.append({"type": "text", "text": content}) | ||
| 207 | + for tc_id, tc_name, tc_input in AnthropicProviderAdapter._iter_tool_call_inputs( | ||
| 208 | + msg.get("tool_calls", []) | ||
| 209 | + ): | ||
| 210 | + blocks.append({ | ||
| 211 | + "type": "tool_use", | ||
| 212 | + "id": tc_id, | ||
| 213 | + "name": tc_name, | ||
| 214 | + "input": tc_input, | ||
| 215 | + }) | ||
| 216 | + # Anthropic requires non-empty content array | ||
| 217 | + if not blocks: | ||
| 218 | + blocks.append({"type": "text", "text": ""}) | ||
| 219 | + result.append({"role": "assistant", "content": blocks}) | ||
| 220 | + else: | ||
| 221 | + # user 消息 | ||
| 222 | + if isinstance(content, str): | ||
| 223 | + result.append({"role": "user", "content": [{"type": "text", "text": content}]}) | ||
| 224 | + elif isinstance(content, list): | ||
| 225 | + converted: List[Dict[str, Any]] = [] | ||
| 226 | + for block in content: | ||
| 227 | + converted.extend( | ||
| 228 | + AnthropicProviderAdapter._convert_content_block(block) | ||
| 229 | + ) | ||
| 230 | + result.append({"role": "user", "content": converted}) | ||
| 231 | + else: | ||
| 232 | + result.append({"role": "user", "content": [{"type": "text", "text": str(content)}]}) | ||
| 233 | + | ||
| 234 | + return result | ||
| 235 | + | ||
| 236 | + @staticmethod | ||
| 237 | + def _convert_content_block(block: Dict[str, Any]) -> List[Dict[str, Any]]: | ||
| 238 | + """将单个 OpenAI 格式 content block 转换为 Anthropic 格式。 | ||
| 239 | + | ||
| 240 | + 返回列表:正常情况返回 [converted_block],不支持的类型返回 [](跳过)。 | ||
| 241 | + """ | ||
| 242 | + if not isinstance(block, dict): | ||
| 243 | + return [{"type": "text", "text": str(block)}] | ||
| 244 | + | ||
| 245 | + block_type = block.get("type", "") | ||
| 246 | + | ||
| 247 | + # text — 透传 | ||
| 248 | + if block_type == "text": | ||
| 249 | + return [block] | ||
| 250 | + | ||
| 251 | + # image_url → image | ||
| 252 | + if block_type == "image_url": | ||
| 253 | + image_url = block.get("image_url", {}) | ||
| 254 | + url = image_url.get("url", "") if isinstance(image_url, dict) else "" | ||
| 255 | + if is_data_uri(url): | ||
| 256 | + mime_type, data = parse_data_uri(url) | ||
| 257 | + return [{"type": "image", "source": { | ||
| 258 | + "type": "base64", "media_type": mime_type, "data": data, | ||
| 259 | + }}] | ||
| 260 | + else: | ||
| 261 | + # HTTPS URL 直接用 url source;HTTP 也传过去(Anthropic 会拒绝) | ||
| 262 | + if url.startswith("http://"): | ||
| 263 | + logger.warning( | ||
| 264 | + "Anthropic 仅支持 HTTPS 图片 URL,当前为 HTTP: %s", url[:100] | ||
| 265 | + ) | ||
| 266 | + return [{"type": "image", "source": {"type": "url", "url": url}}] | ||
| 267 | + | ||
| 268 | + # file → document | ||
| 269 | + if block_type == "file": | ||
| 270 | + file_info = block.get("file", {}) | ||
| 271 | + file_data = file_info.get("file_data", "") | ||
| 272 | + if file_data and is_data_uri(file_data): | ||
| 273 | + mime_type, data = parse_data_uri(file_data) | ||
| 274 | + return [{"type": "document", "source": { | ||
| 275 | + "type": "base64", "media_type": mime_type, "data": data, | ||
| 276 | + }}] | ||
| 277 | + file_id = file_info.get("file_id", "") | ||
| 278 | + if file_id: | ||
| 279 | + if file_id.startswith("http"): | ||
| 280 | + return [{"type": "document", "source": {"type": "url", "url": file_id}}] | ||
| 281 | + return [{"type": "document", "source": {"type": "file", "file_id": file_id}}] | ||
| 282 | + return [] | ||
| 283 | + | ||
| 284 | + # input_audio — Anthropic 不支持 | ||
| 285 | + if block_type == "input_audio": | ||
| 286 | + logger.warning("Anthropic 不支持音频输入,跳过 input_audio block") | ||
| 287 | + return [] | ||
| 288 | + | ||
| 289 | + # video_url — Anthropic 不支持 | ||
| 290 | + if block_type == "video_url": | ||
| 291 | + logger.warning("Anthropic 不支持视频输入,跳过 video_url block") | ||
| 292 | + return [] | ||
| 293 | + | ||
| 294 | + # 未知类型 — 透传(兼容调用方直接传入 Anthropic 原生格式) | ||
| 295 | + return [block] | ||
| 296 | + | ||
| 297 | + @staticmethod | ||
| 298 | + def _convert_tools(tools: List[Dict[str, Any]]) -> List[Dict[str, Any]]: | ||
| 299 | + """OpenAI tools -> Anthropic tools""" | ||
| 300 | + result = [] | ||
| 301 | + for name, description, parameters in AnthropicProviderAdapter._iter_tool_specs(tools): | ||
| 302 | + result.append({ | ||
| 303 | + "name": name, | ||
| 304 | + "description": description, | ||
| 305 | + "input_schema": parameters, | ||
| 306 | + }) | ||
| 307 | + return result | ||
| 308 | + | ||
| 309 | + @staticmethod | ||
| 310 | + def _convert_tool_choice(tool_choice: Any) -> Dict[str, Any]: | ||
| 311 | + """OpenAI tool_choice -> Anthropic tool_choice""" | ||
| 312 | + if isinstance(tool_choice, str): | ||
| 313 | + if tool_choice == "any": | ||
| 314 | + return {"type": "any"} | ||
| 315 | + elif tool_choice == "none": | ||
| 316 | + return {"type": "none"} | ||
| 317 | + return {"type": "auto"} | ||
| 318 | + if isinstance(tool_choice, dict): | ||
| 319 | + tc_type = tool_choice.get("type", "") | ||
| 320 | + if tc_type == "function": | ||
| 321 | + func = tool_choice.get("function", {}) | ||
| 322 | + return {"type": "tool", "name": func.get("name", "")} | ||
| 323 | + if tc_type in ("auto", "any", "none"): | ||
| 324 | + return {"type": tc_type} | ||
| 325 | + return {"type": "auto"} | ||
| @@ -0,0 +1,236 @@ | |||
| 1 | +"""Provider 适配器基类定义。""" | ||
| 2 | +import json | ||
| 3 | +import time | ||
| 4 | +from abc import ABC, abstractmethod | ||
| 5 | +from typing import TYPE_CHECKING, AsyncIterator, Dict, List, Optional, Any, Tuple | ||
| 6 | + | ||
| 7 | +from ..core.deployment import Deployment | ||
| 8 | + | ||
| 9 | +if TYPE_CHECKING: | ||
| 10 | + import httpx | ||
| 11 | + | ||
| 12 | + | ||
| 13 | +class BaseProviderAdapter(ABC): | ||
| 14 | + """Provider 适配器基类 | ||
| 15 | + | ||
| 16 | + 每个 provider(OpenAI / Anthropic / Gemini 等)实现此接口, | ||
| 17 | + intelli_router 在 HTTP 请求层插入 adapter 完成格式转换。 | ||
| 18 | + """ | ||
| 19 | + | ||
| 20 | + # Anthropic 系(Anthropic / Bedrock Converse)通用 stop_reason -> OpenAI finish_reason 映射。 | ||
| 21 | + # Bedrock 多出的 content_filtered 键对纯 Anthropic 无害(查表只命中存在的键)。 | ||
| 22 | + ANTHROPIC_STOP_REASON_MAP = { | ||
| 23 | + "end_turn": "stop", | ||
| 24 | + "stop_sequence": "stop", | ||
| 25 | + "tool_use": "tool_calls", | ||
| 26 | + "max_tokens": "length", | ||
| 27 | + "content_filtered": "content_filter", | ||
| 28 | + } | ||
| 29 | + | ||
| 30 | + @abstractmethod | ||
| 31 | + def get_api_url(self, deployment: Deployment, stream: bool = False) -> str: | ||
| 32 | + """构建 API URL(含路径和 query params)""" | ||
| 33 | + ... | ||
| 34 | + | ||
| 35 | + @abstractmethod | ||
| 36 | + def get_headers(self, deployment: Deployment) -> Dict[str, str]: | ||
| 37 | + """构建 HTTP 请求头(含认证信息)""" | ||
| 38 | + ... | ||
| 39 | + | ||
| 40 | + def transform_request( | ||
| 41 | + self, | ||
| 42 | + model: str, | ||
| 43 | + messages: List[Dict[str, Any]], | ||
| 44 | + deployment: Deployment, | ||
| 45 | + **kwargs, | ||
| 46 | + ) -> Dict[str, Any]: | ||
| 47 | + """将 OpenAI 风格参数转换为 Provider 原生请求体 | ||
| 48 | + | ||
| 49 | + 默认实现:直接合并 model + messages + kwargs(OpenAI 兼容行为)。 | ||
| 50 | + """ | ||
| 51 | + return {"model": model, "messages": messages, **kwargs} | ||
| 52 | + | ||
| 53 | + def transform_response( | ||
| 54 | + self, | ||
| 55 | + raw_response: Dict[str, Any], | ||
| 56 | + model: str, | ||
| 57 | + deployment: Deployment, | ||
| 58 | + ) -> Dict[str, Any]: | ||
| 59 | + """将 Provider 原生响应转换为 OpenAI 统一格式 | ||
| 60 | + | ||
| 61 | + 返回结果中必须包含 usage.completion_tokens 字段(ReliableRouter 依赖)。 | ||
| 62 | + 默认实现:透传(OpenAI 兼容行为)。 | ||
| 63 | + """ | ||
| 64 | + return raw_response | ||
| 65 | + | ||
| 66 | + def transform_stream_chunk( | ||
| 67 | + self, | ||
| 68 | + chunk: Dict[str, Any], | ||
| 69 | + model: str, | ||
| 70 | + deployment: Deployment, | ||
| 71 | + ) -> Optional[Dict[str, Any]]: | ||
| 72 | + """将 Provider 原生 streaming chunk 转换为 OpenAI 统一 SSE 格式 | ||
| 73 | + | ||
| 74 | + 返回 None 表示该 chunk 应被跳过(例如 keepalive / ping)。 | ||
| 75 | + 默认实现:透传(OpenAI 兼容行为)。 | ||
| 76 | + """ | ||
| 77 | + return chunk | ||
| 78 | + | ||
| 79 | + def sign_request( | ||
| 80 | + self, | ||
| 81 | + method: str, | ||
| 82 | + url: str, | ||
| 83 | + headers: Dict[str, str], | ||
| 84 | + body: bytes, | ||
| 85 | + deployment: Deployment, | ||
| 86 | + ) -> Dict[str, str]: | ||
| 87 | + """签名请求并返回更新后的 headers。 | ||
| 88 | + | ||
| 89 | + 默认实现:原样返回(无需签名)。 | ||
| 90 | + 需要请求签名的 provider(如 AWS Bedrock SigV4)应覆盖此方法。 | ||
| 91 | + """ | ||
| 92 | + return headers | ||
| 93 | + | ||
| 94 | + async def iter_stream_events( | ||
| 95 | + self, | ||
| 96 | + response: "httpx.Response", | ||
| 97 | + ) -> AsyncIterator[Dict[str, Any]]: | ||
| 98 | + """从 HTTP 响应中迭代流式事件。 | ||
| 99 | + | ||
| 100 | + 默认实现:标准 SSE 格式解析(data: ... 行)。 | ||
| 101 | + 使用非 SSE 流式协议的 provider(如 Bedrock 二进制 Event Stream)应覆盖此方法。 | ||
| 102 | + """ | ||
| 103 | + async for line in response.aiter_lines(): | ||
| 104 | + if not line: | ||
| 105 | + continue | ||
| 106 | + if line.startswith("data: "): | ||
| 107 | + data = line[6:].strip() | ||
| 108 | + if data == "[DONE]": | ||
| 109 | + break | ||
| 110 | + try: | ||
| 111 | + yield json.loads(data) | ||
| 112 | + except json.JSONDecodeError: | ||
| 113 | + continue | ||
| 114 | + | ||
| 115 | + @staticmethod | ||
| 116 | + def _build_completion_response( | ||
| 117 | + id: str, | ||
| 118 | + model: str, | ||
| 119 | + message: Dict[str, Any], | ||
| 120 | + finish_reason: str, | ||
| 121 | + prompt_tokens: int, | ||
| 122 | + completion_tokens: int, | ||
| 123 | + ) -> Dict[str, Any]: | ||
| 124 | + """构建标准 OpenAI chat.completion 响应信封。""" | ||
| 125 | + return { | ||
| 126 | + "id": id, | ||
| 127 | + "object": "chat.completion", | ||
| 128 | + "created": int(time.time()), | ||
| 129 | + "model": model, | ||
| 130 | + "choices": [{"index": 0, "message": message, "finish_reason": finish_reason}], | ||
| 131 | + "usage": { | ||
| 132 | + "prompt_tokens": prompt_tokens, | ||
| 133 | + "completion_tokens": completion_tokens, | ||
| 134 | + "total_tokens": prompt_tokens + completion_tokens, | ||
| 135 | + }, | ||
| 136 | + } | ||
| 137 | + | ||
| 138 | + @staticmethod | ||
| 139 | + def _build_chunk_response( | ||
| 140 | + model: str, | ||
| 141 | + delta: Dict[str, Any], | ||
| 142 | + finish_reason: Optional[str] = None, | ||
| 143 | + id: str = "", | ||
| 144 | + usage: Optional[Dict[str, Any]] = None, | ||
| 145 | + ) -> Dict[str, Any]: | ||
| 146 | + """构建标准 OpenAI chat.completion.chunk 响应信封。""" | ||
| 147 | + chunk: Dict[str, Any] = { | ||
| 148 | + "id": id, | ||
| 149 | + "object": "chat.completion.chunk", | ||
| 150 | + "created": int(time.time()), | ||
| 151 | + "model": model, | ||
| 152 | + "choices": [{"index": 0, "delta": delta, "finish_reason": finish_reason}], | ||
| 153 | + } | ||
| 154 | + if usage: | ||
| 155 | + chunk["usage"] = usage | ||
| 156 | + return chunk | ||
| 157 | + | ||
| 158 | + @staticmethod | ||
| 159 | + def _extract_system_messages( | ||
| 160 | + messages: List[Dict[str, Any]], | ||
| 161 | + ) -> Tuple[List[Dict[str, Any]], List[Dict[str, Any]]]: | ||
| 162 | + """将 system 消息与非 system 消息分离。 | ||
| 163 | + | ||
| 164 | + Returns: | ||
| 165 | + (system_messages, non_system_messages) | ||
| 166 | + """ | ||
| 167 | + system: List[Dict[str, Any]] = [] | ||
| 168 | + non_system: List[Dict[str, Any]] = [] | ||
| 169 | + for msg in messages: | ||
| 170 | + if msg.get("role") == "system": | ||
| 171 | + system.append(msg) | ||
| 172 | + else: | ||
| 173 | + non_system.append(msg) | ||
| 174 | + return system, non_system | ||
| 175 | + | ||
| 176 | + @staticmethod | ||
| 177 | + def _iter_tool_call_inputs( | ||
| 178 | + tool_calls: List[Dict[str, Any]], | ||
| 179 | + ) -> List[Tuple[str, str, Dict[str, Any]]]: | ||
| 180 | + """从 OpenAI assistant.tool_calls 中解析出 (id, name, input_dict)。 | ||
| 181 | + | ||
| 182 | + 封装 arguments 的 JSON 反序列化与异常回退(失败时 input 为 {}), | ||
| 183 | + 各 provider 据此拼装自己的 tool_use / toolUse / functionCall 块。 | ||
| 184 | + """ | ||
| 185 | + result: List[Tuple[str, str, Dict[str, Any]]] = [] | ||
| 186 | + for tc in tool_calls or []: | ||
| 187 | + func = tc.get("function", {}) | ||
| 188 | + args_str = func.get("arguments", "{}") | ||
| 189 | + try: | ||
| 190 | + args_input = json.loads(args_str) if args_str else {} | ||
| 191 | + except (json.JSONDecodeError, TypeError): | ||
| 192 | + args_input = {} | ||
| 193 | + result.append((tc.get("id", ""), func.get("name", ""), args_input)) | ||
| 194 | + return result | ||
| 195 | + | ||
| 196 | + @staticmethod | ||
| 197 | + def _iter_tool_specs( | ||
| 198 | + tools: List[Dict[str, Any]], | ||
| 199 | + ) -> List[Tuple[str, str, Dict[str, Any]]]: | ||
| 200 | + """从 OpenAI tools 中解析出 (name, description, parameters)。 | ||
| 201 | + | ||
| 202 | + 兼容两种输入:标准 {"function": {...}} 包裹,或直接的 function 对象。 | ||
| 203 | + """ | ||
| 204 | + specs: List[Tuple[str, str, Dict[str, Any]]] = [] | ||
| 205 | + for tool in tools or []: | ||
| 206 | + func = tool.get("function", tool) | ||
| 207 | + specs.append(( | ||
| 208 | + func.get("name", ""), | ||
| 209 | + func.get("description", ""), | ||
| 210 | + func.get("parameters", {}), | ||
| 211 | + )) | ||
| 212 | + return specs | ||
| 213 | + | ||
| 214 | + _TOOL_CALL_FUNC_FIELDS = ("name", "arguments") | ||
| 215 | + | ||
| 216 | + @staticmethod | ||
| 217 | + def sanitize_tool_calls(messages: List[Dict[str, Any]]) -> None: | ||
| 218 | + """清洗 tool_calls,只保留 OpenAI 标准字段:id/type/function.name/function.arguments | ||
| 219 | + | ||
| 220 | + 一些 provider(SiliconFlow、InferenceAffinity 等)对 tool_calls 中 | ||
| 221 | + 的非标准字段(如 index)敏感,需要原地清洗。 | ||
| 222 | + | ||
| 223 | + Args: | ||
| 224 | + messages: 消息列表(原地修改) | ||
| 225 | + """ | ||
| 226 | + def _normalize(call: Dict[str, Any]) -> Dict[str, Any]: | ||
| 227 | + raw_func = call.get("function") or {} | ||
| 228 | + func = {k: raw_func.get(k, "") for k in BaseProviderAdapter._TOOL_CALL_FUNC_FIELDS} | ||
| 229 | + return {"id": call.get("id", ""), "type": "function", "function": func} | ||
| 230 | + | ||
| 231 | + for msg in messages: | ||
| 232 | + if msg.get("role") != "assistant": | ||
| 233 | + continue | ||
| 234 | + calls = msg.get("tool_calls") | ||
| 235 | + if isinstance(calls, list): | ||
| 236 | + msg["tool_calls"] = [_normalize(c) for c in calls if isinstance(c, dict)] | ||
【review】XSS跨站脚本。web_dashboard.py 的 update() 函数中,使用 innerHTML += 模板字面量将服务器返回的统计数据直接插入 DOM,未做 HTML 转义。具体位置:L217 的 ${dep}(deployment ID)、L226 的 ${model}(模型名)、L233 的 ${err}(错误类型)。这些值来自 /api/stats 接口的 JSON 响应,而该响应的数据来源是用户配置的 Deployment 对象的 id/model_name 字段以及异常的类型名称。如果攻击者能够控制 deployment ID(例如注册 id 为 '
' 的 deployment),当任何用户访问 Web Dashboard 时,将在其浏览器中执行任意 JavaScript 代码。由于 Dashboard 通常在本地开发环境运行(默认端口 8765),攻击面相对有限,但在多用户共享环境下(如团队共用监控面板)仍构成实际风险。自动化扫描正确识别了 6 处 innerHTML 赋值,但将其统一标记为 XSS 缺乏具体上下文分析——其中 L214/222/225/234 的 innerHTML = '' 和 innerHTML = 'No data' 是纯静态 HTML,无注入风险;而 L217/226/233 的 innerHTML += 包含动态数据插值,才是真正的 XSS 点。。方案1(推荐):改用 textContent 替代 innerHTML,避免 HTML 解析。例如将 dt.innerHTML +=
<tr>...改为 document.createElement('tr') + textContent 逐单元格设置。方案2:对动态值进行 HTML 转义,添加辅助函数 escapeHtml(s) { return s.replace(/&/g,'&').replace(/</g,'<').replace(/>/g,'>').replace(/"/g,'"'); },然后使用 ${escapeHtml(dep)} 替代 ${dep}。方案3:使用现代框架(如 React/Vue 的虚拟 DOM)自动转义,但对当前轻量级看板来说过重。。