已开启
feat: harden RAM-A-MEM validation and observability #18
DoraA_Mengjie创建于 4 天前
feat: harden RAM-A-MEM validation and observability #18
已开启
共 36 个文件变更+5953-254
| @@ -1208,6 +1208,7 @@ dependencies = [ | |||
| 1208 | "thiserror 1.0.69", | 1208 | "thiserror 1.0.69", |
| 1209 | "tokio", | 1209 | "tokio", |
| 1210 | "tracing", | 1210 | "tracing", |
| 1211 | + "tracing-subscriber", | ||
| 1211 | ] | 1212 | ] |
| 1212 | 1213 | ||
| 1213 | [[package]] | 1214 | [[package]] |
| @@ -242,6 +242,10 @@ guidance, see [`plugins/mcp/case-tool-instruction.md`](plugins/mcp/case-tool-ins | |||
| 242 | xiaoO + RAM-A knowledge base configuration can start from those files and then apply the | 242 | xiaoO + RAM-A knowledge base configuration can start from those files and then apply the |
| 243 | field changes listed below. | 243 | field changes listed below. |
| 244 | 244 | ||
| 245 | +The full server-field reference and the input/output contract of each memory pipeline stage are | ||
| 246 | +documented in | ||
| 247 | +[`docs/guides/ram-a-mem-configuration-and-pipeline.zh-CN.md`](docs/guides/ram-a-mem-configuration-and-pipeline.zh-CN.md). | ||
| 248 | + | ||
| 245 | ```json | 249 | ```json |
| 246 | { | 250 | { |
| 247 | "auth": { | 251 | "auth": { |
| @@ -273,7 +277,7 @@ field changes listed below. | |||
| 273 | "allowed_hosts": ["127.0.0.1:18081"] | 277 | "allowed_hosts": ["127.0.0.1:18081"] |
| 274 | }, | 278 | }, |
| 275 | "limits": { | 279 | "limits": { |
| 276 | - "max_body_bytes": 1048576, | 280 | + "max_body_bytes": 16777216, |
| 277 | "requests_per_second": 20, | 281 | "requests_per_second": 20, |
| 278 | "rate_burst": 40, | 282 | "rate_burst": 40, |
| 279 | "max_in_flight_per_principal_tool": 4, | 283 | "max_in_flight_per_principal_tool": 4, |
| @@ -283,6 +287,10 @@ field changes listed below. | |||
| 283 | "max_active_sessions_global": 256, | 287 | "max_active_sessions_global": 256, |
| 284 | "session_idle_timeout_seconds": 1800 | 288 | "session_idle_timeout_seconds": 1800 |
| 285 | }, | 289 | }, |
| 290 | + "pipeline": { | ||
| 291 | + "fail_fast": true, | ||
| 292 | + "max_memory_chars": 500 | ||
| 293 | + }, | ||
| 286 | "storage": { | 294 | "storage": { |
| 287 | "database_path": "data/ram-a-memory.sqlite" | 295 | "database_path": "data/ram-a-memory.sqlite" |
| 288 | }, | 296 | }, |
| @@ -856,6 +856,18 @@ fn validate_json_content(content: &str) -> Result<(), LlmAttemptError> { | |||
| 856 | async fn read_bounded_response_body( | 856 | async fn read_bounded_response_body( |
| 857 | response: &mut reqwest::Response, | 857 | response: &mut reqwest::Response, |
| 858 | ) -> Result<String, LlmAttemptError> { | 858 | ) -> Result<String, LlmAttemptError> { |
| 859 | + if response | ||
| 860 | + .content_length() | ||
| 861 | + .is_some_and(|length| length > MAX_GRAPH_LLM_RESPONSE_BYTES as u64) | ||
| 862 | + { | ||
| 863 | + return Err(LlmAttemptError { | ||
| 864 | + retryable: false, | ||
| 865 | + message: format!( | ||
| 866 | + "graph LLM response body exceeds {MAX_GRAPH_LLM_RESPONSE_BYTES} byte limit" | ||
| 867 | + ), | ||
| 868 | + }); | ||
| 869 | + } | ||
| 870 | + | ||
| 859 | let mut body = Vec::new(); | 871 | let mut body = Vec::new(); |
| 860 | while let Some(chunk) = response.chunk().await.map_err(|error| { | 872 | while let Some(chunk) = response.chunk().await.map_err(|error| { |
| 861 | LlmAttemptError::retryable(format!("failed to read graph LLM response body: {error}")) | 873 | LlmAttemptError::retryable(format!("failed to read graph LLM response body: {error}")) |
| @@ -12,14 +12,13 @@ fn retry_backoff(attempt: usize) -> Duration { | |||
| 12 | Duration::from_secs(1 << (attempt - 1)) | 12 | Duration::from_secs(1 << (attempt - 1)) |
| 13 | } | 13 | } |
| 14 | 14 | ||
| 15 | -/// Whether a rerank failure is worth retrying (network drop, retryable HTTP status, | 15 | +/// Whether a rerank failure is worth retrying. Invalid provider responses are |
| 16 | -/// decode hiccup). Mirrors the embedding retry policy. | 16 | +/// deterministic contract failures and are not retried. |
| 17 | fn is_retryable_rerank_failure(message: &str) -> bool { | 17 | fn is_retryable_rerank_failure(message: &str) -> bool { |
| 18 | let lower = message.to_ascii_lowercase(); | 18 | let lower = message.to_ascii_lowercase(); |
| 19 | lower.contains("error sending request") | 19 | lower.contains("error sending request") |
| 20 | || lower.contains("failed to read") | 20 | || lower.contains("failed to read") |
| 21 | || lower.contains("operation timed out") | 21 | || lower.contains("operation timed out") |
| 22 | - || lower.contains("decode failed") | ||
| 23 | || lower.contains("http 408") | 22 | || lower.contains("http 408") |
| 24 | || lower.contains("http 425") | 23 | || lower.contains("http 425") |
| 25 | || lower.contains("http 429") | 24 | || lower.contains("http 429") |
| @@ -320,6 +319,34 @@ mod tests { | |||
| 320 | assert_eq!(retry_backoff(7), Duration::from_secs(64)); | 319 | assert_eq!(retry_backoff(7), Duration::from_secs(64)); |
| 321 | } | 320 | } |
| 322 | 321 | ||
| 322 | + | ||
| 323 | + fn retry_classification_is_limited_to_transient_failures() { | ||
| 324 | + for message in [ | ||
| 325 | + "error sending request", | ||
| 326 | + "failed to read response", | ||
| 327 | + "operation timed out", | ||
| 328 | + "HTTP 408", | ||
| 329 | + "HTTP 425", | ||
| 330 | + "HTTP 429", | ||
| 331 | + "HTTP 500", | ||
| 332 | + "HTTP 502", | ||
| 333 | + "HTTP 503", | ||
| 334 | + "HTTP 504", | ||
| 335 | + ] { | ||
| 336 | + assert!(is_retryable_rerank_failure(message), "{message}"); | ||
| 337 | + } | ||
| 338 | + for message in [ | ||
| 339 | + "HTTP 400", | ||
| 340 | + "HTTP 401", | ||
| 341 | + "HTTP 403", | ||
| 342 | + "decode failed", | ||
| 343 | + "duplicate index", | ||
| 344 | + "non-finite score", | ||
| 345 | + ] { | ||
| 346 | + assert!(!is_retryable_rerank_failure(message), "{message}"); | ||
| 347 | + } | ||
| 348 | + } | ||
| 349 | + | ||
| 323 | fn candidate(id: &str) -> ScoredMemory { | 350 | fn candidate(id: &str) -> ScoredMemory { |
| 324 | ScoredMemory { | 351 | ScoredMemory { |
| 325 | record: MemoryRecord { | 352 | record: MemoryRecord { |
| @@ -700,3 +700,49 @@ fn blob_to_embedding(bytes: &[u8]) -> MemoryResult<Vec<f32>> { | |||
| 700 | .map(|chunk| f32::from_le_bytes([chunk[0], chunk[1], chunk[2], chunk[3]])) | 700 | .map(|chunk| f32::from_le_bytes([chunk[0], chunk[1], chunk[2], chunk[3]])) |
| 701 | .collect()) | 701 | .collect()) |
| 702 | } | 702 | } |
| 703 | + | ||
| 704 | + | ||
| 705 | +mod tests { | ||
| 706 | + use serde_json::json; | ||
| 707 | + | ||
| 708 | + use super::normalize_lower_is_better_scores; | ||
| 709 | + use crate::{MemoryRecord, ScoredMemory}; | ||
| 710 | + | ||
| 711 | + fn candidate(id: &str, score: f32) -> ScoredMemory { | ||
| 712 | + ScoredMemory { | ||
| 713 | + record: MemoryRecord { | ||
| 714 | + id: id.to_string(), | ||
| 715 | + text: id.to_string(), | ||
| 716 | + metadata: json!({}), | ||
| 717 | + embedding: Vec::new(), | ||
| 718 | + created_at_ms: 0, | ||
| 719 | + updated_at_ms: 0, | ||
| 720 | + }, | ||
| 721 | + score, | ||
| 722 | + } | ||
| 723 | + } | ||
| 724 | + | ||
| 725 | + | ||
| 726 | + fn bm25_scores_are_reverse_min_max_normalized() { | ||
| 727 | + let mut candidates = vec![ | ||
| 728 | + candidate("best", -3.0), | ||
| 729 | + candidate("middle", -2.0), | ||
| 730 | + candidate("worst", -1.0), | ||
| 731 | + ]; | ||
| 732 | + | ||
| 733 | + normalize_lower_is_better_scores(&mut candidates); | ||
| 734 | + | ||
| 735 | + assert_eq!(candidates[0].score, 1.0); | ||
| 736 | + assert_eq!(candidates[1].score, 0.5); | ||
| 737 | + assert_eq!(candidates[2].score, 0.0); | ||
| 738 | + } | ||
| 739 | + | ||
| 740 | + | ||
| 741 | + fn equal_bm25_scores_normalize_to_one() { | ||
| 742 | + let mut candidates = vec![candidate("a", -1.0), candidate("b", -1.0)]; | ||
| 743 | + | ||
| 744 | + normalize_lower_is_better_scores(&mut candidates); | ||
| 745 | + | ||
| 746 | + assert!(candidates.iter().all(|candidate| candidate.score == 1.0)); | ||
| 747 | + } | ||
| 748 | +} | ||
| @@ -16,6 +16,8 @@ pub struct ServerConfig { | |||
| 16 | 16 | ||
| 17 | pub limits: LimitsConfig, | 17 | pub limits: LimitsConfig, |
| 18 | 18 | ||
| 19 | + pub pipeline: PipelineServiceConfig, | ||
| 20 | + | ||
| 19 | pub storage: Option<StorageConfig>, | 21 | pub storage: Option<StorageConfig>, |
| 20 | 22 | ||
| 21 | pub providers: Option<ProvidersConfig>, | 23 | pub providers: Option<ProvidersConfig>, |
| @@ -27,6 +29,20 @@ pub struct ServerConfig { | |||
| 27 | pub graph_memory: Option<GraphMemoryServiceConfig>, | 29 | pub graph_memory: Option<GraphMemoryServiceConfig>, |
| 28 | } | 30 | } |
| 29 | 31 | ||
| 32 | +pub const MAX_MCP_BODY_BYTES: usize = 64 * 1024 * 1024; | ||
| 33 | +pub const MAX_REQUESTS_PER_SECOND: u32 = 10_000; | ||
| 34 | +pub const MAX_RATE_BURST: u32 = 100_000; | ||
| 35 | +pub const MAX_IN_FLIGHT_PER_PRINCIPAL_TOOL: usize = 1_024; | ||
| 36 | +pub const MAX_INITIALIZE_REQUESTS_PER_SECOND: u32 = 1_000; | ||
| 37 | +pub const MAX_INITIALIZE_RATE_BURST: u32 = 10_000; | ||
| 38 | +pub const MAX_ACTIVE_SESSIONS_PER_PRINCIPAL: usize = 1_024; | ||
| 39 | +pub const MAX_ACTIVE_SESSIONS_GLOBAL: usize = 100_000; | ||
| 40 | +pub const MAX_SESSION_IDLE_TIMEOUT_SECONDS: u64 = 86_400; | ||
| 41 | +pub const DEFAULT_RERANK_TIMEOUT_MS: u64 = 30_000; | ||
| 42 | +pub const MAX_RERANK_TIMEOUT_MS: u64 = 120_000; | ||
| 43 | +const SUPPORTED_PERMISSIONS: [&str; 4] = | ||
| 44 | + ["memory:read", "memory:write", "cases:read", "cases:write"]; | ||
| 45 | + | ||
| 30 | 46 | ||
| 31 | pub struct FeatureFlags { | 47 | pub struct FeatureFlags { |
| 32 | pub memory: bool, | 48 | pub memory: bool, |
| @@ -284,6 +300,69 @@ impl Default for LimitsConfig { | |||
| 284 | } | 300 | } |
| 285 | } | 301 | } |
| 286 | 302 | ||
| 303 | +impl LimitsConfig { | ||
| 304 | + fn validate(&self) -> Result<()> { | ||
| 305 | + let within_bounds = self.max_body_bytes <= MAX_MCP_BODY_BYTES | ||
| 306 | + && self.requests_per_second <= MAX_REQUESTS_PER_SECOND | ||
| 307 | + && self.rate_burst <= MAX_RATE_BURST | ||
| 308 | + && self.max_in_flight_per_principal_tool <= MAX_IN_FLIGHT_PER_PRINCIPAL_TOOL | ||
| 309 | + && self.initialize_requests_per_second <= MAX_INITIALIZE_REQUESTS_PER_SECOND | ||
| 310 | + && self.initialize_rate_burst <= MAX_INITIALIZE_RATE_BURST | ||
| 311 | + && self.max_active_sessions_per_principal <= MAX_ACTIVE_SESSIONS_PER_PRINCIPAL | ||
| 312 | + && self.max_active_sessions_global <= MAX_ACTIVE_SESSIONS_GLOBAL | ||
| 313 | + && self.session_idle_timeout_seconds <= MAX_SESSION_IDLE_TIMEOUT_SECONDS; | ||
| 314 | + let all_nonzero = self.max_body_bytes > 0 | ||
| 315 | + && self.requests_per_second > 0 | ||
| 316 | + && self.rate_burst > 0 | ||
| 317 | + && self.max_in_flight_per_principal_tool > 0 | ||
| 318 | + && self.initialize_requests_per_second > 0 | ||
| 319 | + && self.initialize_rate_burst > 0 | ||
| 320 | + && self.max_active_sessions_per_principal > 0 | ||
| 321 | + && self.max_active_sessions_global > 0 | ||
| 322 | + && self.session_idle_timeout_seconds > 0; | ||
| 323 | + if !all_nonzero || !within_bounds { | ||
| 324 | + anyhow::bail!("HTTP limits are outside the supported range"); | ||
| 325 | + } | ||
| 326 | + if self.max_active_sessions_global < self.max_active_sessions_per_principal { | ||
| 327 | + anyhow::bail!("global active session limit must be at least the per-principal limit"); | ||
| 328 | + } | ||
| 329 | + Ok(()) | ||
| 330 | + } | ||
| 331 | +} | ||
| 332 | + | ||
| 333 | + | ||
| 334 | + | ||
| 335 | +pub struct PipelineServiceConfig { | ||
| 336 | + pub fail_fast: bool, | ||
| 337 | + pub max_memory_chars: usize, | ||
| 338 | +} | ||
| 339 | + | ||
| 340 | +impl Default for PipelineServiceConfig { | ||
| 341 | + fn default() -> Self { | ||
| 342 | + let defaults = memory_pipeline::pipeline::PipelineConfig::default(); | ||
| 343 | + Self { | ||
| 344 | + fail_fast: defaults.fail_fast, | ||
| 345 | + max_memory_chars: defaults.validation.max_memory_chars, | ||
| 346 | + } | ||
| 347 | + } | ||
| 348 | +} | ||
| 349 | + | ||
| 350 | +impl PipelineServiceConfig { | ||
| 351 | + pub fn pipeline_config(&self) -> memory_pipeline::pipeline::PipelineConfig { | ||
| 352 | + let mut config = memory_pipeline::pipeline::PipelineConfig::default(); | ||
| 353 | + config.fail_fast = self.fail_fast; | ||
| 354 | + config.validation.max_memory_chars = self.max_memory_chars; | ||
| 355 | + config | ||
| 356 | + } | ||
| 357 | + | ||
| 358 | + fn validate(&self) -> Result<()> { | ||
| 359 | + if !(1..=crate::MAX_MESSAGE_TEXT_CHARS).contains(&self.max_memory_chars) { | ||
| 360 | + anyhow::bail!("pipeline max_memory_chars must be between 1 and 32000"); | ||
| 361 | + } | ||
| 362 | + Ok(()) | ||
| 363 | + } | ||
| 364 | +} | ||
| 365 | + | ||
| 287 | 366 | ||
| 288 | 367 | ||
| 289 | pub struct StorageConfig { | 368 | pub struct StorageConfig { |
| @@ -351,6 +430,18 @@ pub struct ProvidersConfig { | |||
| 351 | pub max_retries: usize, | 430 | pub max_retries: usize, |
| 352 | } | 431 | } |
| 353 | 432 | ||
| 433 | +impl ProvidersConfig { | ||
| 434 | + pub fn resolved_embedding_api_key_env(&self) -> &str { | ||
| 435 | + self.embedding_api_key_env | ||
| 436 | + .as_deref() | ||
| 437 | + .unwrap_or(&self.api_key_env) | ||
| 438 | + } | ||
| 439 | + | ||
| 440 | + pub fn resolved_embedding_base_url(&self) -> &str { | ||
| 441 | + self.embedding_base_url.as_deref().unwrap_or(&self.base_url) | ||
| 442 | + } | ||
| 443 | +} | ||
| 444 | + | ||
| 354 | 445 | ||
| 355 | 446 | ||
| 356 | pub struct RetrievalServiceConfig { | 447 | pub struct RetrievalServiceConfig { |
| @@ -440,7 +531,7 @@ impl Default for RerankServiceConfig { | |||
| 440 | api_key_env: Some(defaults.api_key_env), | 531 | api_key_env: Some(defaults.api_key_env), |
| 441 | base_url: defaults.base_url, | 532 | base_url: defaults.base_url, |
| 442 | input_k: defaults.input_k, | 533 | input_k: defaults.input_k, |
| 443 | - timeout_ms: defaults.timeout_ms, | 534 | + timeout_ms: Some(DEFAULT_RERANK_TIMEOUT_MS), |
| 444 | fail_open: defaults.fail_open, | 535 | fail_open: defaults.fail_open, |
| 445 | } | 536 | } |
| 446 | } | 537 | } |
| @@ -485,8 +576,10 @@ impl RerankServiceConfig { | |||
| 485 | if !(1..=500).contains(&self.input_k) { | 576 | if !(1..=500).contains(&self.input_k) { |
| 486 | anyhow::bail!("rerank input_k must be between 1 and 500"); | 577 | anyhow::bail!("rerank input_k must be between 1 and 500"); |
| 487 | } | 578 | } |
| 488 | - if self.timeout_ms == Some(0) { | 579 | + if !matches!(self.timeout_ms, Some(1..=MAX_RERANK_TIMEOUT_MS)) { |
| 489 | - anyhow::bail!("rerank timeout_ms must be non-zero when configured"); | 580 | + anyhow::bail!( |
| 581 | + "enabled rerank timeout_ms must be between 1 and {MAX_RERANK_TIMEOUT_MS}" | ||
| 582 | + ); | ||
| 490 | } | 583 | } |
| 491 | Ok(()) | 584 | Ok(()) |
| 492 | } | 585 | } |
| @@ -572,6 +665,30 @@ impl CaseServiceConfig { | |||
| 572 | } | 665 | } |
| 573 | 666 | ||
| 574 | impl CaseLibraryServiceConfig { | 667 | impl CaseLibraryServiceConfig { |
| 668 | + pub fn resolved_embedding_api_key_env<'a>(&'a self, providers: &'a ProvidersConfig) -> &'a str { | ||
| 669 | + self.embedding_api_key_env | ||
| 670 | + .as_deref() | ||
| 671 | + .unwrap_or(&providers.api_key_env) | ||
| 672 | + } | ||
| 673 | + | ||
| 674 | + pub fn resolved_embedding_base_url<'a>(&'a self, providers: &'a ProvidersConfig) -> &'a str { | ||
| 675 | + self.embedding_base_url | ||
| 676 | + .as_deref() | ||
| 677 | + .unwrap_or(&providers.base_url) | ||
| 678 | + } | ||
| 679 | + | ||
| 680 | + pub fn resolved_summary_api_key_env<'a>(&'a self, providers: &'a ProvidersConfig) -> &'a str { | ||
| 681 | + self.summary_llm_api_key_env | ||
| 682 | + .as_deref() | ||
| 683 | + .unwrap_or(&providers.api_key_env) | ||
| 684 | + } | ||
| 685 | + | ||
| 686 | + pub fn resolved_summary_base_url<'a>(&'a self, providers: &'a ProvidersConfig) -> &'a str { | ||
| 687 | + self.summary_llm_base_url | ||
| 688 | + .as_deref() | ||
| 689 | + .unwrap_or(&providers.base_url) | ||
| 690 | + } | ||
| 691 | + | ||
| 575 | pub fn validate(&self, memory_database_path: Option<&Path>) -> Result<()> { | 692 | pub fn validate(&self, memory_database_path: Option<&Path>) -> Result<()> { |
| 576 | validate_case_library_mappings(self.default_library.as_str(), &self.libraries)?; | 693 | validate_case_library_mappings(self.default_library.as_str(), &self.libraries)?; |
| 577 | if self.rag_store.as_os_str().is_empty() | 694 | if self.rag_store.as_os_str().is_empty() |
| @@ -627,6 +744,13 @@ impl CaseLibraryServiceConfig { | |||
| 627 | if let Some(summary_base_url) = self.summary_llm_base_url.as_deref() { | 744 | if let Some(summary_base_url) = self.summary_llm_base_url.as_deref() { |
| 628 | validate_provider_base_url(summary_base_url, "case library summary LLM base URL")?; | 745 | validate_provider_base_url(summary_base_url, "case library summary LLM base URL")?; |
| 629 | } | 746 | } |
| 747 | + if self | ||
| 748 | + .summary_llm_model | ||
| 749 | + .as_deref() | ||
| 750 | + .is_some_and(|model| model.trim().is_empty()) | ||
| 751 | + { | ||
| 752 | + anyhow::bail!("case library summary LLM model must not be empty when configured"); | ||
| 753 | + } | ||
| 630 | Ok(()) | 754 | Ok(()) |
| 631 | } | 755 | } |
| 632 | } | 756 | } |
| @@ -660,7 +784,7 @@ fn is_loopback_host(value: &str) -> bool { | |||
| 660 | } | 784 | } |
| 661 | 785 | ||
| 662 | fn default_max_body_bytes() -> usize { | 786 | fn default_max_body_bytes() -> usize { |
| 663 | - 1_048_576 | 787 | + 16 * 1024 * 1024 |
| 664 | } | 788 | } |
| 665 | 789 | ||
| 666 | fn default_requests_per_second() -> u32 { | 790 | fn default_requests_per_second() -> u32 { |
| @@ -762,7 +886,10 @@ impl ServerConfig { | |||
| 762 | 886 | ||
| 763 | pub fn validate_runtime(&self) -> Result<()> { | 887 | pub fn validate_runtime(&self) -> Result<()> { |
| 764 | self.http.validate_bind()?; | 888 | self.http.validate_bind()?; |
| 889 | + self.limits.validate()?; | ||
| 890 | + self.pipeline.validate()?; | ||
| 765 | self.retrieval.validate()?; | 891 | self.retrieval.validate()?; |
| 892 | + self.auth.validate()?; | ||
| 766 | if self.features.case_library.enabled == Some(true) && self.case_library.is_none() { | 893 | if self.features.case_library.enabled == Some(true) && self.case_library.is_none() { |
| 767 | anyhow::bail!("case_library feature requires case_library configuration"); | 894 | anyhow::bail!("case_library feature requires case_library configuration"); |
| 768 | } | 895 | } |
| @@ -772,21 +899,6 @@ impl ServerConfig { | |||
| 772 | if self.features.graph_memory.enabled && !self.features.memory.enabled { | 899 | if self.features.graph_memory.enabled && !self.features.memory.enabled { |
| 773 | anyhow::bail!("graph_memory feature requires the memory feature"); | 900 | anyhow::bail!("graph_memory feature requires the memory feature"); |
| 774 | } | 901 | } |
| 775 | - if self.auth.tokens.is_empty() { | ||
| 776 | - anyhow::bail!("production runtime requires at least one authenticated principal"); | ||
| 777 | - } | ||
| 778 | - if self.limits.max_body_bytes == 0 | ||
| 779 | - || self.limits.requests_per_second == 0 | ||
| 780 | - || self.limits.rate_burst == 0 | ||
| 781 | - || self.limits.max_in_flight_per_principal_tool == 0 | ||
| 782 | - || self.limits.initialize_requests_per_second == 0 | ||
| 783 | - || self.limits.initialize_rate_burst == 0 | ||
| 784 | - || self.limits.max_active_sessions_per_principal == 0 | ||
| 785 | - || self.limits.max_active_sessions_global == 0 | ||
| 786 | - || self.limits.session_idle_timeout_seconds == 0 | ||
| 787 | - { | ||
| 788 | - anyhow::bail!("HTTP limits must all be non-zero"); | ||
| 789 | - } | ||
| 790 | let storage = self | 902 | let storage = self |
| 791 | .storage | 903 | .storage |
| 792 | .as_ref() | 904 | .as_ref() |
| @@ -836,48 +948,24 @@ impl ServerConfig { | |||
| 836 | } | 948 | } |
| 837 | if providers.embedding_provider == EmbeddingProviderKind::OpenAiCompatible { | 949 | if providers.embedding_provider == EmbeddingProviderKind::OpenAiCompatible { |
| 838 | validate_authenticated_provider_base_url( | 950 | validate_authenticated_provider_base_url( |
| 839 | - providers | 951 | + providers.resolved_embedding_base_url(), |
| 840 | - .embedding_base_url | ||
| 841 | - .as_deref() | ||
| 842 | - .unwrap_or(&providers.base_url), | ||
| 843 | "embedding base URL", | 952 | "embedding base URL", |
| 844 | - Some( | 953 | + Some(providers.resolved_embedding_api_key_env()), |
| 845 | - providers | ||
| 846 | - .embedding_api_key_env | ||
| 847 | - .as_deref() | ||
| 848 | - .unwrap_or(&providers.api_key_env), | ||
| 849 | - ), | ||
| 850 | )?; | 954 | )?; |
| 851 | } | 955 | } |
| 852 | if let Some(case_library) = &self.case_library { | 956 | if let Some(case_library) = &self.case_library { |
| 853 | if case_library.embedding_provider == EmbeddingProviderKind::OpenAiCompatible { | 957 | if case_library.embedding_provider == EmbeddingProviderKind::OpenAiCompatible { |
| 854 | validate_authenticated_provider_base_url( | 958 | validate_authenticated_provider_base_url( |
| 855 | - case_library | 959 | + case_library.resolved_embedding_base_url(providers), |
| 856 | - .embedding_base_url | ||
| 857 | - .as_deref() | ||
| 858 | - .unwrap_or(&providers.base_url), | ||
| 859 | "case library embedding base URL", | 960 | "case library embedding base URL", |
| 860 | - Some( | 961 | + Some(case_library.resolved_embedding_api_key_env(providers)), |
| 861 | - case_library | ||
| 862 | - .embedding_api_key_env | ||
| 863 | - .as_deref() | ||
| 864 | - .unwrap_or(&providers.api_key_env), | ||
| 865 | - ), | ||
| 866 | )?; | 962 | )?; |
| 867 | } | 963 | } |
| 868 | if case_library.summary_llm_model.is_some() { | 964 | if case_library.summary_llm_model.is_some() { |
| 869 | validate_authenticated_provider_base_url( | 965 | validate_authenticated_provider_base_url( |
| 870 | - case_library | 966 | + case_library.resolved_summary_base_url(providers), |
| 871 | - .summary_llm_base_url | ||
| 872 | - .as_deref() | ||
| 873 | - .unwrap_or(&providers.base_url), | ||
| 874 | "case library summary LLM base URL", | 967 | "case library summary LLM base URL", |
| 875 | - Some( | 968 | + Some(case_library.resolved_summary_api_key_env(providers)), |
| 876 | - case_library | ||
| 877 | - .summary_llm_api_key_env | ||
| 878 | - .as_deref() | ||
| 879 | - .unwrap_or(&providers.api_key_env), | ||
| 880 | - ), | ||
| 881 | )?; | 969 | )?; |
| 882 | } | 970 | } |
| 883 | } | 971 | } |
| @@ -981,11 +1069,736 @@ fn is_loopback_or_private(value: &url::Url) -> bool { | |||
| 981 | 1069 | ||
| 982 | mod tests { | 1070 | mod tests { |
| 983 | use super::{ | 1071 | use super::{ |
| 984 | - validate_provider_base_url, EmbeddingProviderKind, RerankServiceConfig, | 1072 | + validate_provider_base_url, AuthConfig, CaseLibraryServiceConfig, EmbeddingProviderKind, |
| 985 | - RetrievalServiceConfig, ServerConfig, | 1073 | + FeaturesConfig, GraphMemoryRetrievalConfig, GraphMemoryServiceConfig, HttpConfig, |
| 1074 | + LimitsConfig, PipelineServiceConfig, ProvidersConfig, RerankServiceConfig, | ||
| 1075 | + RetrievalServiceConfig, ServerConfig, TokenConfig, DEFAULT_RERANK_TIMEOUT_MS, | ||
| 1076 | + MAX_ACTIVE_SESSIONS_GLOBAL, MAX_ACTIVE_SESSIONS_PER_PRINCIPAL, MAX_INITIALIZE_RATE_BURST, | ||
| 1077 | + MAX_INITIALIZE_REQUESTS_PER_SECOND, MAX_IN_FLIGHT_PER_PRINCIPAL_TOOL, MAX_MCP_BODY_BYTES, | ||
| 1078 | + MAX_RATE_BURST, MAX_REQUESTS_PER_SECOND, MAX_RERANK_TIMEOUT_MS, | ||
| 1079 | + MAX_SESSION_IDLE_TIMEOUT_SECONDS, | ||
| 986 | }; | 1080 | }; |
| 987 | use memory_core::SearchMode; | 1081 | use memory_core::SearchMode; |
| 988 | 1082 | ||
| 1083 | + | ||
| 1084 | + fn feature_http_and_provider_defaults_are_stable() { | ||
| 1085 | + let features = FeaturesConfig::default(); | ||
| 1086 | + assert!(features.memory.enabled); | ||
| 1087 | + assert_eq!(features.case_library.enabled, None); | ||
| 1088 | + assert!(!features.graph_memory.enabled); | ||
| 1089 | + | ||
| 1090 | + let http = HttpConfig::default(); | ||
| 1091 | + assert_eq!(http.bind_address.to_string(), "127.0.0.1"); | ||
| 1092 | + assert_eq!(http.port, 8080); | ||
| 1093 | + assert!(http.allowed_origins.is_empty()); | ||
| 1094 | + assert_eq!(http.allowed_hosts, ["localhost", "127.0.0.1", "::1"]); | ||
| 1095 | + assert!(!http.tls_termination_acknowledged); | ||
| 1096 | + assert!(http.validate_bind().is_ok()); | ||
| 1097 | + | ||
| 1098 | + let providers: ProvidersConfig = serde_json::from_value(serde_json::json!({ | ||
| 1099 | + "api_key_env": "RAM_A_PROVIDER_KEY", | ||
| 1100 | + "embedding_model": "embedding-model", | ||
| 1101 | + "embedding_dimensions": 1024, | ||
| 1102 | + "extractor_model": "extractor-model", | ||
| 1103 | + "verifier_model": "verifier-model" | ||
| 1104 | + })) | ||
| 1105 | + .expect("parse provider defaults"); | ||
| 1106 | + assert_eq!(providers.base_url, "https://openrouter.ai/api/v1"); | ||
| 1107 | + assert_eq!( | ||
| 1108 | + providers.embedding_provider, | ||
| 1109 | + EmbeddingProviderKind::OpenAiCompatible | ||
| 1110 | + ); | ||
| 1111 | + assert_eq!(providers.embedding_api_key_env, None); | ||
| 1112 | + assert_eq!(providers.embedding_base_url, None); | ||
| 1113 | + assert_eq!( | ||
| 1114 | + providers.resolved_embedding_api_key_env(), | ||
| 1115 | + "RAM_A_PROVIDER_KEY" | ||
| 1116 | + ); | ||
| 1117 | + assert_eq!( | ||
| 1118 | + providers.resolved_embedding_base_url(), | ||
| 1119 | + "https://openrouter.ai/api/v1" | ||
| 1120 | + ); | ||
| 1121 | + assert_eq!(providers.timeout_seconds, 120); | ||
| 1122 | + assert_eq!(providers.max_retries, 3); | ||
| 1123 | + | ||
| 1124 | + let minimal: ServerConfig = serde_json::from_value(serde_json::json!({ | ||
| 1125 | + "auth": {"tokens": []} | ||
| 1126 | + })) | ||
| 1127 | + .expect("parse top-level defaults"); | ||
| 1128 | + assert!(minimal.features.memory.enabled); | ||
| 1129 | + assert_eq!(minimal.http.port, 8080); | ||
| 1130 | + assert_eq!(minimal.limits.max_body_bytes, 16 * 1024 * 1024); | ||
| 1131 | + assert!(minimal.pipeline.fail_fast); | ||
| 1132 | + assert_eq!(minimal.pipeline.max_memory_chars, 500); | ||
| 1133 | + assert!(minimal.storage.is_none()); | ||
| 1134 | + assert!(minimal.providers.is_none()); | ||
| 1135 | + assert_eq!(minimal.retrieval.mode, SearchMode::Hybrid); | ||
| 1136 | + assert!(minimal.case_library.is_none()); | ||
| 1137 | + assert!(minimal.graph_memory.is_none()); | ||
| 1138 | + } | ||
| 1139 | + | ||
| 1140 | + | ||
| 1141 | + fn graph_and_case_library_defaults_are_stable() { | ||
| 1142 | + let graph: GraphMemoryServiceConfig = serde_json::from_value(serde_json::json!({ | ||
| 1143 | + "llm_api_key_env": "RAM_A_GRAPH_KEY", | ||
| 1144 | + "llm_model": "graph-model" | ||
| 1145 | + })) | ||
| 1146 | + .expect("parse graph defaults"); | ||
| 1147 | + assert_eq!(graph.llm_base_url, "https://openrouter.ai/api/v1"); | ||
| 1148 | + assert_eq!(graph.llm_timeout_ms, 60_000); | ||
| 1149 | + assert_eq!(graph.build_concurrency, 1); | ||
| 1150 | + assert_eq!( | ||
| 1151 | + graph.retrieval, | ||
| 1152 | + GraphMemoryRetrievalConfig { | ||
| 1153 | + weight: 0.2, | ||
| 1154 | + rerank_with_graph: false, | ||
| 1155 | + allow_graph_only: false, | ||
| 1156 | + max_graph_only_results: None, | ||
| 1157 | + seed_limit: None, | ||
| 1158 | + max_evidence_records_per_fact: None, | ||
| 1159 | + fail_open: false, | ||
| 1160 | + } | ||
| 1161 | + ); | ||
| 1162 | + assert!(graph.validate().is_ok()); | ||
| 1163 | + | ||
| 1164 | + let cases: CaseLibraryServiceConfig = serde_json::from_value(serde_json::json!({ | ||
| 1165 | + "default_library": "ops", | ||
| 1166 | + "libraries": [{ | ||
| 1167 | + "name": "ops", | ||
| 1168 | + "dataset_id": "ops-dataset", | ||
| 1169 | + "tenant_ids": ["tenant-a"] | ||
| 1170 | + }] | ||
| 1171 | + })) | ||
| 1172 | + .expect("parse case library defaults"); | ||
| 1173 | + assert_eq!( | ||
| 1174 | + cases.rag_store.to_string_lossy(), | ||
| 1175 | + "data/memory-cases.sqlite" | ||
| 1176 | + ); | ||
| 1177 | + assert_eq!( | ||
| 1178 | + cases.index_store.to_string_lossy(), | ||
| 1179 | + "data/memory-cases-index.sqlite" | ||
| 1180 | + ); | ||
| 1181 | + assert_eq!(cases.source_dir, None); | ||
| 1182 | + assert_eq!(cases.api_token_env, None); | ||
| 1183 | + assert_eq!(cases.ingestion_poll_ms, 1_000); | ||
| 1184 | + assert_eq!( | ||
| 1185 | + cases.embedding_provider, | ||
| 1186 | + EmbeddingProviderKind::OpenAiCompatible | ||
| 1187 | + ); | ||
| 1188 | + assert_eq!(cases.embedding_api_key_env, None); | ||
| 1189 | + assert_eq!(cases.embedding_base_url, None); | ||
| 1190 | + assert_eq!(cases.embedding_model, "hash"); | ||
| 1191 | + assert_eq!(cases.embedding_dimensions, 1_024); | ||
| 1192 | + assert_eq!(cases.chunk_size, 160); | ||
| 1193 | + assert_eq!(cases.summary_llm_model, None); | ||
| 1194 | + assert_eq!(cases.summary_llm_api_key_env, None); | ||
| 1195 | + assert_eq!(cases.summary_llm_base_url, None); | ||
| 1196 | + assert_eq!(cases.summary_llm_timeout_ms, 30_000); | ||
| 1197 | + assert!(cases | ||
| 1198 | + .validate(Some(std::path::Path::new("memory.sqlite"))) | ||
| 1199 | + .is_ok()); | ||
| 1200 | + } | ||
| 1201 | + | ||
| 1202 | + | ||
| 1203 | + fn graph_configuration_rejects_invalid_configurable_values() { | ||
| 1204 | + let valid: GraphMemoryServiceConfig = serde_json::from_value(serde_json::json!({ | ||
| 1205 | + "llm_api_key_env": "RAM_A_GRAPH_KEY", | ||
| 1206 | + "llm_model": "graph-model" | ||
| 1207 | + })) | ||
| 1208 | + .expect("parse graph config"); | ||
| 1209 | + | ||
| 1210 | + let mut invalid = Vec::new(); | ||
| 1211 | + let mut config = valid.clone(); | ||
| 1212 | + config.llm_api_key_env.clear(); | ||
| 1213 | + invalid.push(config); | ||
| 1214 | + let mut config = valid.clone(); | ||
| 1215 | + config.llm_model.clear(); | ||
| 1216 | + invalid.push(config); | ||
| 1217 | + let mut config = valid.clone(); | ||
| 1218 | + config.llm_api_key_env = " ".to_string(); | ||
| 1219 | + invalid.push(config); | ||
| 1220 | + let mut config = valid.clone(); | ||
| 1221 | + config.llm_model = " ".to_string(); | ||
| 1222 | + invalid.push(config); | ||
| 1223 | + let mut config = valid.clone(); | ||
| 1224 | + config.llm_base_url = "not-a-url".to_string(); | ||
| 1225 | + invalid.push(config); | ||
| 1226 | + let mut config = valid.clone(); | ||
| 1227 | + config.llm_timeout_ms = 0; | ||
| 1228 | + invalid.push(config); | ||
| 1229 | + let mut config = valid.clone(); | ||
| 1230 | + config.build_concurrency = 0; | ||
| 1231 | + invalid.push(config); | ||
| 1232 | + for weight in [-0.1, 1.1, f32::NAN] { | ||
| 1233 | + let mut config = valid.clone(); | ||
| 1234 | + config.retrieval.weight = weight; | ||
| 1235 | + invalid.push(config); | ||
| 1236 | + } | ||
| 1237 | + let mut config = valid.clone(); | ||
| 1238 | + config.retrieval.max_graph_only_results = Some(0); | ||
| 1239 | + invalid.push(config); | ||
| 1240 | + for seed_limit in [Some(0), Some(memory_core::MAX_GRAPH_SEED_LIMIT + 1)] { | ||
| 1241 | + let mut config = valid.clone(); | ||
| 1242 | + config.retrieval.seed_limit = seed_limit; | ||
| 1243 | + invalid.push(config); | ||
| 1244 | + } | ||
| 1245 | + for evidence_limit in [ | ||
| 1246 | + Some(0), | ||
| 1247 | + Some(memory_core::MAX_GRAPH_EVIDENCE_RECORDS_PER_FACT + 1), | ||
| 1248 | + ] { | ||
| 1249 | + let mut config = valid.clone(); | ||
| 1250 | + config.retrieval.max_evidence_records_per_fact = evidence_limit; | ||
| 1251 | + invalid.push(config); | ||
| 1252 | + } | ||
| 1253 | + | ||
| 1254 | + assert!(invalid.into_iter().all(|config| config.validate().is_err())); | ||
| 1255 | + } | ||
| 1256 | + | ||
| 1257 | + | ||
| 1258 | + fn graph_configuration_accepts_all_documented_boundaries() { | ||
| 1259 | + let base: GraphMemoryServiceConfig = serde_json::from_value(serde_json::json!({ | ||
| 1260 | + "llm_api_key_env": "RAM_A_GRAPH_KEY", | ||
| 1261 | + "llm_model": "graph-model" | ||
| 1262 | + })) | ||
| 1263 | + .expect("parse graph config"); | ||
| 1264 | + | ||
| 1265 | + for weight in [0.0, 1.0] { | ||
| 1266 | + let mut config = base.clone(); | ||
| 1267 | + config.llm_timeout_ms = 1; | ||
| 1268 | + config.build_concurrency = 1; | ||
| 1269 | + config.retrieval.weight = weight; | ||
| 1270 | + config.retrieval.max_graph_only_results = Some(1); | ||
| 1271 | + config.retrieval.seed_limit = Some(memory_core::MAX_GRAPH_SEED_LIMIT); | ||
| 1272 | + config.retrieval.max_evidence_records_per_fact = | ||
| 1273 | + Some(memory_core::MAX_GRAPH_EVIDENCE_RECORDS_PER_FACT); | ||
| 1274 | + assert!(config.validate().is_ok(), "weight={weight}"); | ||
| 1275 | + } | ||
| 1276 | + | ||
| 1277 | + let mut lower = base; | ||
| 1278 | + lower.retrieval.seed_limit = Some(1); | ||
| 1279 | + lower.retrieval.max_evidence_records_per_fact = Some(1); | ||
| 1280 | + assert!(lower.validate().is_ok()); | ||
| 1281 | + } | ||
| 1282 | + | ||
| 1283 | + | ||
| 1284 | + fn case_library_configuration_rejects_invalid_configurable_values() { | ||
| 1285 | + let valid: CaseLibraryServiceConfig = serde_json::from_value(serde_json::json!({ | ||
| 1286 | + "default_library": "ops", | ||
| 1287 | + "libraries": [{ | ||
| 1288 | + "name": "ops", | ||
| 1289 | + "dataset_id": "ops-dataset", | ||
| 1290 | + "tenant_ids": ["tenant-a"] | ||
| 1291 | + }] | ||
| 1292 | + })) | ||
| 1293 | + .expect("parse case library config"); | ||
| 1294 | + | ||
| 1295 | + let validate = |config: &CaseLibraryServiceConfig| { | ||
| 1296 | + config.validate(Some(std::path::Path::new("memory.sqlite"))) | ||
| 1297 | + }; | ||
| 1298 | + let mut invalid = Vec::new(); | ||
| 1299 | + let mut config = valid.clone(); | ||
| 1300 | + config.rag_store = ":memory:".into(); | ||
| 1301 | + invalid.push(config); | ||
| 1302 | + let mut config = valid.clone(); | ||
| 1303 | + config.index_store = config.rag_store.clone(); | ||
| 1304 | + invalid.push(config); | ||
| 1305 | + let mut config = valid.clone(); | ||
| 1306 | + config.index_store = "memory.sqlite".into(); | ||
| 1307 | + invalid.push(config); | ||
| 1308 | + let mut config = valid.clone(); | ||
| 1309 | + config.source_dir = Some("".into()); | ||
| 1310 | + invalid.push(config); | ||
| 1311 | + let mut config = valid.clone(); | ||
| 1312 | + config.api_token_env = Some(" RAM_A_CASE_TOKEN".to_string()); | ||
| 1313 | + invalid.push(config); | ||
| 1314 | + let mut config = valid.clone(); | ||
| 1315 | + config.ingestion_poll_ms = 0; | ||
| 1316 | + invalid.push(config); | ||
| 1317 | + let mut config = valid.clone(); | ||
| 1318 | + config.embedding_model.clear(); | ||
| 1319 | + invalid.push(config); | ||
| 1320 | + let mut config = valid.clone(); | ||
| 1321 | + config.embedding_dimensions = 0; | ||
| 1322 | + invalid.push(config); | ||
| 1323 | + let mut config = valid.clone(); | ||
| 1324 | + config.chunk_size = 0; | ||
| 1325 | + invalid.push(config); | ||
| 1326 | + let mut config = valid.clone(); | ||
| 1327 | + config.summary_llm_timeout_ms = 0; | ||
| 1328 | + invalid.push(config); | ||
| 1329 | + let mut config = valid.clone(); | ||
| 1330 | + config.summary_llm_model = Some(" ".to_string()); | ||
| 1331 | + invalid.push(config); | ||
| 1332 | + let mut config = valid.clone(); | ||
| 1333 | + config.embedding_api_key_env = Some(" ".to_string()); | ||
| 1334 | + invalid.push(config); | ||
| 1335 | + let mut config = valid.clone(); | ||
| 1336 | + config.summary_llm_api_key_env = Some(" ".to_string()); | ||
| 1337 | + invalid.push(config); | ||
| 1338 | + let mut config = valid.clone(); | ||
| 1339 | + config.default_library = "missing".to_string(); | ||
| 1340 | + invalid.push(config); | ||
| 1341 | + let mut config = valid.clone(); | ||
| 1342 | + config.libraries.push(config.libraries[0].clone()); | ||
| 1343 | + invalid.push(config); | ||
| 1344 | + | ||
| 1345 | + assert!(invalid.into_iter().all(|config| validate(&config).is_err())); | ||
| 1346 | + } | ||
| 1347 | + | ||
| 1348 | + | ||
| 1349 | + fn case_library_positive_only_fields_accept_one() { | ||
| 1350 | + let mut config: CaseLibraryServiceConfig = serde_json::from_value(serde_json::json!({ | ||
| 1351 | + "default_library": "ops", | ||
| 1352 | + "libraries": [{ | ||
| 1353 | + "name": "ops", | ||
| 1354 | + "dataset_id": "ops-dataset", | ||
| 1355 | + "tenant_ids": ["tenant-a"] | ||
| 1356 | + }] | ||
| 1357 | + })) | ||
| 1358 | + .expect("parse case library config"); | ||
| 1359 | + config.ingestion_poll_ms = 1; | ||
| 1360 | + config.embedding_dimensions = 1; | ||
| 1361 | + config.chunk_size = 1; | ||
| 1362 | + config.summary_llm_timeout_ms = 1; | ||
| 1363 | + assert!(config | ||
| 1364 | + .validate(Some(std::path::Path::new("memory.sqlite"))) | ||
| 1365 | + .is_ok()); | ||
| 1366 | + } | ||
| 1367 | + | ||
| 1368 | + | ||
| 1369 | + fn provider_configuration_rejects_incomplete_configurable_values() { | ||
| 1370 | + let valid = packaged_config(); | ||
| 1371 | + let mut invalid = Vec::new(); | ||
| 1372 | + for field in [ | ||
| 1373 | + "api_key_env", | ||
| 1374 | + "base_url", | ||
| 1375 | + "embedding_model", | ||
| 1376 | + "extractor_model", | ||
| 1377 | + "verifier_model", | ||
| 1378 | + ] { | ||
| 1379 | + let mut config = valid.clone(); | ||
| 1380 | + let providers = config.providers.as_mut().expect("providers"); | ||
| 1381 | + match field { | ||
| 1382 | + "api_key_env" => providers.api_key_env.clear(), | ||
| 1383 | + "base_url" => providers.base_url.clear(), | ||
| 1384 | + "embedding_model" => providers.embedding_model.clear(), | ||
| 1385 | + "extractor_model" => providers.extractor_model.clear(), | ||
| 1386 | + "verifier_model" => providers.verifier_model.clear(), | ||
| 1387 | + _ => unreachable!(), | ||
| 1388 | + } | ||
| 1389 | + invalid.push(config); | ||
| 1390 | + } | ||
| 1391 | + let mut config = valid.clone(); | ||
| 1392 | + config | ||
| 1393 | + .providers | ||
| 1394 | + .as_mut() | ||
| 1395 | + .expect("providers") | ||
| 1396 | + .embedding_dimensions = 0; | ||
| 1397 | + invalid.push(config); | ||
| 1398 | + let mut config = valid.clone(); | ||
| 1399 | + config | ||
| 1400 | + .providers | ||
| 1401 | + .as_mut() | ||
| 1402 | + .expect("providers") | ||
| 1403 | + .timeout_seconds = 0; | ||
| 1404 | + invalid.push(config); | ||
| 1405 | + let mut config = valid; | ||
| 1406 | + config.providers.as_mut().expect("providers").max_retries = 0; | ||
| 1407 | + invalid.push(config); | ||
| 1408 | + let mut config = packaged_config(); | ||
| 1409 | + config | ||
| 1410 | + .providers | ||
| 1411 | + .as_mut() | ||
| 1412 | + .expect("providers") | ||
| 1413 | + .embedding_api_key_env = Some(" ".to_string()); | ||
| 1414 | + invalid.push(config); | ||
| 1415 | + let mut config = packaged_config(); | ||
| 1416 | + config | ||
| 1417 | + .providers | ||
| 1418 | + .as_mut() | ||
| 1419 | + .expect("providers") | ||
| 1420 | + .embedding_base_url = Some("not-a-url".to_string()); | ||
| 1421 | + invalid.push(config); | ||
| 1422 | + | ||
| 1423 | + assert!(invalid | ||
| 1424 | + .into_iter() | ||
| 1425 | + .all(|config| config.validate_runtime().is_err())); | ||
| 1426 | + } | ||
| 1427 | + | ||
| 1428 | + | ||
| 1429 | + fn provider_positive_only_fields_accept_one() { | ||
| 1430 | + let mut config = packaged_config(); | ||
| 1431 | + let providers = config.providers.as_mut().expect("providers"); | ||
| 1432 | + providers.embedding_dimensions = 1; | ||
| 1433 | + providers.timeout_seconds = 1; | ||
| 1434 | + providers.max_retries = 1; | ||
| 1435 | + assert!(config.validate_runtime().is_ok()); | ||
| 1436 | + } | ||
| 1437 | + | ||
| 1438 | + | ||
| 1439 | + fn provider_and_case_library_fallbacks_are_explicit_and_overridable() { | ||
| 1440 | + let mut config = packaged_config(); | ||
| 1441 | + let providers = config.providers.as_mut().expect("providers"); | ||
| 1442 | + providers.api_key_env = "PRIMARY_KEY".to_string(); | ||
| 1443 | + providers.base_url = "https://primary.example/v1".to_string(); | ||
| 1444 | + providers.embedding_api_key_env = None; | ||
| 1445 | + providers.embedding_base_url = None; | ||
| 1446 | + assert_eq!(providers.resolved_embedding_api_key_env(), "PRIMARY_KEY"); | ||
| 1447 | + assert_eq!( | ||
| 1448 | + providers.resolved_embedding_base_url(), | ||
| 1449 | + "https://primary.example/v1" | ||
| 1450 | + ); | ||
| 1451 | + | ||
| 1452 | + let cases = config.case_library.as_ref().expect("case library"); | ||
| 1453 | + assert_eq!( | ||
| 1454 | + cases.resolved_embedding_api_key_env(providers), | ||
| 1455 | + "PRIMARY_KEY" | ||
| 1456 | + ); | ||
| 1457 | + assert_eq!( | ||
| 1458 | + cases.resolved_embedding_base_url(providers), | ||
| 1459 | + "https://primary.example/v1" | ||
| 1460 | + ); | ||
| 1461 | + assert_eq!(cases.resolved_summary_api_key_env(providers), "PRIMARY_KEY"); | ||
| 1462 | + assert_eq!( | ||
| 1463 | + cases.resolved_summary_base_url(providers), | ||
| 1464 | + "https://primary.example/v1" | ||
| 1465 | + ); | ||
| 1466 | + | ||
| 1467 | + let providers = config.providers.as_mut().expect("providers"); | ||
| 1468 | + providers.embedding_api_key_env = Some("EMBEDDING_KEY".to_string()); | ||
| 1469 | + providers.embedding_base_url = Some("https://embedding.example/v1".to_string()); | ||
| 1470 | + let cases = config.case_library.as_mut().expect("case library"); | ||
| 1471 | + cases.embedding_api_key_env = Some("CASE_EMBEDDING_KEY".to_string()); | ||
| 1472 | + cases.embedding_base_url = Some("https://case-embedding.example/v1".to_string()); | ||
| 1473 | + cases.summary_llm_model = Some("summary-model".to_string()); | ||
| 1474 | + cases.summary_llm_api_key_env = Some("SUMMARY_KEY".to_string()); | ||
| 1475 | + cases.summary_llm_base_url = Some("https://summary.example/v1".to_string()); | ||
| 1476 | + | ||
| 1477 | + assert_eq!(providers.resolved_embedding_api_key_env(), "EMBEDDING_KEY"); | ||
| 1478 | + assert_eq!( | ||
| 1479 | + providers.resolved_embedding_base_url(), | ||
| 1480 | + "https://embedding.example/v1" | ||
| 1481 | + ); | ||
| 1482 | + assert_eq!( | ||
| 1483 | + cases.resolved_embedding_api_key_env(providers), | ||
| 1484 | + "CASE_EMBEDDING_KEY" | ||
| 1485 | + ); | ||
| 1486 | + assert_eq!( | ||
| 1487 | + cases.resolved_embedding_base_url(providers), | ||
| 1488 | + "https://case-embedding.example/v1" | ||
| 1489 | + ); | ||
| 1490 | + assert_eq!(cases.resolved_summary_api_key_env(providers), "SUMMARY_KEY"); | ||
| 1491 | + assert_eq!( | ||
| 1492 | + cases.resolved_summary_base_url(providers), | ||
| 1493 | + "https://summary.example/v1" | ||
| 1494 | + ); | ||
| 1495 | + assert!(config.validate_runtime().is_ok()); | ||
| 1496 | + } | ||
| 1497 | + | ||
| 1498 | + | ||
| 1499 | + fn storage_configuration_rejects_nonpersistent_paths_and_accepts_file_paths() { | ||
| 1500 | + for database_path in ["", ":memory:"] { | ||
| 1501 | + let mut config = packaged_config(); | ||
| 1502 | + config.storage.as_mut().expect("storage").database_path = database_path.into(); | ||
| 1503 | + assert!(config.validate_runtime().is_err(), "path={database_path}"); | ||
| 1504 | + } | ||
| 1505 | + | ||
| 1506 | + for database_path in [ | ||
| 1507 | + "data/ram-a-memory.sqlite", | ||
| 1508 | + "/var/lib/ram-a/ram-a-memory.sqlite", | ||
| 1509 | + ] { | ||
| 1510 | + let mut config = packaged_config(); | ||
| 1511 | + config.storage.as_mut().expect("storage").database_path = database_path.into(); | ||
| 1512 | + assert!(config.validate_runtime().is_ok(), "path={database_path}"); | ||
| 1513 | + } | ||
| 1514 | + } | ||
| 1515 | + | ||
| 1516 | + | ||
| 1517 | + fn case_library_paths_and_mappings_reject_every_invalid_shape() { | ||
| 1518 | + let valid = packaged_config(); | ||
| 1519 | + let mut invalid = Vec::new(); | ||
| 1520 | + | ||
| 1521 | + for field in ["rag_empty", "index_empty", "index_memory"] { | ||
| 1522 | + let mut config = valid.clone(); | ||
| 1523 | + let cases = config.case_library.as_mut().expect("case library"); | ||
| 1524 | + match field { | ||
| 1525 | + "rag_empty" => cases.rag_store = "".into(), | ||
| 1526 | + "index_empty" => cases.index_store = "".into(), | ||
| 1527 | + "index_memory" => cases.index_store = ":memory:".into(), | ||
| 1528 | + _ => unreachable!(), | ||
| 1529 | + } | ||
| 1530 | + invalid.push(config); | ||
| 1531 | + } | ||
| 1532 | + | ||
| 1533 | + for field in [ | ||
| 1534 | + "default_empty", | ||
| 1535 | + "libraries_empty", | ||
| 1536 | + "name_empty", | ||
| 1537 | + "name_noncanonical", | ||
| 1538 | + "dataset_empty", | ||
| 1539 | + "dataset_noncanonical", | ||
| 1540 | + "tenants_empty", | ||
| 1541 | + "tenant_empty", | ||
| 1542 | + "tenant_noncanonical", | ||
| 1543 | + ] { | ||
| 1544 | + let mut config = valid.clone(); | ||
| 1545 | + let cases = config.case_library.as_mut().expect("case library"); | ||
| 1546 | + match field { | ||
| 1547 | + "default_empty" => cases.default_library.clear(), | ||
| 1548 | + "libraries_empty" => cases.libraries.clear(), | ||
| 1549 | + "name_empty" => cases.libraries[0].name.clear(), | ||
| 1550 | + "name_noncanonical" => cases.libraries[0].name = " ops".to_string(), | ||
| 1551 | + "dataset_empty" => cases.libraries[0].dataset_id.clear(), | ||
| 1552 | + "dataset_noncanonical" => { | ||
| 1553 | + cases.libraries[0].dataset_id = "ops-dataset ".to_string() | ||
| 1554 | + } | ||
| 1555 | + "tenants_empty" => cases.libraries[0].tenant_ids.clear(), | ||
| 1556 | + "tenant_empty" => cases.libraries[0].tenant_ids[0].clear(), | ||
| 1557 | + "tenant_noncanonical" => { | ||
| 1558 | + cases.libraries[0].tenant_ids[0] = " tenant-local".to_string() | ||
| 1559 | + } | ||
| 1560 | + _ => unreachable!(), | ||
| 1561 | + } | ||
| 1562 | + invalid.push(config); | ||
| 1563 | + } | ||
| 1564 | + | ||
| 1565 | + for field in ["embedding_url", "summary_url"] { | ||
| 1566 | + let mut config = valid.clone(); | ||
| 1567 | + let cases = config.case_library.as_mut().expect("case library"); | ||
| 1568 | + match field { | ||
| 1569 | + "embedding_url" => cases.embedding_base_url = Some("not-a-url".to_string()), | ||
| 1570 | + "summary_url" => { | ||
| 1571 | + cases.summary_llm_model = Some("summary-model".to_string()); | ||
| 1572 | + cases.summary_llm_base_url = Some("not-a-url".to_string()); | ||
| 1573 | + } | ||
| 1574 | + _ => unreachable!(), | ||
| 1575 | + } | ||
| 1576 | + invalid.push(config); | ||
| 1577 | + } | ||
| 1578 | + | ||
| 1579 | + assert!(invalid | ||
| 1580 | + .into_iter() | ||
| 1581 | + .all(|config| config.validate_runtime().is_err())); | ||
| 1582 | + } | ||
| 1583 | + | ||
| 1584 | + | ||
| 1585 | + fn http_configuration_covers_host_and_port_boundaries() { | ||
| 1586 | + let default = HttpConfig::default(); | ||
| 1587 | + for port in [0_u16, 1, u16::MAX] { | ||
| 1588 | + let config: HttpConfig = serde_json::from_value(serde_json::json!({"port": port})) | ||
| 1589 | + .expect("port must fit u16"); | ||
| 1590 | + assert_eq!(config.port, port); | ||
| 1591 | + assert!(config.validate_bind().is_ok()); | ||
| 1592 | + } | ||
| 1593 | + assert!(serde_json::from_value::<HttpConfig>(serde_json::json!({"port": 65_536})).is_err()); | ||
| 1594 | + assert!(serde_json::from_value::<HttpConfig>(serde_json::json!({"port": -1})).is_err()); | ||
| 1595 | + | ||
| 1596 | + for allowed_hosts in [Vec::new(), vec![String::new()], vec![" ".to_string()]] { | ||
| 1597 | + let config = HttpConfig { | ||
| 1598 | + allowed_hosts, | ||
| 1599 | + ..default.clone() | ||
| 1600 | + }; | ||
| 1601 | + assert!(config.validate_bind().is_err()); | ||
| 1602 | + } | ||
| 1603 | + | ||
| 1604 | + let external = HttpConfig { | ||
| 1605 | + bind_address: "0.0.0.0".parse().unwrap(), | ||
| 1606 | + allowed_hosts: vec!["memory.example.test".to_string()], | ||
| 1607 | + tls_termination_acknowledged: true, | ||
| 1608 | + ..default | ||
| 1609 | + }; | ||
| 1610 | + assert!(external.validate_bind().is_ok()); | ||
| 1611 | + } | ||
| 1612 | + | ||
| 1613 | + | ||
| 1614 | + fn authentication_configuration_enforces_fixed_permissions_and_canonical_ids() { | ||
| 1615 | + let valid = AuthConfig { | ||
| 1616 | + tokens: vec![TokenConfig { | ||
| 1617 | + token_env: "RAM_A_TOKEN".to_string(), | ||
| 1618 | + tenant_id: "tenant-a".to_string(), | ||
| 1619 | + user_id: "alice".to_string(), | ||
| 1620 | + agent_id: "xiaoo".to_string(), | ||
| 1621 | + permissions: vec![ | ||
| 1622 | + "memory:read".to_string(), | ||
| 1623 | + "memory:write".to_string(), | ||
| 1624 | + "cases:read".to_string(), | ||
| 1625 | + "cases:write".to_string(), | ||
| 1626 | + ], | ||
| 1627 | + }], | ||
| 1628 | + }; | ||
| 1629 | + assert!(valid.validate().is_ok()); | ||
| 1630 | + | ||
| 1631 | + let mut unknown_permission = valid.clone(); | ||
| 1632 | + unknown_permission.tokens[0].permissions = vec!["memory:admin".to_string()]; | ||
| 1633 | + assert!(unknown_permission.validate().is_err()); | ||
| 1634 | + | ||
| 1635 | + let mut duplicate_permission = valid.clone(); | ||
| 1636 | + duplicate_permission.tokens[0].permissions = | ||
| 1637 | + vec!["memory:read".to_string(), "memory:read".to_string()]; | ||
| 1638 | + assert!(duplicate_permission.validate().is_err()); | ||
| 1639 | + | ||
| 1640 | + for field in ["token_env", "tenant_id", "user_id", "agent_id"] { | ||
| 1641 | + let mut noncanonical = valid.clone(); | ||
| 1642 | + match field { | ||
| 1643 | + "token_env" => noncanonical.tokens[0].token_env = " RAM_A_TOKEN".to_string(), | ||
| 1644 | + "tenant_id" => noncanonical.tokens[0].tenant_id = "tenant-a ".to_string(), | ||
| 1645 | + "user_id" => noncanonical.tokens[0].user_id.clear(), | ||
| 1646 | + "agent_id" => noncanonical.tokens[0].agent_id = " ".to_string(), | ||
| 1647 | + _ => unreachable!(), | ||
| 1648 | + } | ||
| 1649 | + assert!(noncanonical.validate().is_err(), "field={field}"); | ||
| 1650 | + } | ||
| 1651 | + | ||
| 1652 | + let mut duplicate_environment = valid.clone(); | ||
| 1653 | + duplicate_environment.tokens.push(valid.tokens[0].clone()); | ||
| 1654 | + assert!(duplicate_environment.validate().is_err()); | ||
| 1655 | + } | ||
| 1656 | + | ||
| 1657 | + | ||
| 1658 | + fn configurable_enums_reject_unsupported_values() { | ||
| 1659 | + assert!(serde_json::from_str::<EmbeddingProviderKind>(r#""hash""#).is_ok()); | ||
| 1660 | + assert!(serde_json::from_str::<EmbeddingProviderKind>(r#""openai_compatible""#).is_ok()); | ||
| 1661 | + assert_eq!( | ||
| 1662 | + serde_json::from_str::<EmbeddingProviderKind>(r#""open_router""#).unwrap(), | ||
| 1663 | + EmbeddingProviderKind::OpenAiCompatible | ||
| 1664 | + ); | ||
| 1665 | + assert!(serde_json::from_str::<EmbeddingProviderKind>(r#""unknown""#).is_err()); | ||
| 1666 | + assert!(serde_json::from_str::<SearchMode>(r#""dense""#).is_ok()); | ||
| 1667 | + assert!(serde_json::from_str::<SearchMode>(r#""bm25""#).is_ok()); | ||
| 1668 | + assert!(serde_json::from_str::<SearchMode>(r#""hybrid""#).is_ok()); | ||
| 1669 | + assert!(serde_json::from_str::<SearchMode>(r#""unknown""#).is_err()); | ||
| 1670 | + assert!(serde_json::from_str::<memory_core::RerankProvider>(r#""openrouter""#).is_ok()); | ||
| 1671 | + assert!(serde_json::from_str::<memory_core::RerankProvider>(r#""unknown""#).is_err()); | ||
| 1672 | + | ||
| 1673 | + let graph_mode = RetrievalServiceConfig { | ||
| 1674 | + mode: SearchMode::Graph, | ||
| 1675 | + ..RetrievalServiceConfig::default() | ||
| 1676 | + }; | ||
| 1677 | + assert!(graph_mode.validate().is_err()); | ||
| 1678 | + } | ||
| 1679 | + | ||
| 1680 | + | ||
| 1681 | + fn pipeline_defaults_and_boundaries_are_stable() { | ||
| 1682 | + let defaults = PipelineServiceConfig::default(); | ||
| 1683 | + assert!(defaults.fail_fast); | ||
| 1684 | + assert_eq!(defaults.max_memory_chars, 500); | ||
| 1685 | + assert!(defaults.validate().is_ok()); | ||
| 1686 | + | ||
| 1687 | + for max_memory_chars in [1, crate::MAX_MESSAGE_TEXT_CHARS] { | ||
| 1688 | + let config = PipelineServiceConfig { | ||
| 1689 | + fail_fast: false, | ||
| 1690 | + max_memory_chars, | ||
| 1691 | + }; | ||
| 1692 | + assert!(config.validate().is_ok()); | ||
| 1693 | + let pipeline = config.pipeline_config(); | ||
| 1694 | + assert!(!pipeline.fail_fast); | ||
| 1695 | + assert_eq!(pipeline.validation.max_memory_chars, max_memory_chars); | ||
| 1696 | + } | ||
| 1697 | + for max_memory_chars in [0, crate::MAX_MESSAGE_TEXT_CHARS + 1] { | ||
| 1698 | + assert!(PipelineServiceConfig { | ||
| 1699 | + fail_fast: true, | ||
| 1700 | + max_memory_chars, | ||
| 1701 | + } | ||
| 1702 | + .validate() | ||
| 1703 | + .is_err()); | ||
| 1704 | + } | ||
| 1705 | + } | ||
| 1706 | + | ||
| 1707 | + | ||
| 1708 | + fn http_limit_defaults_are_stable_and_supported() { | ||
| 1709 | + let limits = LimitsConfig::default(); | ||
| 1710 | + assert_eq!(limits.max_body_bytes, 16 * 1024 * 1024); | ||
| 1711 | + assert_eq!(limits.requests_per_second, 20); | ||
| 1712 | + assert_eq!(limits.rate_burst, 40); | ||
| 1713 | + assert_eq!(limits.max_in_flight_per_principal_tool, 4); | ||
| 1714 | + assert_eq!(limits.initialize_requests_per_second, 4); | ||
| 1715 | + assert_eq!(limits.initialize_rate_burst, 8); | ||
| 1716 | + assert_eq!(limits.max_active_sessions_per_principal, 8); | ||
| 1717 | + assert_eq!(limits.max_active_sessions_global, 256); | ||
| 1718 | + assert_eq!(limits.session_idle_timeout_seconds, 1_800); | ||
| 1719 | + assert!(limits.validate().is_ok()); | ||
| 1720 | + } | ||
| 1721 | + | ||
| 1722 | + | ||
| 1723 | + fn http_limits_accept_documented_upper_boundaries() { | ||
| 1724 | + let limits = LimitsConfig { | ||
| 1725 | + max_body_bytes: MAX_MCP_BODY_BYTES, | ||
| 1726 | + requests_per_second: MAX_REQUESTS_PER_SECOND, | ||
| 1727 | + rate_burst: MAX_RATE_BURST, | ||
| 1728 | + max_in_flight_per_principal_tool: MAX_IN_FLIGHT_PER_PRINCIPAL_TOOL, | ||
| 1729 | + initialize_requests_per_second: MAX_INITIALIZE_REQUESTS_PER_SECOND, | ||
| 1730 | + initialize_rate_burst: MAX_INITIALIZE_RATE_BURST, | ||
| 1731 | + max_active_sessions_per_principal: MAX_ACTIVE_SESSIONS_PER_PRINCIPAL, | ||
| 1732 | + max_active_sessions_global: MAX_ACTIVE_SESSIONS_GLOBAL, | ||
| 1733 | + session_idle_timeout_seconds: MAX_SESSION_IDLE_TIMEOUT_SECONDS, | ||
| 1734 | + }; | ||
| 1735 | + assert!(limits.validate().is_ok()); | ||
| 1736 | + } | ||
| 1737 | + | ||
| 1738 | + | ||
| 1739 | + fn http_limits_accept_documented_lower_boundaries() { | ||
| 1740 | + let limits = LimitsConfig { | ||
| 1741 | + max_body_bytes: 1, | ||
| 1742 | + requests_per_second: 1, | ||
| 1743 | + rate_burst: 1, | ||
| 1744 | + max_in_flight_per_principal_tool: 1, | ||
| 1745 | + initialize_requests_per_second: 1, | ||
| 1746 | + initialize_rate_burst: 1, | ||
| 1747 | + max_active_sessions_per_principal: 1, | ||
| 1748 | + max_active_sessions_global: 1, | ||
| 1749 | + session_idle_timeout_seconds: 1, | ||
| 1750 | + }; | ||
| 1751 | + assert!(limits.validate().is_ok()); | ||
| 1752 | + } | ||
| 1753 | + | ||
| 1754 | + | ||
| 1755 | + fn http_limits_reject_zero_out_of_range_and_inconsistent_sessions() { | ||
| 1756 | + let mut invalid = Vec::new(); | ||
| 1757 | + macro_rules! invalid_limit { | ||
| 1758 | + ($field:ident, $value:expr) => {{ | ||
| 1759 | + let mut limits = LimitsConfig::default(); | ||
| 1760 | + limits.$field = $value; | ||
| 1761 | + invalid.push(limits); | ||
| 1762 | + }}; | ||
| 1763 | + } | ||
| 1764 | + invalid_limit!(max_body_bytes, 0); | ||
| 1765 | + invalid_limit!(max_body_bytes, MAX_MCP_BODY_BYTES + 1); | ||
| 1766 | + invalid_limit!(requests_per_second, 0); | ||
| 1767 | + invalid_limit!(requests_per_second, MAX_REQUESTS_PER_SECOND + 1); | ||
| 1768 | + invalid_limit!(rate_burst, 0); | ||
| 1769 | + invalid_limit!(rate_burst, MAX_RATE_BURST + 1); | ||
| 1770 | + invalid_limit!(max_in_flight_per_principal_tool, 0); | ||
| 1771 | + invalid_limit!( | ||
| 1772 | + max_in_flight_per_principal_tool, | ||
| 1773 | + MAX_IN_FLIGHT_PER_PRINCIPAL_TOOL + 1 | ||
| 1774 | + ); | ||
| 1775 | + invalid_limit!(initialize_requests_per_second, 0); | ||
| 1776 | + invalid_limit!( | ||
| 1777 | + initialize_requests_per_second, | ||
| 1778 | + MAX_INITIALIZE_REQUESTS_PER_SECOND + 1 | ||
| 1779 | + ); | ||
| 1780 | + invalid_limit!(initialize_rate_burst, 0); | ||
| 1781 | + invalid_limit!(initialize_rate_burst, MAX_INITIALIZE_RATE_BURST + 1); | ||
| 1782 | + invalid_limit!(max_active_sessions_per_principal, 0); | ||
| 1783 | + invalid_limit!( | ||
| 1784 | + max_active_sessions_per_principal, | ||
| 1785 | + MAX_ACTIVE_SESSIONS_PER_PRINCIPAL + 1 | ||
| 1786 | + ); | ||
| 1787 | + invalid_limit!(max_active_sessions_global, 0); | ||
| 1788 | + invalid_limit!(max_active_sessions_global, MAX_ACTIVE_SESSIONS_GLOBAL + 1); | ||
| 1789 | + invalid_limit!(session_idle_timeout_seconds, 0); | ||
| 1790 | + invalid_limit!( | ||
| 1791 | + session_idle_timeout_seconds, | ||
| 1792 | + MAX_SESSION_IDLE_TIMEOUT_SECONDS + 1 | ||
| 1793 | + ); | ||
| 1794 | + let mut inconsistent = LimitsConfig::default(); | ||
| 1795 | + inconsistent.max_active_sessions_per_principal = 9; | ||
| 1796 | + inconsistent.max_active_sessions_global = 8; | ||
| 1797 | + invalid.push(inconsistent); | ||
| 1798 | + | ||
| 1799 | + assert!(invalid.into_iter().all(|limits| limits.validate().is_err())); | ||
| 1800 | + } | ||
| 1801 | + | ||
| 989 | 1802 | ||
| 990 | fn provider_base_url_rejects_credentials_query_and_fragment() { | 1803 | fn provider_base_url_rejects_credentials_query_and_fragment() { |
| 991 | for value in [ | 1804 | for value in [ |
| @@ -1014,6 +1827,19 @@ mod tests { | |||
| 1014 | assert_eq!(config.bm25_weight, 0.3); | 1827 | assert_eq!(config.bm25_weight, 0.3); |
| 1015 | assert_eq!(config.candidate_k, None); | 1828 | assert_eq!(config.candidate_k, None); |
| 1016 | assert!(!config.rerank.enabled); | 1829 | assert!(!config.rerank.enabled); |
| 1830 | + assert_eq!( | ||
| 1831 | + config.rerank.provider, | ||
| 1832 | + memory_core::RerankProvider::OpenRouter | ||
| 1833 | + ); | ||
| 1834 | + assert_eq!(config.rerank.model, "cohere/rerank-v3.5"); | ||
| 1835 | + assert_eq!( | ||
| 1836 | + config.rerank.api_key_env.as_deref(), | ||
| 1837 | + Some("OPENROUTER_API_KEY") | ||
| 1838 | + ); | ||
| 1839 | + assert_eq!(config.rerank.base_url, "https://openrouter.ai/api/v1"); | ||
| 1840 | + assert_eq!(config.rerank.input_k, 40); | ||
| 1841 | + assert_eq!(config.rerank.timeout_ms, Some(DEFAULT_RERANK_TIMEOUT_MS)); | ||
| 1842 | + assert!(!config.rerank.fail_open); | ||
| 1017 | assert!(config.validate().is_ok()); | 1843 | assert!(config.validate().is_ok()); |
| 1018 | } | 1844 | } |
| 1019 | 1845 | ||
| @@ -1145,12 +1971,23 @@ mod tests { | |||
| 1145 | 1971 | ||
| 1146 | 1972 | ||
| 1147 | fn retrieval_rejects_invalid_weights_and_non_hybrid_rerank() { | 1973 | fn retrieval_rejects_invalid_weights_and_non_hybrid_rerank() { |
| 1148 | - let invalid_weights = RetrievalServiceConfig { | 1974 | + for (embedding_weight, bm25_weight) in [ |
| 1149 | - embedding_weight: 0.8, | 1975 | + (0.8, 0.3), |
| 1150 | - bm25_weight: 0.3, | 1976 | + (-0.1, 1.1), |
| 1151 | - ..RetrievalServiceConfig::default() | 1977 | + (1.1, -0.1), |
| 1152 | - }; | 1978 | + (f32::NAN, 0.0), |
| 1153 | - assert!(invalid_weights.validate().is_err()); | 1979 | + (f32::INFINITY, 0.0), |
| 1980 | + ] { | ||
| 1981 | + let invalid_weights = RetrievalServiceConfig { | ||
| 1982 | + embedding_weight, | ||
| 1983 | + bm25_weight, | ||
| 1984 | + ..RetrievalServiceConfig::default() | ||
| 1985 | + }; | ||
| 1986 | + assert!( | ||
| 1987 | + invalid_weights.validate().is_err(), | ||
| 1988 | + "embedding_weight={embedding_weight}, bm25_weight={bm25_weight}" | ||
| 1989 | + ); | ||
| 1990 | + } | ||
| 1154 | 1991 | ||
| 1155 | let dense_rerank = RetrievalServiceConfig { | 1992 | let dense_rerank = RetrievalServiceConfig { |
| 1156 | mode: SearchMode::Dense, | 1993 | mode: SearchMode::Dense, |
| @@ -1163,11 +2000,160 @@ mod tests { | |||
| 1163 | assert!(dense_rerank.validate().is_err()); | 2000 | assert!(dense_rerank.validate().is_err()); |
| 1164 | } | 2001 | } |
| 1165 | 2002 | ||
| 2003 | + | ||
| 2004 | + fn retrieval_accepts_hybrid_weight_boundaries() { | ||
| 2005 | + for (embedding_weight, bm25_weight) in [(0.0, 1.0), (1.0, 0.0), (0.7, 0.3)] { | ||
| 2006 | + let config = RetrievalServiceConfig { | ||
| 2007 | + embedding_weight, | ||
| 2008 | + bm25_weight, | ||
| 2009 | + ..RetrievalServiceConfig::default() | ||
| 2010 | + }; | ||
| 2011 | + assert!(config.validate().is_ok()); | ||
| 2012 | + } | ||
| 2013 | + } | ||
| 2014 | + | ||
| 2015 | + | ||
| 2016 | + fn disabled_rerank_ignores_inactive_provider_fields() { | ||
| 2017 | + let config = RetrievalServiceConfig { | ||
| 2018 | + rerank: RerankServiceConfig { | ||
| 2019 | + enabled: false, | ||
| 2020 | + model: String::new(), | ||
| 2021 | + api_key_env: Some(String::new()), | ||
| 2022 | + base_url: String::new(), | ||
| 2023 | + input_k: 0, | ||
| 2024 | + timeout_ms: None, | ||
| 2025 | + fail_open: true, | ||
| 2026 | + ..RerankServiceConfig::default() | ||
| 2027 | + }, | ||
| 2028 | + ..RetrievalServiceConfig::default() | ||
| 2029 | + }; | ||
| 2030 | + | ||
| 2031 | + assert!(config.validate().is_ok()); | ||
| 2032 | + let core = config.core_config(memory_core::GraphRetrievalConfig::default()); | ||
| 2033 | + assert!(!core.rerank.enabled); | ||
| 2034 | + assert!(core.rerank.fail_open); | ||
| 2035 | + } | ||
| 2036 | + | ||
| 2037 | + | ||
| 2038 | + fn enabled_rerank_rejects_every_invalid_provider_field() { | ||
| 2039 | + let mut invalid = Vec::new(); | ||
| 2040 | + for field in ["model", "base_url", "api_key_env"] { | ||
| 2041 | + let mut config = RetrievalServiceConfig::default(); | ||
| 2042 | + config.rerank.enabled = true; | ||
| 2043 | + match field { | ||
| 2044 | + "model" => config.rerank.model = " ".to_string(), | ||
| 2045 | + "base_url" => config.rerank.base_url = " ".to_string(), | ||
| 2046 | + "api_key_env" => config.rerank.api_key_env = Some(" ".to_string()), | ||
| 2047 | + _ => unreachable!(), | ||
| 2048 | + } | ||
| 2049 | + invalid.push(config); | ||
| 2050 | + } | ||
| 2051 | + let mut invalid_url = RetrievalServiceConfig::default(); | ||
| 2052 | + invalid_url.rerank.enabled = true; | ||
| 2053 | + invalid_url.rerank.base_url = "not-a-url".to_string(); | ||
| 2054 | + invalid.push(invalid_url); | ||
| 2055 | + | ||
| 2056 | + assert!(invalid.into_iter().all(|config| config.validate().is_err())); | ||
| 2057 | + } | ||
| 2058 | + | ||
| 2059 | + | ||
| 2060 | + fn retrieval_candidate_and_rerank_limits_accept_boundaries() { | ||
| 2061 | + for candidate_k in [1, 500] { | ||
| 2062 | + let config = RetrievalServiceConfig { | ||
| 2063 | + candidate_k: Some(candidate_k), | ||
| 2064 | + ..RetrievalServiceConfig::default() | ||
| 2065 | + }; | ||
| 2066 | + assert!(config.validate().is_ok(), "candidate_k={candidate_k}"); | ||
| 2067 | + } | ||
| 2068 | + | ||
| 2069 | + for input_k in [1, 500] { | ||
| 2070 | + let config = RetrievalServiceConfig { | ||
| 2071 | + rerank: RerankServiceConfig { | ||
| 2072 | + enabled: true, | ||
| 2073 | + input_k, | ||
| 2074 | + timeout_ms: Some(1), | ||
| 2075 | + ..RerankServiceConfig::default() | ||
| 2076 | + }, | ||
| 2077 | + ..RetrievalServiceConfig::default() | ||
| 2078 | + }; | ||
| 2079 | + assert!(config.validate().is_ok(), "input_k={input_k}"); | ||
| 2080 | + } | ||
| 2081 | + | ||
| 2082 | + for timeout_ms in [1, MAX_RERANK_TIMEOUT_MS] { | ||
| 2083 | + let config = RetrievalServiceConfig { | ||
| 2084 | + rerank: RerankServiceConfig { | ||
| 2085 | + enabled: true, | ||
| 2086 | + timeout_ms: Some(timeout_ms), | ||
| 2087 | + ..RerankServiceConfig::default() | ||
| 2088 | + }, | ||
| 2089 | + ..RetrievalServiceConfig::default() | ||
| 2090 | + }; | ||
| 2091 | + assert!(config.validate().is_ok(), "timeout_ms={timeout_ms}"); | ||
| 2092 | + } | ||
| 2093 | + } | ||
| 2094 | + | ||
| 2095 | + | ||
| 2096 | + fn retrieval_rejects_candidate_and_rerank_values_outside_limits() { | ||
| 2097 | + for candidate_k in [0, 501] { | ||
| 2098 | + let config = RetrievalServiceConfig { | ||
| 2099 | + candidate_k: Some(candidate_k), | ||
| 2100 | + ..RetrievalServiceConfig::default() | ||
| 2101 | + }; | ||
| 2102 | + assert!(config.validate().is_err(), "candidate_k={candidate_k}"); | ||
| 2103 | + } | ||
| 2104 | + | ||
| 2105 | + for input_k in [0, 501] { | ||
| 2106 | + let config = RetrievalServiceConfig { | ||
| 2107 | + rerank: RerankServiceConfig { | ||
| 2108 | + enabled: true, | ||
| 2109 | + input_k, | ||
| 2110 | + ..RerankServiceConfig::default() | ||
| 2111 | + }, | ||
| 2112 | + ..RetrievalServiceConfig::default() | ||
| 2113 | + }; | ||
| 2114 | + assert!(config.validate().is_err(), "input_k={input_k}"); | ||
| 2115 | + } | ||
| 2116 | + | ||
| 2117 | + let zero_timeout = RetrievalServiceConfig { | ||
| 2118 | + rerank: RerankServiceConfig { | ||
| 2119 | + enabled: true, | ||
| 2120 | + timeout_ms: Some(0), | ||
| 2121 | + ..RerankServiceConfig::default() | ||
| 2122 | + }, | ||
| 2123 | + ..RetrievalServiceConfig::default() | ||
| 2124 | + }; | ||
| 2125 | + assert!(zero_timeout.validate().is_err()); | ||
| 2126 | + | ||
| 2127 | + for timeout_ms in [None, Some(MAX_RERANK_TIMEOUT_MS + 1)] { | ||
| 2128 | + let invalid_timeout = RetrievalServiceConfig { | ||
| 2129 | + rerank: RerankServiceConfig { | ||
| 2130 | + enabled: true, | ||
| 2131 | + timeout_ms, | ||
| 2132 | + ..RerankServiceConfig::default() | ||
| 2133 | + }, | ||
| 2134 | + ..RetrievalServiceConfig::default() | ||
| 2135 | + }; | ||
| 2136 | + assert!(invalid_timeout.validate().is_err()); | ||
| 2137 | + } | ||
| 2138 | + } | ||
| 2139 | + | ||
| 1166 | 2140 | ||
| 1167 | fn packaged_rpm_example_matches_server_schema() { | 2141 | fn packaged_rpm_example_matches_server_schema() { |
| 1168 | - let config = packaged_config(); | 2142 | + let source: serde_json::Value = |
| 2143 | + serde_json::from_str(include_str!("../../../plugins/mcp/ram-a-mem.json")) | ||
| 2144 | + .expect("packaged config is JSON"); | ||
| 2145 | + let config: ServerConfig = | ||
| 2146 | + serde_json::from_value(source.clone()).expect("packaged config parses"); | ||
| 1169 | 2147 | ||
| 1170 | assert!(config.validate_runtime().is_ok()); | 2148 | assert!(config.validate_runtime().is_ok()); |
| 2149 | + assert_eq!( | ||
| 2150 | + serde_json::to_value(&config).expect("config serializes"), | ||
| 2151 | + source, | ||
| 2152 | + "the full example must explicitly contain every serializable config field" | ||
| 2153 | + ); | ||
| 2154 | + assert_eq!(config.limits.max_body_bytes, 16 * 1024 * 1024); | ||
| 2155 | + assert!(config.pipeline.fail_fast); | ||
| 2156 | + assert_eq!(config.pipeline.max_memory_chars, 500); | ||
| 1171 | assert_eq!(config.retrieval.mode, SearchMode::Hybrid); | 2157 | assert_eq!(config.retrieval.mode, SearchMode::Hybrid); |
| 1172 | assert_eq!(config.retrieval.embedding_weight, 0.7); | 2158 | assert_eq!(config.retrieval.embedding_weight, 0.7); |
| 1173 | assert_eq!(config.retrieval.bm25_weight, 0.3); | 2159 | assert_eq!(config.retrieval.bm25_weight, 0.3); |
| @@ -1194,3 +2180,39 @@ pub struct TokenConfig { | |||
| 1194 | pub agent_id: String, | 2180 | pub agent_id: String, |
| 1195 | pub permissions: Vec<String>, | 2181 | pub permissions: Vec<String>, |
| 1196 | } | 2182 | } |
| 2183 | + | ||
| 2184 | +impl AuthConfig { | ||
| 2185 | + fn validate(&self) -> Result<()> { | ||
| 2186 | + if self.tokens.is_empty() { | ||
| 2187 | + anyhow::bail!("production runtime requires at least one authenticated principal"); | ||
| 2188 | + } | ||
| 2189 | + | ||
| 2190 | + let mut token_environments = HashSet::with_capacity(self.tokens.len()); | ||
| 2191 | + for token in &self.tokens { | ||
| 2192 | + for (label, value) in [ | ||
| 2193 | + ("token_env", token.token_env.as_str()), | ||
| 2194 | + ("tenant_id", token.tenant_id.as_str()), | ||
| 2195 | + ("user_id", token.user_id.as_str()), | ||
| 2196 | + ("agent_id", token.agent_id.as_str()), | ||
| 2197 | + ] { | ||
| 2198 | + if value.trim().is_empty() || value.trim() != value { | ||
| 2199 | + anyhow::bail!("authentication {label} must be canonical and non-empty"); | ||
| 2200 | + } | ||
| 2201 | + } | ||
| 2202 | + if !token_environments.insert(token.token_env.as_str()) { | ||
| 2203 | + anyhow::bail!("authentication token environment names must be unique"); | ||
| 2204 | + } | ||
| 2205 | + | ||
| 2206 | + let mut permissions = HashSet::with_capacity(token.permissions.len()); | ||
| 2207 | + for permission in &token.permissions { | ||
| 2208 | + if !SUPPORTED_PERMISSIONS.contains(&permission.as_str()) { | ||
| 2209 | + anyhow::bail!("authentication permission `{permission}` is not supported"); | ||
| 2210 | + } | ||
| 2211 | + if !permissions.insert(permission.as_str()) { | ||
| 2212 | + anyhow::bail!("authentication permissions must be unique per token"); | ||
| 2213 | + } | ||
| 2214 | + } | ||
| 2215 | + } | ||
| 2216 | + Ok(()) | ||
| 2217 | + } | ||
| 2218 | +} | ||
| @@ -308,7 +308,13 @@ pub fn create_http_router( | |||
| 308 | let features = runtime.features; | 308 | let features = runtime.features; |
| 309 | let cancellation_token = runtime.cancellation_token.clone(); | 309 | let cancellation_token = runtime.cancellation_token.clone(); |
| 310 | let service_cancellation_token = cancellation_token.clone(); | 310 | let service_cancellation_token = cancellation_token.clone(); |
| 311 | - let session_manager = Arc::new(LocalSessionManager::default()); | 311 | + let session_idle_timeout = Duration::from_secs(limits.session_idle_timeout_seconds.max(1)); |
| 312 | + let mut local_session_manager = LocalSessionManager::default(); | ||
| 313 | + local_session_manager | ||
| 314 | + .session_config | ||
| 315 | + .keep_alive | ||
| 316 | + .replace(session_idle_timeout); | ||
| 317 | + let session_manager = Arc::new(local_session_manager); | ||
| 312 | let mcp_service: StreamableHttpService<MemoryMcpServer, LocalSessionManager> = | 318 | let mcp_service: StreamableHttpService<MemoryMcpServer, LocalSessionManager> = |
| 313 | StreamableHttpService::new( | 319 | StreamableHttpService::new( |
| 314 | move || { | 320 | move || { |
| @@ -353,7 +359,7 @@ pub fn create_http_router( | |||
| 353 | session_admission: Arc::new(SessionAdmission::new( | 359 | session_admission: Arc::new(SessionAdmission::new( |
| 354 | limits.max_active_sessions_per_principal.max(1), | 360 | limits.max_active_sessions_per_principal.max(1), |
| 355 | limits.max_active_sessions_global.max(1), | 361 | limits.max_active_sessions_global.max(1), |
| 356 | - Duration::from_secs(limits.session_idle_timeout_seconds.max(1)), | 362 | + session_idle_timeout, |
| 357 | )), | 363 | )), |
| 358 | session_manager, | 364 | session_manager, |
| 359 | }; | 365 | }; |
| @@ -27,6 +27,8 @@ pub enum Reservation { | |||
| 27 | 27 | ||
| 28 | pub enum IdempotencyError { | 28 | pub enum IdempotencyError { |
| 29 | Conflict, | 29 | Conflict, |
| 30 | + Busy, | ||
| 31 | + ReadOnly, | ||
| 30 | Storage, | 32 | Storage, |
| 31 | } | 33 | } |
| 32 | 34 | ||
| @@ -34,6 +36,8 @@ impl fmt::Display for IdempotencyError { | |||
| 34 | fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { | 36 | fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { |
| 35 | formatter.write_str(match self { | 37 | formatter.write_str(match self { |
| 36 | Self::Conflict => "idempotency key conflicts with an earlier request", | 38 | Self::Conflict => "idempotency key conflicts with an earlier request", |
| 39 | + Self::Busy => "idempotency database is busy", | ||
| 40 | + Self::ReadOnly => "idempotency database is read-only", | ||
| 37 | Self::Storage => "idempotency storage operation failed", | 41 | Self::Storage => "idempotency storage operation failed", |
| 38 | }) | 42 | }) |
| 39 | } | 43 | } |
| @@ -102,14 +106,14 @@ fn open_connection(path: &Path) -> Result<Connection, IdempotencyError> { | |||
| 102 | std::fs::create_dir_all(parent).map_err(|_| IdempotencyError::Storage)?; | 106 | std::fs::create_dir_all(parent).map_err(|_| IdempotencyError::Storage)?; |
| 103 | } | 107 | } |
| 104 | } | 108 | } |
| 105 | - let connection = Connection::open(path).map_err(|_| IdempotencyError::Storage)?; | 109 | + let connection = Connection::open(path).map_err(map_sqlite_error)?; |
| 106 | connection | 110 | connection |
| 107 | .busy_timeout(Duration::from_millis(5_000)) | 111 | .busy_timeout(Duration::from_millis(5_000)) |
| 108 | - .map_err(|_| IdempotencyError::Storage)?; | 112 | + .map_err(map_sqlite_error)?; |
| 109 | if path != Path::new(":memory:") { | 113 | if path != Path::new(":memory:") { |
| 110 | connection | 114 | connection |
| 111 | .query_row("PRAGMA journal_mode = WAL", [], |_| Ok(())) | 115 | .query_row("PRAGMA journal_mode = WAL", [], |_| Ok(())) |
| 112 | - .map_err(|_| IdempotencyError::Storage)?; | 116 | + .map_err(map_sqlite_error)?; |
| 113 | } | 117 | } |
| 114 | connection | 118 | connection |
| 115 | .execute_batch( | 119 | .execute_batch( |
| @@ -128,7 +132,7 @@ fn open_connection(path: &Path) -> Result<Connection, IdempotencyError> { | |||
| 128 | ); | 132 | ); |
| 129 | "#, | 133 | "#, |
| 130 | ) | 134 | ) |
| 131 | - .map_err(|_| IdempotencyError::Storage)?; | 135 | + .map_err(map_sqlite_error)?; |
| 132 | Ok(connection) | 136 | Ok(connection) |
| 133 | } | 137 | } |
| 134 | 138 | ||
| @@ -138,9 +142,7 @@ fn reserve_sync( | |||
| 138 | pipeline_run_id: &str, | 142 | pipeline_run_id: &str, |
| 139 | ) -> Result<Reservation, IdempotencyError> { | 143 | ) -> Result<Reservation, IdempotencyError> { |
| 140 | let mut connection = open_connection(path)?; | 144 | let mut connection = open_connection(path)?; |
| 141 | - let transaction = connection | 145 | + let transaction = connection.transaction().map_err(map_sqlite_error)?; |
| 142 | - .transaction() | ||
| 143 | - .map_err(|_| IdempotencyError::Storage)?; | ||
| 144 | let mut cached = Vec::new(); | 146 | let mut cached = Vec::new(); |
| 145 | let mut candidate_message_ids = Vec::new(); | 147 | let mut candidate_message_ids = Vec::new(); |
| 146 | let now = current_time_ms(); | 148 | let now = current_time_ms(); |
| @@ -160,7 +162,7 @@ fn reserve_sync( | |||
| 160 | }, | 162 | }, |
| 161 | ) | 163 | ) |
| 162 | .optional() | 164 | .optional() |
| 163 | - .map_err(|_| IdempotencyError::Storage)?; | 165 | + .map_err(map_sqlite_error)?; |
| 164 | 166 | ||
| 165 | match existing { | 167 | match existing { |
| 166 | Some((stored_hash, _, _)) if stored_hash != entry.content_hash => { | 168 | Some((stored_hash, _, _)) if stored_hash != entry.content_hash => { |
| @@ -183,7 +185,7 @@ fn reserve_sync( | |||
| 183 | now, | 185 | now, |
| 184 | ], | 186 | ], |
| 185 | ) | 187 | ) |
| 186 | - .map_err(|_| IdempotencyError::Storage)?; | 188 | + .map_err(map_sqlite_error)?; |
| 187 | candidate_message_ids.push(entry.message_id.clone()); | 189 | candidate_message_ids.push(entry.message_id.clone()); |
| 188 | } | 190 | } |
| 189 | None => { | 191 | None => { |
| @@ -202,15 +204,13 @@ fn reserve_sync( | |||
| 202 | now, | 204 | now, |
| 203 | ], | 205 | ], |
| 204 | ) | 206 | ) |
| 205 | - .map_err(|_| IdempotencyError::Storage)?; | 207 | + .map_err(map_sqlite_error)?; |
| 206 | candidate_message_ids.push(entry.message_id.clone()); | 208 | candidate_message_ids.push(entry.message_id.clone()); |
| 207 | } | 209 | } |
| 208 | } | 210 | } |
| 209 | } | 211 | } |
| 210 | 212 | ||
| 211 | - transaction | 213 | + transaction.commit().map_err(map_sqlite_error)?; |
| 212 | - .commit() | ||
| 213 | - .map_err(|_| IdempotencyError::Storage)?; | ||
| 214 | if candidate_message_ids.is_empty() { | 214 | if candidate_message_ids.is_empty() { |
| 215 | Ok(Reservation::Cached { results: cached }) | 215 | Ok(Reservation::Cached { results: cached }) |
| 216 | } else { | 216 | } else { |
| @@ -228,9 +228,7 @@ fn complete_sync( | |||
| 228 | result_json: &str, | 228 | result_json: &str, |
| 229 | ) -> Result<(), IdempotencyError> { | 229 | ) -> Result<(), IdempotencyError> { |
| 230 | let mut connection = open_connection(path)?; | 230 | let mut connection = open_connection(path)?; |
| 231 | - let transaction = connection | 231 | + let transaction = connection.transaction().map_err(map_sqlite_error)?; |
| 232 | - .transaction() | ||
| 233 | - .map_err(|_| IdempotencyError::Storage)?; | ||
| 234 | let now = current_time_ms(); | 232 | let now = current_time_ms(); |
| 235 | for entry in entries { | 233 | for entry in entries { |
| 236 | let updated = transaction | 234 | let updated = transaction |
| @@ -249,12 +247,32 @@ fn complete_sync( | |||
| 249 | now, | 247 | now, |
| 250 | ], | 248 | ], |
| 251 | ) | 249 | ) |
| 252 | - .map_err(|_| IdempotencyError::Storage)?; | 250 | + .map_err(map_sqlite_error)?; |
| 253 | if updated != 1 { | 251 | if updated != 1 { |
| 254 | return Err(IdempotencyError::Storage); | 252 | return Err(IdempotencyError::Storage); |
| 255 | } | 253 | } |
| 256 | } | 254 | } |
| 257 | - transaction.commit().map_err(|_| IdempotencyError::Storage) | 255 | + transaction.commit().map_err(map_sqlite_error) |
| 256 | +} | ||
| 257 | + | ||
| 258 | +fn map_sqlite_error(error: rusqlite::Error) -> IdempotencyError { | ||
| 259 | + match error { | ||
| 260 | + rusqlite::Error::SqliteFailure( | ||
| 261 | + rusqlite::ffi::Error { | ||
| 262 | + code: rusqlite::ErrorCode::DatabaseBusy | rusqlite::ErrorCode::DatabaseLocked, | ||
| 263 | + .. | ||
| 264 | + }, | ||
| 265 | + _, | ||
| 266 | + ) => IdempotencyError::Busy, | ||
| 267 | + rusqlite::Error::SqliteFailure( | ||
| 268 | + rusqlite::ffi::Error { | ||
| 269 | + code: rusqlite::ErrorCode::ReadOnly, | ||
| 270 | + .. | ||
| 271 | + }, | ||
| 272 | + _, | ||
| 273 | + ) => IdempotencyError::ReadOnly, | ||
| 274 | + _ => IdempotencyError::Storage, | ||
| 275 | + } | ||
| 258 | } | 276 | } |
| 259 | 277 | ||
| 260 | fn current_time_ms() -> i64 { | 278 | fn current_time_ms() -> i64 { |
| @@ -270,6 +288,19 @@ mod tests { | |||
| 270 | 288 | ||
| 271 | use super::{IdempotencyEntry, IdempotencyError, IdempotencyRepository, Reservation}; | 289 | use super::{IdempotencyEntry, IdempotencyError, IdempotencyRepository, Reservation}; |
| 272 | 290 | ||
| 291 | + | ||
| 292 | + fn sqlite_errors_keep_busy_and_read_only_classification() { | ||
| 293 | + for (sqlite_code, expected) in [ | ||
| 294 | + (rusqlite::ffi::SQLITE_BUSY, IdempotencyError::Busy), | ||
| 295 | + (rusqlite::ffi::SQLITE_LOCKED, IdempotencyError::Busy), | ||
| 296 | + (rusqlite::ffi::SQLITE_READONLY, IdempotencyError::ReadOnly), | ||
| 297 | + ] { | ||
| 298 | + let error = | ||
| 299 | + rusqlite::Error::SqliteFailure(rusqlite::ffi::Error::new(sqlite_code), None); | ||
| 300 | + assert_eq!(super::map_sqlite_error(error), expected); | ||
| 301 | + } | ||
| 302 | + } | ||
| 303 | + | ||
| 273 | fn entry(hash: &str) -> IdempotencyEntry { | 304 | fn entry(hash: &str) -> IdempotencyEntry { |
| 274 | IdempotencyEntry { | 305 | IdempotencyEntry { |
| 275 | scope_id: "scope-a".to_string(), | 306 | scope_id: "scope-a".to_string(), |
| @@ -6,6 +6,7 @@ pub mod config; | |||
| 6 | pub mod http; | 6 | pub mod http; |
| 7 | pub mod idempotency; | 7 | pub mod idempotency; |
| 8 | pub mod mcp_server; | 8 | pub mod mcp_server; |
| 9 | +pub mod observability; | ||
| 9 | pub mod service; | 10 | pub mod service; |
| 10 | pub mod types; | 11 | pub mod types; |
| 11 | 12 | ||
| @@ -20,8 +21,12 @@ pub use config::{ | |||
| 20 | AuthConfig, CaseLibraryConfig, CaseLibraryFeatureConfig, CaseLibraryServiceConfig, | 21 | AuthConfig, CaseLibraryConfig, CaseLibraryFeatureConfig, CaseLibraryServiceConfig, |
| 21 | CaseServiceConfig, EmbeddingProviderKind, FeatureFlags, FeaturesConfig, | 22 | CaseServiceConfig, EmbeddingProviderKind, FeatureFlags, FeaturesConfig, |
| 22 | GraphMemoryFeatureConfig, GraphMemoryRetrievalConfig, GraphMemoryServiceConfig, HttpConfig, | 23 | GraphMemoryFeatureConfig, GraphMemoryRetrievalConfig, GraphMemoryServiceConfig, HttpConfig, |
| 23 | - LimitsConfig, MemoryFeatureConfig, ProvidersConfig, RerankServiceConfig, | 24 | + LimitsConfig, MemoryFeatureConfig, PipelineServiceConfig, ProvidersConfig, RerankServiceConfig, |
| 24 | - RetrievalServiceConfig, ServerConfig, StorageConfig, TokenConfig, | 25 | + RetrievalServiceConfig, ServerConfig, StorageConfig, TokenConfig, DEFAULT_RERANK_TIMEOUT_MS, |
| 26 | + MAX_ACTIVE_SESSIONS_GLOBAL, MAX_ACTIVE_SESSIONS_PER_PRINCIPAL, MAX_INITIALIZE_RATE_BURST, | ||
| 27 | + MAX_INITIALIZE_REQUESTS_PER_SECOND, MAX_IN_FLIGHT_PER_PRINCIPAL_TOOL, MAX_MCP_BODY_BYTES, | ||
| 28 | + MAX_RATE_BURST, MAX_REQUESTS_PER_SECOND, MAX_RERANK_TIMEOUT_MS, | ||
| 29 | + MAX_SESSION_IDLE_TIMEOUT_SECONDS, | ||
| 25 | }; | 30 | }; |
| 26 | pub use http::{create_http_router, HttpRuntime, RequestId, AGENT_ID_HEADER, REQUEST_ID_HEADER}; | 31 | pub use http::{create_http_router, HttpRuntime, RequestId, AGENT_ID_HEADER, REQUEST_ID_HEADER}; |
| 27 | pub use idempotency::IdempotencyRepository; | 32 | pub use idempotency::IdempotencyRepository; |
| @@ -32,6 +37,7 @@ pub use types::{ | |||
| 32 | CaseMutationConfirmationRequest, CaseSearchRequest, IngestMessage, IngestRequest, | 37 | CaseMutationConfirmationRequest, CaseSearchRequest, IngestMessage, IngestRequest, |
| 33 | SearchRequest, MAX_CASE_DELETION_REASON_CHARS, MAX_CASE_DIAGNOSIS_CHARS, | 38 | SearchRequest, MAX_CASE_DELETION_REASON_CHARS, MAX_CASE_DIAGNOSIS_CHARS, |
| 34 | MAX_CASE_DOCUMENT_CHARS, MAX_CASE_DOCUMENT_ID_CHARS, MAX_CASE_DOCUMENT_NAME_CHARS, | 39 | MAX_CASE_DOCUMENT_CHARS, MAX_CASE_DOCUMENT_ID_CHARS, MAX_CASE_DOCUMENT_NAME_CHARS, |
| 35 | - MAX_CASE_FILE_NAME_CHARS, MAX_CASE_TOP_K, MAX_INGEST_MESSAGES, MAX_MESSAGE_TEXT_CHARS, | 40 | + MAX_CASE_FILE_NAME_CHARS, MAX_CASE_LIBRARY_CHARS, MAX_CASE_TOP_K, MAX_CONVERSATION_ID_CHARS, |
| 36 | - MAX_QUERY_CHARS, MAX_TOP_K, | 41 | + MAX_INGEST_MESSAGES, MAX_MESSAGE_ID_CHARS, MAX_MESSAGE_TEXT_CHARS, MAX_QUERY_CHARS, |
| 42 | + MAX_SPEAKER_CHARS, MAX_TOP_K, | ||
| 37 | }; | 43 | }; |
| @@ -12,6 +12,7 @@ use memory_core::{ | |||
| 12 | sqlite::GraphRepository, EmbeddingProvider, GraphBuildPipeline, HashEmbedding, MemoryManager, | 12 | sqlite::GraphRepository, EmbeddingProvider, GraphBuildPipeline, HashEmbedding, MemoryManager, |
| 13 | OpenRouterEmbedding, OpenRouterReranker, RerankProvider, SqliteMemoryStore, | 13 | OpenRouterEmbedding, OpenRouterReranker, RerankProvider, SqliteMemoryStore, |
| 14 | }; | 14 | }; |
| 15 | +use memory_mcp::observability::{LogSettings, RamLogFormatter}; | ||
| 15 | use memory_mcp::{ | 16 | use memory_mcp::{ |
| 16 | create_http_router, EmbeddingProviderKind, HttpRuntime, IdempotencyRepository, MemoryService, | 17 | create_http_router, EmbeddingProviderKind, HttpRuntime, IdempotencyRepository, MemoryService, |
| 17 | ServerConfig, TokenAuthenticator, | 18 | ServerConfig, TokenAuthenticator, |
| @@ -21,6 +22,7 @@ use memory_pipeline::extraction::{LlmMemoryExtractor, MemoryExtractor}; | |||
| 21 | use memory_pipeline::grounding::{GroundingVerifier, LlmGroundingVerifier}; | 22 | use memory_pipeline::grounding::{GroundingVerifier, LlmGroundingVerifier}; |
| 22 | use tokio_util::sync::CancellationToken; | 23 | use tokio_util::sync::CancellationToken; |
| 23 | use tracing::Instrument; | 24 | use tracing::Instrument; |
| 25 | +use tracing_subscriber::fmt::format::JsonFields; | ||
| 24 | use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt, EnvFilter}; | 26 | use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt, EnvFilter}; |
| 25 | use uuid::Uuid; | 27 | use uuid::Uuid; |
| 26 | 28 | ||
| @@ -35,7 +37,7 @@ struct Args { | |||
| 35 | 37 | ||
| 36 | 38 | ||
| 37 | async fn main() -> Result<()> { | 39 | async fn main() -> Result<()> { |
| 38 | - init_tracing()?; | 40 | + let log_settings = init_tracing()?; |
| 39 | let startup_run_id = Uuid::new_v4().to_string(); | 41 | let startup_run_id = Uuid::new_v4().to_string(); |
| 40 | let args = Args::parse(); | 42 | let args = Args::parse(); |
| 41 | let config_path = resolve_config_path(args.config)?; | 43 | let config_path = resolve_config_path(args.config)?; |
| @@ -75,20 +77,16 @@ async fn main() -> Result<()> { | |||
| 75 | embedding_dimensions = providers.embedding_dimensions, | 77 | embedding_dimensions = providers.embedding_dimensions, |
| 76 | rerank_enabled = config.retrieval.rerank.enabled, | 78 | rerank_enabled = config.retrieval.rerank.enabled, |
| 77 | rerank_provider = ?config.retrieval.rerank.provider, | 79 | rerank_provider = ?config.retrieval.rerank.provider, |
| 78 | - rerank_model = config.retrieval.rerank.model | 80 | + rerank_model = config.retrieval.rerank.model, |
| 81 | + log_format = log_settings.format.as_str(), | ||
| 82 | + log_source = log_settings.source | ||
| 79 | ); | 83 | ); |
| 80 | let provider_key = resolve_secret_env(&providers.api_key_env)?; | 84 | let provider_key = resolve_secret_env(&providers.api_key_env)?; |
| 81 | let embedder: Arc<dyn EmbeddingProvider> = match providers.embedding_provider { | 85 | let embedder: Arc<dyn EmbeddingProvider> = match providers.embedding_provider { |
| 82 | EmbeddingProviderKind::OpenAiCompatible => { | 86 | EmbeddingProviderKind::OpenAiCompatible => { |
| 83 | - let embedding_key_env = providers | 87 | + let embedding_key_env = providers.resolved_embedding_api_key_env(); |
| 84 | - .embedding_api_key_env | ||
| 85 | - .as_deref() | ||
| 86 | - .unwrap_or(&providers.api_key_env); | ||
| 87 | let embedding_key = resolve_secret_env(embedding_key_env)?; | 88 | let embedding_key = resolve_secret_env(embedding_key_env)?; |
| 88 | - let embedding_base_url = providers | 89 | + let embedding_base_url = providers.resolved_embedding_base_url(); |
| 89 | - .embedding_base_url | ||
| 90 | - .as_deref() | ||
| 91 | - .unwrap_or(&providers.base_url); | ||
| 92 | Arc::new(OpenRouterEmbedding::with_base_url( | 90 | Arc::new(OpenRouterEmbedding::with_base_url( |
| 93 | embedding_key, | 91 | embedding_key, |
| 94 | embedding_base_url, | 92 | embedding_base_url, |
| @@ -153,7 +151,13 @@ async fn main() -> Result<()> { | |||
| 153 | retrieval_config, | 151 | retrieval_config, |
| 154 | )) | 152 | )) |
| 155 | }; | 153 | }; |
| 156 | - let mut service = MemoryService::new(manager, idempotency, extractor, verifier); | 154 | + let mut service = MemoryService::with_pipeline_config( |
| 155 | + manager, | ||
| 156 | + idempotency, | ||
| 157 | + extractor, | ||
| 158 | + verifier, | ||
| 159 | + config.pipeline.pipeline_config(), | ||
| 160 | + ); | ||
| 157 | if features.memory && config.features.graph_memory.enabled { | 161 | if features.memory && config.features.graph_memory.enabled { |
| 158 | let graph = config | 162 | let graph = config |
| 159 | .graph_memory | 163 | .graph_memory |
| @@ -209,25 +213,21 @@ async fn main() -> Result<()> { | |||
| 209 | } | 213 | } |
| 210 | }, | 214 | }, |
| 211 | embedding_api_key_env: case_library | 215 | embedding_api_key_env: case_library |
| 212 | - .embedding_api_key_env | 216 | + .resolved_embedding_api_key_env(providers) |
| 213 | - .clone() | 217 | + .to_string(), |
| 214 | - .unwrap_or_else(|| providers.api_key_env.clone()), | ||
| 215 | embedding_base_url: case_library | 218 | embedding_base_url: case_library |
| 216 | - .embedding_base_url | 219 | + .resolved_embedding_base_url(providers) |
| 217 | - .clone() | 220 | + .to_string(), |
| 218 | - .unwrap_or_else(|| providers.base_url.clone()), | ||
| 219 | embedding_model: case_library.embedding_model.clone(), | 221 | embedding_model: case_library.embedding_model.clone(), |
| 220 | embedding_dimensions: case_library.embedding_dimensions, | 222 | embedding_dimensions: case_library.embedding_dimensions, |
| 221 | chunk_size: case_library.chunk_size, | 223 | chunk_size: case_library.chunk_size, |
| 222 | summary_llm_model: case_library.summary_llm_model.clone(), | 224 | summary_llm_model: case_library.summary_llm_model.clone(), |
| 223 | summary_llm_api_key_env: case_library | 225 | summary_llm_api_key_env: case_library |
| 224 | - .summary_llm_api_key_env | 226 | + .resolved_summary_api_key_env(providers) |
| 225 | - .clone() | 227 | + .to_string(), |
| 226 | - .unwrap_or_else(|| providers.api_key_env.clone()), | ||
| 227 | summary_llm_base_url: case_library | 228 | summary_llm_base_url: case_library |
| 228 | - .summary_llm_base_url | 229 | + .resolved_summary_base_url(providers) |
| 229 | - .clone() | 230 | + .to_string(), |
| 230 | - .unwrap_or_else(|| providers.base_url.clone()), | ||
| 231 | summary_llm_timeout_ms: case_library.summary_llm_timeout_ms, | 231 | summary_llm_timeout_ms: case_library.summary_llm_timeout_ms, |
| 232 | }; | 232 | }; |
| 233 | let case_service = memory_cases::build_service(&case_options) | 233 | let case_service = memory_cases::build_service(&case_options) |
| @@ -344,13 +344,20 @@ async fn main() -> Result<()> { | |||
| 344 | server_result.context("HTTP server failed") | 344 | server_result.context("HTTP server failed") |
| 345 | } | 345 | } |
| 346 | 346 | ||
| 347 | -fn init_tracing() -> Result<()> { | 347 | +fn init_tracing() -> Result<LogSettings> { |
| 348 | + let settings = LogSettings::from_env()?; | ||
| 348 | let filter = EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info")); | 349 | let filter = EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info")); |
| 349 | tracing_subscriber::registry() | 350 | tracing_subscriber::registry() |
| 350 | .with(filter) | 351 | .with(filter) |
| 351 | - .with(tracing_subscriber::fmt::layer().json()) | 352 | + .with( |
| 353 | + tracing_subscriber::fmt::layer() | ||
| 354 | + .fmt_fields(JsonFields::new()) | ||
| 355 | + .event_format(RamLogFormatter::new(settings)) | ||
| 356 | + .with_writer(std::io::stderr), | ||
| 357 | + ) | ||
| 352 | .try_init() | 358 | .try_init() |
| 353 | - .context("failed to initialize structured logging") | 359 | + .context("failed to initialize structured logging")?; |
| 360 | + Ok(settings) | ||
| 354 | } | 361 | } |
| 355 | 362 | ||
| 356 | fn resolve_config_path(explicit: Option<PathBuf>) -> Result<PathBuf> { | 363 | fn resolve_config_path(explicit: Option<PathBuf>) -> Result<PathBuf> { |
| @@ -141,7 +141,7 @@ impl MemoryMcpServer { | |||
| 141 | biased; | 141 | biased; |
| 142 | _ = self.cancellation_token.cancelled() => { | 142 | _ = self.cancellation_token.cancelled() => { |
| 143 | tracing::warn!(event = "ram_a.memory.ingest.failed", stage = "cancelled", error_code = "CANCELLED", retriable = true, latency_ms = started.elapsed().as_millis() as u64); | 143 | tracing::warn!(event = "ram_a.memory.ingest.failed", stage = "cancelled", error_code = "CANCELLED", retriable = true, latency_ms = started.elapsed().as_millis() as u64); |
| 144 | - return tool_error("CANCELLED", true); | 144 | + return tool_error_with_request("CANCELLED", true, &request_id); |
| 145 | } | 145 | } |
| 146 | result = self.service.ingest(principal, request) => result, | 146 | result = self.service.ingest(principal, request) => result, |
| 147 | }; | 147 | }; |
| @@ -163,8 +163,21 @@ impl MemoryMcpServer { | |||
| 163 | ) | 163 | ) |
| 164 | } | 164 | } |
| 165 | Err(error) => { | 165 | Err(error) => { |
| 166 | - tracing::error!(event = "ram_a.memory.ingest.failed", stage = "service", error_code = error.code(), retriable = error.retriable(), latency_ms = started.elapsed().as_millis() as u64); | 166 | + let (origin_file, origin_line) = |
| 167 | - service_error(error) | 167 | + error.error_origin().unwrap_or((file!(), line!())); |
| 168 | + tracing::error!( | ||
| 169 | + event = "ram_a.memory.ingest.failed", | ||
| 170 | + stage = error.stage().unwrap_or("service"), | ||
| 171 | + error_code = error.code(), | ||
| 172 | + retriable = error.retriable(), | ||
| 173 | + error_site = error.error_site().unwrap_or("memory_mcp.ingest.service"), | ||
| 174 | + error_origin_file = origin_file, | ||
| 175 | + error_origin_line = origin_line, | ||
| 176 | + source_error_kind = error.source_error_kind().unwrap_or("internal"), | ||
| 177 | + source_error_message = error.source_error_message().unwrap_or("memory ingest failed"), | ||
| 178 | + latency_ms = started.elapsed().as_millis() as u64 | ||
| 179 | + ); | ||
| 180 | + service_error(error, &request_id) | ||
| 168 | } | 181 | } |
| 169 | } | 182 | } |
| 170 | } | 183 | } |
| @@ -202,7 +215,7 @@ impl MemoryMcpServer { | |||
| 202 | biased; | 215 | biased; |
| 203 | _ = self.cancellation_token.cancelled() => { | 216 | _ = self.cancellation_token.cancelled() => { |
| 204 | tracing::warn!(event = "ram_a.memory.search.failed", stage = "cancelled", error_code = "CANCELLED", retriable = true, latency_ms = started.elapsed().as_millis() as u64); | 217 | tracing::warn!(event = "ram_a.memory.search.failed", stage = "cancelled", error_code = "CANCELLED", retriable = true, latency_ms = started.elapsed().as_millis() as u64); |
| 205 | - return tool_error("CANCELLED", true); | 218 | + return tool_error_with_request("CANCELLED", true, &request_id); |
| 206 | } | 219 | } |
| 207 | result = self.service.search(principal, request) => result, | 220 | result = self.service.search(principal, request) => result, |
| 208 | }; | 221 | }; |
| @@ -214,8 +227,21 @@ impl MemoryMcpServer { | |||
| 214 | ) | 227 | ) |
| 215 | } | 228 | } |
| 216 | Err(error) => { | 229 | Err(error) => { |
| 217 | - tracing::error!(event = "ram_a.memory.search.failed", stage = "service", error_code = error.code(), retriable = error.retriable(), latency_ms = started.elapsed().as_millis() as u64); | 230 | + let (origin_file, origin_line) = |
| 218 | - service_error(error) | 231 | + error.error_origin().unwrap_or((file!(), line!())); |
| 232 | + tracing::error!( | ||
| 233 | + event = "ram_a.memory.search.failed", | ||
| 234 | + stage = error.stage().unwrap_or("service"), | ||
| 235 | + error_code = error.code(), | ||
| 236 | + retriable = error.retriable(), | ||
| 237 | + error_site = error.error_site().unwrap_or("memory_mcp.search.service"), | ||
| 238 | + error_origin_file = origin_file, | ||
| 239 | + error_origin_line = origin_line, | ||
| 240 | + source_error_kind = error.source_error_kind().unwrap_or("internal"), | ||
| 241 | + source_error_message = error.source_error_message().unwrap_or("memory search failed"), | ||
| 242 | + latency_ms = started.elapsed().as_millis() as u64 | ||
| 243 | + ); | ||
| 244 | + service_error(error, &request_id) | ||
| 219 | } | 245 | } |
| 220 | } | 246 | } |
| 221 | } | 247 | } |
| @@ -506,11 +532,29 @@ fn tool_span(tool: &'static str, request_id: &str, principal: &Principal) -> tra | |||
| 506 | ) | 532 | ) |
| 507 | } | 533 | } |
| 508 | 534 | ||
| 509 | -fn service_error(error: ServiceError) -> CallToolResult { | 535 | +fn service_error(error: ServiceError, request_id: &str) -> CallToolResult { |
| 510 | - CallToolResult::structured_error(serde_json::json!({ | 536 | + let mut content = serde_json::json!({ |
| 511 | "code": error.code(), | 537 | "code": error.code(), |
| 512 | "message": error.to_string(), | 538 | "message": error.to_string(), |
| 513 | "retriable": error.retriable(), | 539 | "retriable": error.retriable(), |
| 540 | + "request_id": request_id, | ||
| 541 | + }); | ||
| 542 | + if let Some(stage) = error.stage() { | ||
| 543 | + content["stage"] = serde_json::json!(stage); | ||
| 544 | + } | ||
| 545 | + CallToolResult::structured_error(content) | ||
| 546 | +} | ||
| 547 | + | ||
| 548 | +fn tool_error_with_request( | ||
| 549 | + code: &'static str, | ||
| 550 | + retriable: bool, | ||
| 551 | + request_id: &str, | ||
| 552 | +) -> CallToolResult { | ||
| 553 | + CallToolResult::structured_error(serde_json::json!({ | ||
| 554 | + "code": code, | ||
| 555 | + "message": "memory tool request was rejected", | ||
| 556 | + "retriable": retriable, | ||
| 557 | + "request_id": request_id, | ||
| 514 | })) | 558 | })) |
| 515 | } | 559 | } |
| 516 | 560 | ||
| @@ -530,6 +574,47 @@ fn tool_error(code: &'static str, retriable: bool) -> CallToolResult { | |||
| 530 | })) | 574 | })) |
| 531 | } | 575 | } |
| 532 | 576 | ||
| 577 | + | ||
| 578 | +mod tests { | ||
| 579 | + use memory_pipeline::error::PipelineStage; | ||
| 580 | + use serde_json::json; | ||
| 581 | + | ||
| 582 | + use super::{service_error, ServiceError}; | ||
| 583 | + | ||
| 584 | + | ||
| 585 | + fn structured_service_errors_expose_stable_pipeline_and_rerank_contracts() { | ||
| 586 | + for stage in [PipelineStage::Extract, PipelineStage::Ground] { | ||
| 587 | + let result = service_error( | ||
| 588 | + ServiceError::Pipeline { stage: Some(stage) }, | ||
| 589 | + "request-test", | ||
| 590 | + ); | ||
| 591 | + assert_eq!(result.is_error, Some(true)); | ||
| 592 | + assert_eq!( | ||
| 593 | + result.structured_content, | ||
| 594 | + Some(json!({ | ||
| 595 | + "code": "PIPELINE_FAILED", | ||
| 596 | + "message": "memory pipeline failed", | ||
| 597 | + "retriable": true, | ||
| 598 | + "request_id": "request-test", | ||
| 599 | + "stage": stage.as_str(), | ||
| 600 | + })) | ||
| 601 | + ); | ||
| 602 | + } | ||
| 603 | + | ||
| 604 | + let rerank = service_error(ServiceError::Rerank, "request-test"); | ||
| 605 | + assert_eq!(rerank.is_error, Some(true)); | ||
| 606 | + assert_eq!( | ||
| 607 | + rerank.structured_content, | ||
| 608 | + Some(json!({ | ||
| 609 | + "code": "RERANK_FAILED", | ||
| 610 | + "message": "memory rerank failed", | ||
| 611 | + "retriable": true, | ||
| 612 | + "request_id": "request-test", | ||
| 613 | + })) | ||
| 614 | + ); | ||
| 615 | + } | ||
| 616 | +} | ||
| 617 | + | ||
| 533 | impl ServerHandler for MemoryMcpServer { | 618 | impl ServerHandler for MemoryMcpServer { |
| 534 | fn get_info(&self) -> ServerInfo { | 619 | fn get_info(&self) -> ServerInfo { |
| 535 | ServerInfo::new(ServerCapabilities::builder().enable_tools().build()) | 620 | ServerInfo::new(ServerCapabilities::builder().enable_tools().build()) |
| @@ -0,0 +1,607 @@ | |||
| 1 | +use std::collections::BTreeMap; | ||
| 2 | +use std::env; | ||
| 3 | +use std::fmt; | ||
| 4 | + | ||
| 5 | +use chrono::{SecondsFormat, Utc}; | ||
| 6 | +use serde_json::{Map, Value}; | ||
| 7 | +use tracing::{Event, Subscriber}; | ||
| 8 | +use tracing_subscriber::fmt::format::{FormatEvent, FormatFields, JsonFields, Writer}; | ||
| 9 | +use tracing_subscriber::fmt::{FmtContext, FormattedFields}; | ||
| 10 | +use tracing_subscriber::registry::LookupSpan; | ||
| 11 | + | ||
| 12 | +pub const LOG_FORMAT_ENV: &str = "RAM_A_LOG_FORMAT"; | ||
| 13 | +pub const LOG_SOURCE_ENV: &str = "RAM_A_LOG_SOURCE"; | ||
| 14 | + | ||
| 15 | + | ||
| 16 | +pub enum LogFormat { | ||
| 17 | + Json, | ||
| 18 | + Compact, | ||
| 19 | +} | ||
| 20 | + | ||
| 21 | +impl LogFormat { | ||
| 22 | + pub const fn as_str(self) -> &'static str { | ||
| 23 | + match self { | ||
| 24 | + Self::Json => "json", | ||
| 25 | + Self::Compact => "compact", | ||
| 26 | + } | ||
| 27 | + } | ||
| 28 | +} | ||
| 29 | + | ||
| 30 | + | ||
| 31 | +pub struct LogSettings { | ||
| 32 | + pub format: LogFormat, | ||
| 33 | + pub source: bool, | ||
| 34 | +} | ||
| 35 | + | ||
| 36 | +impl Default for LogSettings { | ||
| 37 | + fn default() -> Self { | ||
| 38 | + Self { | ||
| 39 | + format: LogFormat::Json, | ||
| 40 | + source: false, | ||
| 41 | + } | ||
| 42 | + } | ||
| 43 | +} | ||
| 44 | + | ||
| 45 | +impl LogSettings { | ||
| 46 | + pub fn from_env() -> Result<Self, LogSettingsError> { | ||
| 47 | + Ok(Self { | ||
| 48 | + format: parse_format(env::var_os(LOG_FORMAT_ENV))?, | ||
| 49 | + source: parse_source(env::var_os(LOG_SOURCE_ENV))?, | ||
| 50 | + }) | ||
| 51 | + } | ||
| 52 | +} | ||
| 53 | + | ||
| 54 | + | ||
| 55 | +pub enum LogSettingsError { | ||
| 56 | + InvalidFormat, | ||
| 57 | + InvalidSource, | ||
| 58 | +} | ||
| 59 | + | ||
| 60 | +impl fmt::Display for LogSettingsError { | ||
| 61 | + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { | ||
| 62 | + formatter.write_str(match self { | ||
| 63 | + Self::InvalidFormat => "invalid RAM_A_LOG_FORMAT; expected one of: json, compact", | ||
| 64 | + Self::InvalidSource => "invalid RAM_A_LOG_SOURCE; expected one of: true, false", | ||
| 65 | + }) | ||
| 66 | + } | ||
| 67 | +} | ||
| 68 | + | ||
| 69 | +impl std::error::Error for LogSettingsError {} | ||
| 70 | + | ||
| 71 | +fn parse_format(value: Option<std::ffi::OsString>) -> Result<LogFormat, LogSettingsError> { | ||
| 72 | + match value { | ||
| 73 | + None => Ok(LogFormat::Json), | ||
| 74 | + Some(value) if value == "json" => Ok(LogFormat::Json), | ||
| 75 | + Some(value) if value == "compact" => Ok(LogFormat::Compact), | ||
| 76 | + Some(_) => Err(LogSettingsError::InvalidFormat), | ||
| 77 | + } | ||
| 78 | +} | ||
| 79 | + | ||
| 80 | +fn parse_source(value: Option<std::ffi::OsString>) -> Result<bool, LogSettingsError> { | ||
| 81 | + match value { | ||
| 82 | + None => Ok(false), | ||
| 83 | + Some(value) if value == "true" => Ok(true), | ||
| 84 | + Some(value) if value == "false" => Ok(false), | ||
| 85 | + Some(_) => Err(LogSettingsError::InvalidSource), | ||
| 86 | + } | ||
| 87 | +} | ||
| 88 | + | ||
| 89 | + | ||
| 90 | +pub struct RamLogFormatter { | ||
| 91 | + settings: LogSettings, | ||
| 92 | +} | ||
| 93 | + | ||
| 94 | +impl RamLogFormatter { | ||
| 95 | + pub const fn new(settings: LogSettings) -> Self { | ||
| 96 | + Self { settings } | ||
| 97 | + } | ||
| 98 | +} | ||
| 99 | + | ||
| 100 | +impl<S> FormatEvent<S, JsonFields> for RamLogFormatter | ||
| 101 | +where | ||
| 102 | + S: Subscriber + for<'lookup> LookupSpan<'lookup>, | ||
| 103 | +{ | ||
| 104 | + fn format_event( | ||
| 105 | + &self, | ||
| 106 | + ctx: &FmtContext<'_, S, JsonFields>, | ||
| 107 | + writer: Writer<'_>, | ||
| 108 | + event: &Event<'_>, | ||
| 109 | + ) -> fmt::Result { | ||
| 110 | + let mut fields = collect_fields(ctx, event)?; | ||
| 111 | + enrich_ram_event(event, &mut fields); | ||
| 112 | + match self.settings.format { | ||
| 113 | + LogFormat::Json => self.format_json(writer, event, fields), | ||
| 114 | + LogFormat::Compact => self.format_compact(writer, event, fields), | ||
| 115 | + } | ||
| 116 | + } | ||
| 117 | +} | ||
| 118 | + | ||
| 119 | +impl RamLogFormatter { | ||
| 120 | + fn format_json( | ||
| 121 | + &self, | ||
| 122 | + mut writer: Writer<'_>, | ||
| 123 | + event: &Event<'_>, | ||
| 124 | + fields: BTreeMap<String, Value>, | ||
| 125 | + ) -> fmt::Result { | ||
| 126 | + let metadata = event.metadata(); | ||
| 127 | + let mut record = Map::new(); | ||
| 128 | + record.insert("timestamp".into(), Value::String(timestamp())); | ||
| 129 | + record.insert("level".into(), Value::String(metadata.level().to_string())); | ||
| 130 | + record.insert( | ||
| 131 | + "target".into(), | ||
| 132 | + Value::String(metadata.target().to_string()), | ||
| 133 | + ); | ||
| 134 | + record.insert( | ||
| 135 | + "fields".into(), | ||
| 136 | + Value::Object(fields.into_iter().collect::<Map<_, _>>()), | ||
| 137 | + ); | ||
| 138 | + if self.settings.source { | ||
| 139 | + if let Some(file) = metadata.file() { | ||
| 140 | + record.insert("filename".into(), Value::String(file.to_string())); | ||
| 141 | + } | ||
| 142 | + if let Some(line) = metadata.line() { | ||
| 143 | + record.insert("line_number".into(), Value::Number(line.into())); | ||
| 144 | + } | ||
| 145 | + } | ||
| 146 | + let encoded = serde_json::to_string(&Value::Object(record)).map_err(|_| fmt::Error)?; | ||
| 147 | + writeln!(writer, "{encoded}") | ||
| 148 | + } | ||
| 149 | + | ||
| 150 | + fn format_compact( | ||
| 151 | + &self, | ||
| 152 | + mut writer: Writer<'_>, | ||
| 153 | + event: &Event<'_>, | ||
| 154 | + mut fields: BTreeMap<String, Value>, | ||
| 155 | + ) -> fmt::Result { | ||
| 156 | + let metadata = event.metadata(); | ||
| 157 | + let failure = is_failure(metadata.level(), &fields); | ||
| 158 | + let operation = take_string(&mut fields, "operation").unwrap_or_else(|| "event".into()); | ||
| 159 | + let stage = take_string(&mut fields, "stage"); | ||
| 160 | + let context = stage | ||
| 161 | + .map(|stage| format!("{operation}/{stage}")) | ||
| 162 | + .unwrap_or(operation); | ||
| 163 | + let message = take_string(&mut fields, "message").unwrap_or_else(|| "event emitted".into()); | ||
| 164 | + | ||
| 165 | + write!(writer, "[{}] [{}] ", timestamp(), metadata.level())?; | ||
| 166 | + if failure { | ||
| 167 | + let origin_file = take_string(&mut fields, "error_origin_file") | ||
| 168 | + .or_else(|| metadata.file().map(str::to_owned)) | ||
| 169 | + .unwrap_or_else(|| "unknown".into()); | ||
| 170 | + let origin_line = take_u64(&mut fields, "error_origin_line") | ||
| 171 | + .or_else(|| metadata.line().map(u64::from)) | ||
| 172 | + .unwrap_or_default(); | ||
| 173 | + write!(writer, "[{origin_file}:{origin_line}] [{context}] ")?; | ||
| 174 | + let code = take_string(&mut fields, "error_code").unwrap_or_else(|| "ERROR".into()); | ||
| 175 | + let summary = take_string(&mut fields, "source_error_message").unwrap_or(message); | ||
| 176 | + write!(writer, "{code}: {}", single_line(&summary))?; | ||
| 177 | + fields.insert( | ||
| 178 | + "target".into(), | ||
| 179 | + Value::String(metadata.target().to_string()), | ||
| 180 | + ); | ||
| 181 | + } else { | ||
| 182 | + write!( | ||
| 183 | + writer, | ||
| 184 | + "[{}] [{}] {}", | ||
| 185 | + metadata.target(), | ||
| 186 | + context, | ||
| 187 | + single_line(&message) | ||
| 188 | + )?; | ||
| 189 | + } | ||
| 190 | + | ||
| 191 | + if self.settings.source { | ||
| 192 | + if let (Some(file), Some(line)) = (metadata.file(), metadata.line()) { | ||
| 193 | + fields.insert("log_at".into(), Value::String(format!("{file}:{line}"))); | ||
| 194 | + } | ||
| 195 | + } | ||
| 196 | + write_diagnostics(&mut writer, fields) | ||
| 197 | + } | ||
| 198 | +} | ||
| 199 | + | ||
| 200 | +fn collect_fields<S>( | ||
| 201 | + ctx: &FmtContext<'_, S, JsonFields>, | ||
| 202 | + event: &Event<'_>, | ||
| 203 | +) -> Result<BTreeMap<String, Value>, fmt::Error> | ||
| 204 | +where | ||
| 205 | + S: Subscriber + for<'lookup> LookupSpan<'lookup>, | ||
| 206 | +{ | ||
| 207 | + let mut fields = BTreeMap::new(); | ||
| 208 | + if let Some(scope) = ctx.event_scope() { | ||
| 209 | + for span in scope.from_root() { | ||
| 210 | + let extensions = span.extensions(); | ||
| 211 | + if let Some(formatted) = extensions.get::<FormattedFields<JsonFields>>() { | ||
| 212 | + merge_json_fields(&mut fields, &formatted.fields)?; | ||
| 213 | + } | ||
| 214 | + } | ||
| 215 | + } | ||
| 216 | + | ||
| 217 | + let mut encoded = String::new(); | ||
| 218 | + ctx.format_fields(Writer::new(&mut encoded), event)?; | ||
| 219 | + merge_json_fields(&mut fields, &encoded)?; | ||
| 220 | + Ok(fields) | ||
| 221 | +} | ||
| 222 | + | ||
| 223 | +fn merge_json_fields( | ||
| 224 | + target: &mut BTreeMap<String, Value>, | ||
| 225 | + encoded: &str, | ||
| 226 | +) -> Result<(), fmt::Error> { | ||
| 227 | + if encoded.is_empty() { | ||
| 228 | + return Ok(()); | ||
| 229 | + } | ||
| 230 | + let fields = | ||
| 231 | + serde_json::from_str::<BTreeMap<String, Value>>(encoded).map_err(|_| fmt::Error)?; | ||
| 232 | + target.extend(fields); | ||
| 233 | + Ok(()) | ||
| 234 | +} | ||
| 235 | + | ||
| 236 | +fn enrich_ram_event(event: &Event<'_>, fields: &mut BTreeMap<String, Value>) { | ||
| 237 | + let Some(event_name) = fields | ||
| 238 | + .get("event") | ||
| 239 | + .and_then(Value::as_str) | ||
| 240 | + .map(str::to_owned) | ||
| 241 | + else { | ||
| 242 | + return; | ||
| 243 | + }; | ||
| 244 | + if !event_name.starts_with("ram_a.") { | ||
| 245 | + return; | ||
| 246 | + } | ||
| 247 | + fields | ||
| 248 | + .entry("message".into()) | ||
| 249 | + .or_insert_with(|| Value::String(default_message(&event_name))); | ||
| 250 | + if !fields.contains_key("operation") { | ||
| 251 | + let operation = default_operation(&event_name, fields); | ||
| 252 | + fields.insert("operation".into(), Value::String(operation)); | ||
| 253 | + } | ||
| 254 | + | ||
| 255 | + if is_failure(event.metadata().level(), fields) { | ||
| 256 | + if !fields.contains_key("error_site") { | ||
| 257 | + let error_site = default_error_site(&event_name, fields); | ||
| 258 | + fields.insert("error_site".into(), Value::String(error_site)); | ||
| 259 | + } | ||
| 260 | + if let Some(file) = event.metadata().file() { | ||
| 261 | + fields | ||
| 262 | + .entry("error_origin_file".into()) | ||
| 263 | + .or_insert_with(|| Value::String(file.to_string())); | ||
| 264 | + } | ||
| 265 | + if let Some(line) = event.metadata().line() { | ||
| 266 | + fields | ||
| 267 | + .entry("error_origin_line".into()) | ||
| 268 | + .or_insert_with(|| Value::Number(line.into())); | ||
| 269 | + } | ||
| 270 | + if !fields.contains_key("source_error_kind") { | ||
| 271 | + let error_kind = default_error_kind(fields).to_string(); | ||
| 272 | + fields.insert("source_error_kind".into(), Value::String(error_kind)); | ||
| 273 | + } | ||
| 274 | + if !fields.contains_key("source_error_message") { | ||
| 275 | + let summary = default_error_summary(fields); | ||
| 276 | + fields.insert("source_error_message".into(), Value::String(summary)); | ||
| 277 | + } | ||
| 278 | + } | ||
| 279 | +} | ||
| 280 | + | ||
| 281 | +fn default_message(event: &str) -> String { | ||
| 282 | + if event.ends_with(".started") { | ||
| 283 | + "operation started".into() | ||
| 284 | + } else if event.ends_with(".completed") { | ||
| 285 | + "operation completed".into() | ||
| 286 | + } else if event.ends_with(".failed") { | ||
| 287 | + "operation failed".into() | ||
| 288 | + } else if event.ends_with(".retry") { | ||
| 289 | + "retrying provider request".into() | ||
| 290 | + } else if event.ends_with(".degraded") { | ||
| 291 | + "operation completed in degraded mode".into() | ||
| 292 | + } else { | ||
| 293 | + event | ||
| 294 | + .rsplit('.') | ||
| 295 | + .next() | ||
| 296 | + .unwrap_or("event") | ||
| 297 | + .replace('_', " ") | ||
| 298 | + } | ||
| 299 | +} | ||
| 300 | + | ||
| 301 | +fn default_operation(event: &str, fields: &BTreeMap<String, Value>) -> String { | ||
| 302 | + if let Some(tool) = fields.get("tool").and_then(Value::as_str) { | ||
| 303 | + return tool.to_string(); | ||
| 304 | + } | ||
| 305 | + if let Some(operation) = fields.get("operation").and_then(Value::as_str) { | ||
| 306 | + return operation.to_string(); | ||
| 307 | + } | ||
| 308 | + if event.starts_with("ram_a.memory.ingest") { | ||
| 309 | + "memory_ingest".into() | ||
| 310 | + } else if event.starts_with("ram_a.memory.search") { | ||
| 311 | + "memory_search".into() | ||
| 312 | + } else if event.starts_with("ram_a.provider") { | ||
| 313 | + "provider".into() | ||
| 314 | + } else if event.starts_with("ram_a.case") { | ||
| 315 | + "memory_case".into() | ||
| 316 | + } else if event.starts_with("ram_a.startup") { | ||
| 317 | + "startup".into() | ||
| 318 | + } else { | ||
| 319 | + "ram_a".into() | ||
| 320 | + } | ||
| 321 | +} | ||
| 322 | + | ||
| 323 | +fn default_error_site(event: &str, fields: &BTreeMap<String, Value>) -> String { | ||
| 324 | + let operation = default_operation(event, fields).replace('_', "."); | ||
| 325 | + let stage = fields | ||
| 326 | + .get("stage") | ||
| 327 | + .and_then(Value::as_str) | ||
| 328 | + .unwrap_or("failed"); | ||
| 329 | + format!("ram_a.{operation}.{stage}") | ||
| 330 | +} | ||
| 331 | + | ||
| 332 | +fn default_error_kind(fields: &BTreeMap<String, Value>) -> &'static str { | ||
| 333 | + match fields.get("error_code").and_then(Value::as_str) { | ||
| 334 | + Some("SQLITE_BUSY") => "sqlite_busy", | ||
| 335 | + Some("SQLITE_READONLY") => "sqlite_readonly", | ||
| 336 | + Some("EMBEDDING_FAILED") => "embedding", | ||
| 337 | + Some("RERANK_FAILED") => "rerank", | ||
| 338 | + Some("CANCELLED") => "cancelled", | ||
| 339 | + _ => "internal", | ||
| 340 | + } | ||
| 341 | +} | ||
| 342 | + | ||
| 343 | +fn default_error_summary(fields: &BTreeMap<String, Value>) -> String { | ||
| 344 | + match fields.get("error_code").and_then(Value::as_str) { | ||
| 345 | + Some("INVALID_REQUEST") => "request validation failed", | ||
| 346 | + Some("IDEMPOTENCY_CONFLICT") => "idempotency key conflicts with an earlier request", | ||
| 347 | + Some("PIPELINE_FAILED") => "memory pipeline failed", | ||
| 348 | + Some("PIPELINE_EXTRACT_FAILED") => "memory extraction failed", | ||
| 349 | + Some("PIPELINE_GROUND_FAILED") => "memory grounding failed", | ||
| 350 | + Some("RERANK_FAILED") => "memory rerank failed", | ||
| 351 | + Some("EMBEDDING_FAILED") => "embedding provider failed", | ||
| 352 | + Some("IDEMPOTENCY_STORAGE_FAILED") => "idempotency storage failed", | ||
| 353 | + Some("SQLITE_BUSY") => "SQLite database is busy", | ||
| 354 | + Some("SQLITE_READONLY") => "SQLite database is read-only", | ||
| 355 | + Some("VECTOR_PERSIST_FAILED") => "memory vector persistence failed", | ||
| 356 | + Some("STORAGE_FAILED") => "memory storage failed", | ||
| 357 | + Some("CANCELLED") => "operation was cancelled", | ||
| 358 | + Some(_) | None => "operation failed", | ||
| 359 | + } | ||
| 360 | + .into() | ||
| 361 | +} | ||
| 362 | + | ||
| 363 | +fn is_failure(level: &tracing::Level, fields: &BTreeMap<String, Value>) -> bool { | ||
| 364 | + *level == tracing::Level::ERROR | ||
| 365 | + || fields.contains_key("error_code") | ||
| 366 | + || fields | ||
| 367 | + .get("event") | ||
| 368 | + .and_then(Value::as_str) | ||
| 369 | + .is_some_and(|event| event.ends_with(".failed")) | ||
| 370 | +} | ||
| 371 | + | ||
| 372 | +fn timestamp() -> String { | ||
| 373 | + Utc::now().to_rfc3339_opts(SecondsFormat::Millis, true) | ||
| 374 | +} | ||
| 375 | + | ||
| 376 | +fn take_string(fields: &mut BTreeMap<String, Value>, key: &str) -> Option<String> { | ||
| 377 | + fields.remove(key).and_then(|value| match value { | ||
| 378 | + Value::String(value) => Some(value), | ||
| 379 | + Value::Null => None, | ||
| 380 | + value => Some(value.to_string()), | ||
| 381 | + }) | ||
| 382 | +} | ||
| 383 | + | ||
| 384 | +fn take_u64(fields: &mut BTreeMap<String, Value>, key: &str) -> Option<u64> { | ||
| 385 | + fields.remove(key).and_then(|value| value.as_u64()) | ||
| 386 | +} | ||
| 387 | + | ||
| 388 | +fn write_diagnostics(writer: &mut Writer<'_>, mut fields: BTreeMap<String, Value>) -> fmt::Result { | ||
| 389 | + for key in [ | ||
| 390 | + "message", | ||
| 391 | + "operation", | ||
| 392 | + "stage", | ||
| 393 | + "error_code", | ||
| 394 | + "source_error_message", | ||
| 395 | + "error_origin_file", | ||
| 396 | + "error_origin_line", | ||
| 397 | + ] { | ||
| 398 | + fields.remove(key); | ||
| 399 | + } | ||
| 400 | + if fields.is_empty() { | ||
| 401 | + return writeln!(writer); | ||
| 402 | + } | ||
| 403 | + | ||
| 404 | + write!(writer, " |")?; | ||
| 405 | + for key in [ | ||
| 406 | + "target", | ||
| 407 | + "error_site", | ||
| 408 | + "request_id", | ||
| 409 | + "pipeline_run_id", | ||
| 410 | + "retriable", | ||
| 411 | + "source_error_kind", | ||
| 412 | + "completed_units", | ||
| 413 | + "total_units", | ||
| 414 | + "elapsed_ms", | ||
| 415 | + "latency_ms", | ||
| 416 | + "log_at", | ||
| 417 | + "event", | ||
| 418 | + ] { | ||
| 419 | + if let Some(value) = fields.remove(key) { | ||
| 420 | + write!(writer, " {key}={}", display_value(&value))?; | ||
| 421 | + } | ||
| 422 | + } | ||
| 423 | + for (key, value) in fields { | ||
| 424 | + write!(writer, " {key}={}", display_value(&value))?; | ||
| 425 | + } | ||
| 426 | + writeln!(writer) | ||
| 427 | +} | ||
| 428 | + | ||
| 429 | +fn display_value(value: &Value) -> String { | ||
| 430 | + match value { | ||
| 431 | + Value::String(value) if needs_quotes(value) => { | ||
| 432 | + serde_json::to_string(value).unwrap_or_else(|_| "\"<invalid>\"".into()) | ||
| 433 | + } | ||
| 434 | + Value::String(value) => single_line(value), | ||
| 435 | + value => single_line(&value.to_string()), | ||
| 436 | + } | ||
| 437 | +} | ||
| 438 | + | ||
| 439 | +fn needs_quotes(value: &str) -> bool { | ||
| 440 | + value.chars().any(|character| { | ||
| 441 | + character.is_whitespace() || matches!(character, '=' | '"' | '\\' | '|' | '[' | ']') | ||
| 442 | + }) | ||
| 443 | +} | ||
| 444 | + | ||
| 445 | +fn single_line(value: &str) -> String { | ||
| 446 | + value | ||
| 447 | + .replace('\\', "\\\\") | ||
| 448 | + .replace('\n', "\\n") | ||
| 449 | + .replace('\r', "\\r") | ||
| 450 | + .replace('\t', "\\t") | ||
| 451 | +} | ||
| 452 | + | ||
| 453 | + | ||
| 454 | +mod tests { | ||
| 455 | + use std::io::{self, Write}; | ||
| 456 | + use std::sync::{Arc, Mutex}; | ||
| 457 | + | ||
| 458 | + use tracing_subscriber::fmt::MakeWriter; | ||
| 459 | + use tracing_subscriber::prelude::*; | ||
| 460 | + | ||
| 461 | + use super::*; | ||
| 462 | + | ||
| 463 | + | ||
| 464 | + struct LogBuffer(Arc<Mutex<Vec<u8>>>); | ||
| 465 | + | ||
| 466 | + struct LogWriter(Arc<Mutex<Vec<u8>>>); | ||
| 467 | + | ||
| 468 | + impl Write for LogWriter { | ||
| 469 | + fn write(&mut self, bytes: &[u8]) -> io::Result<usize> { | ||
| 470 | + self.0.lock().expect("log buffer lock").extend(bytes); | ||
| 471 | + Ok(bytes.len()) | ||
| 472 | + } | ||
| 473 | + | ||
| 474 | + fn flush(&mut self) -> io::Result<()> { | ||
| 475 | + Ok(()) | ||
| 476 | + } | ||
| 477 | + } | ||
| 478 | + | ||
| 479 | + impl<'writer> MakeWriter<'writer> for LogBuffer { | ||
| 480 | + type Writer = LogWriter; | ||
| 481 | + | ||
| 482 | + fn make_writer(&'writer self) -> Self::Writer { | ||
| 483 | + LogWriter(self.0.clone()) | ||
| 484 | + } | ||
| 485 | + } | ||
| 486 | + | ||
| 487 | + impl LogBuffer { | ||
| 488 | + fn text(&self) -> String { | ||
| 489 | + String::from_utf8(self.0.lock().expect("log buffer lock").clone()) | ||
| 490 | + .expect("UTF-8 log output") | ||
| 491 | + } | ||
| 492 | + } | ||
| 493 | + | ||
| 494 | + fn capture(settings: LogSettings, emit: impl FnOnce()) -> String { | ||
| 495 | + let logs = LogBuffer::default(); | ||
| 496 | + let layer = tracing_subscriber::fmt::layer() | ||
| 497 | + .fmt_fields(JsonFields::new()) | ||
| 498 | + .event_format(RamLogFormatter::new(settings)) | ||
| 499 | + .with_writer(logs.clone()); | ||
| 500 | + let subscriber = tracing_subscriber::registry().with(layer); | ||
| 501 | + tracing::subscriber::with_default(subscriber, emit); | ||
| 502 | + logs.text() | ||
| 503 | + } | ||
| 504 | + | ||
| 505 | + fn emit_failure() { | ||
| 506 | + let span = tracing::info_span!( | ||
| 507 | + "ram_a.tool", | ||
| 508 | + request_id = "request-123", | ||
| 509 | + tool = "memory_ingest" | ||
| 510 | + ); | ||
| 511 | + let _guard = span.enter(); | ||
| 512 | + tracing::error!( | ||
| 513 | + event = "ram_a.memory.ingest.failed", | ||
| 514 | + stage = "vector_persist", | ||
| 515 | + error_code = "EMBEDDING_FAILED", | ||
| 516 | + retriable = true, | ||
| 517 | + error_site = "memory_mcp.ingest.vector_persist", | ||
| 518 | + error_origin_file = "crates/memory-core/src/embedding.rs", | ||
| 519 | + error_origin_line = 153_u64, | ||
| 520 | + source_error_kind = "timeout", | ||
| 521 | + source_error_message = "embedding provider timed out" | ||
| 522 | + ); | ||
| 523 | + } | ||
| 524 | + | ||
| 525 | + | ||
| 526 | + fn settings_are_strict_and_have_production_defaults() { | ||
| 527 | + assert_eq!(parse_format(None).unwrap(), LogFormat::Json); | ||
| 528 | + assert!(!parse_source(None).unwrap()); | ||
| 529 | + assert_eq!( | ||
| 530 | + parse_format(Some("compact".into())).unwrap(), | ||
| 531 | + LogFormat::Compact | ||
| 532 | + ); | ||
| 533 | + assert!(parse_source(Some("true".into())).unwrap()); | ||
| 534 | + | ||
| 535 | + for invalid in ["", "JSON", " compact", "text"] { | ||
| 536 | + assert_eq!( | ||
| 537 | + parse_format(Some(invalid.into())), | ||
| 538 | + Err(LogSettingsError::InvalidFormat) | ||
| 539 | + ); | ||
| 540 | + } | ||
| 541 | + for invalid in ["", "TRUE", "1", " false"] { | ||
| 542 | + assert_eq!( | ||
| 543 | + parse_source(Some(invalid.into())), | ||
| 544 | + Err(LogSettingsError::InvalidSource) | ||
| 545 | + ); | ||
| 546 | + } | ||
| 547 | + } | ||
| 548 | + | ||
| 549 | + | ||
| 550 | + fn compact_values_are_unambiguous_and_single_line() { | ||
| 551 | + assert_eq!(display_value(&Value::String("plain".into())), "plain"); | ||
| 552 | + assert_eq!( | ||
| 553 | + display_value(&Value::String("two words".into())), | ||
| 554 | + "\"two words\"" | ||
| 555 | + ); | ||
| 556 | + assert_eq!(display_value(&Value::String("a\nb".into())), "\"a\\nb\""); | ||
| 557 | + assert_eq!(single_line("a\rb\tc"), "a\\rb\\tc"); | ||
| 558 | + } | ||
| 559 | + | ||
| 560 | + | ||
| 561 | + fn json_keeps_error_origin_when_source_is_disabled() { | ||
| 562 | + let output = capture(LogSettings::default(), emit_failure); | ||
| 563 | + let record: Value = serde_json::from_str(output.trim()).expect("JSON log record"); | ||
| 564 | + | ||
| 565 | + assert_eq!(record["fields"]["request_id"], "request-123"); | ||
| 566 | + assert_eq!( | ||
| 567 | + record["fields"]["error_origin_file"], | ||
| 568 | + "crates/memory-core/src/embedding.rs" | ||
| 569 | + ); | ||
| 570 | + assert_eq!(record["fields"]["error_origin_line"], 153); | ||
| 571 | + assert!(record.get("filename").is_none()); | ||
| 572 | + assert!(record.get("line_number").is_none()); | ||
| 573 | + } | ||
| 574 | + | ||
| 575 | + | ||
| 576 | + fn compact_puts_origin_and_error_summary_before_diagnostics() { | ||
| 577 | + let output = capture( | ||
| 578 | + LogSettings { | ||
| 579 | + format: LogFormat::Compact, | ||
| 580 | + source: false, | ||
| 581 | + }, | ||
| 582 | + emit_failure, | ||
| 583 | + ); | ||
| 584 | + | ||
| 585 | + assert!(output.contains( | ||
| 586 | + "[ERROR] [crates/memory-core/src/embedding.rs:153] [memory_ingest/vector_persist] EMBEDDING_FAILED: embedding provider timed out |" | ||
| 587 | + )); | ||
| 588 | + assert!(output.contains("request_id=request-123")); | ||
| 589 | + assert!(output.contains("site=memory_mcp.ingest.vector_persist")); | ||
| 590 | + assert!(!output.contains("log_at=")); | ||
| 591 | + assert_eq!(output.lines().count(), 1); | ||
| 592 | + } | ||
| 593 | + | ||
| 594 | + | ||
| 595 | + fn source_switch_adds_log_call_site_without_replacing_error_origin() { | ||
| 596 | + let output = capture( | ||
| 597 | + LogSettings { | ||
| 598 | + format: LogFormat::Compact, | ||
| 599 | + source: true, | ||
| 600 | + }, | ||
| 601 | + emit_failure, | ||
| 602 | + ); | ||
| 603 | + | ||
| 604 | + assert!(output.contains("[crates/memory-core/src/embedding.rs:153]")); | ||
| 605 | + assert!(output.contains("log_at=crates/memory-mcp/src/observability.rs:")); | ||
| 606 | + } | ||
| 607 | +} | ||
| @@ -5,9 +5,10 @@ use std::time::Instant; | |||
| 5 | 5 | ||
| 6 | use chrono::{DateTime, NaiveDate, Utc}; | 6 | use chrono::{DateTime, NaiveDate, Utc}; |
| 7 | use memory_core::{ | 7 | use memory_core::{ |
| 8 | - AddMemoryRequest, GraphAddMemoryRequest, GraphBuildPipeline, LongTermMemory, MemoryManager, | 8 | + AddMemoryRequest, GraphAddMemoryRequest, GraphBuildPipeline, LongTermMemory, MemoryError, |
| 9 | - MemoryRecord, SearchMemoryRequest as CoreSearchRequest, | 9 | + MemoryManager, MemoryRecord, SearchMemoryRequest as CoreSearchRequest, |
| 10 | }; | 10 | }; |
| 11 | +use memory_pipeline::error::{PipelineError, PipelineStage}; | ||
| 11 | use memory_pipeline::extraction::MemoryExtractor; | 12 | use memory_pipeline::extraction::MemoryExtractor; |
| 12 | use memory_pipeline::grounding::GroundingVerifier; | 13 | use memory_pipeline::grounding::GroundingVerifier; |
| 13 | use memory_pipeline::pipeline::{run_memory_pipeline, PipelineConfig}; | 14 | use memory_pipeline::pipeline::{run_memory_pipeline, PipelineConfig}; |
| @@ -70,22 +71,92 @@ struct GraphMemoryRuntime { | |||
| 70 | pub enum ServiceError { | 71 | pub enum ServiceError { |
| 71 | InvalidRequest, | 72 | InvalidRequest, |
| 72 | IdempotencyConflict, | 73 | IdempotencyConflict, |
| 73 | - Pipeline, | 74 | + Pipeline { |
| 75 | + stage: Option<PipelineStage>, | ||
| 76 | + }, | ||
| 77 | + Rerank, | ||
| 74 | Storage, | 78 | Storage, |
| 79 | + Observed { | ||
| 80 | + code: &'static str, | ||
| 81 | + message: &'static str, | ||
| 82 | + retriable: bool, | ||
| 83 | + stage: Option<PipelineStage>, | ||
| 84 | + error_site: &'static str, | ||
| 85 | + error_origin_file: &'static str, | ||
| 86 | + error_origin_line: u32, | ||
| 87 | + source_error_kind: &'static str, | ||
| 88 | + source_error_message: &'static str, | ||
| 89 | + }, | ||
| 75 | } | 90 | } |
| 76 | 91 | ||
| 77 | impl ServiceError { | 92 | impl ServiceError { |
| 78 | - pub fn code(self) -> &'static str { | 93 | + pub fn code(&self) -> &'static str { |
| 79 | match self { | 94 | match self { |
| 80 | Self::InvalidRequest => "INVALID_REQUEST", | 95 | Self::InvalidRequest => "INVALID_REQUEST", |
| 81 | Self::IdempotencyConflict => "IDEMPOTENCY_CONFLICT", | 96 | Self::IdempotencyConflict => "IDEMPOTENCY_CONFLICT", |
| 82 | - Self::Pipeline => "PIPELINE_FAILED", | 97 | + Self::Pipeline { .. } => "PIPELINE_FAILED", |
| 98 | + Self::Rerank => "RERANK_FAILED", | ||
| 83 | Self::Storage => "STORAGE_FAILED", | 99 | Self::Storage => "STORAGE_FAILED", |
| 100 | + Self::Observed { code, .. } => code, | ||
| 84 | } | 101 | } |
| 85 | } | 102 | } |
| 86 | 103 | ||
| 87 | - pub fn retriable(self) -> bool { | 104 | + pub fn retriable(&self) -> bool { |
| 88 | - matches!(self, Self::Pipeline | Self::Storage) | 105 | + match self { |
| 106 | + Self::Observed { retriable, .. } => *retriable, | ||
| 107 | + error => matches!(error, Self::Pipeline { .. } | Self::Rerank | Self::Storage), | ||
| 108 | + } | ||
| 109 | + } | ||
| 110 | + | ||
| 111 | + pub fn stage(&self) -> Option<&'static str> { | ||
| 112 | + match self { | ||
| 113 | + Self::Pipeline { stage: Some(stage) } => Some(stage.as_str()), | ||
| 114 | + Self::Observed { | ||
| 115 | + stage: Some(stage), .. | ||
| 116 | + } => Some(stage.as_str()), | ||
| 117 | + _ => None, | ||
| 118 | + } | ||
| 119 | + } | ||
| 120 | + | ||
| 121 | + pub fn error_site(&self) -> Option<&'static str> { | ||
| 122 | + match self { | ||
| 123 | + Self::Observed { error_site, .. } => Some(error_site), | ||
| 124 | + _ => None, | ||
| 125 | + } | ||
| 126 | + } | ||
| 127 | + | ||
| 128 | + pub fn error_origin(&self) -> Option<(&'static str, u32)> { | ||
| 129 | + match self { | ||
| 130 | + Self::Observed { | ||
| 131 | + error_origin_file, | ||
| 132 | + error_origin_line, | ||
| 133 | + .. | ||
| 134 | + } => Some((error_origin_file, *error_origin_line)), | ||
| 135 | + _ => None, | ||
| 136 | + } | ||
| 137 | + } | ||
| 138 | + | ||
| 139 | + pub fn source_error_kind(&self) -> Option<&'static str> { | ||
| 140 | + match self { | ||
| 141 | + Self::Observed { | ||
| 142 | + source_error_kind, .. | ||
| 143 | + } => Some(source_error_kind), | ||
| 144 | + _ => None, | ||
| 145 | + } | ||
| 146 | + } | ||
| 147 | + | ||
| 148 | + pub fn source_error_message(&self) -> Option<&'static str> { | ||
| 149 | + match self { | ||
| 150 | + Self::Observed { | ||
| 151 | + source_error_message, | ||
| 152 | + .. | ||
| 153 | + } => Some(source_error_message), | ||
| 154 | + _ => None, | ||
| 155 | + } | ||
| 156 | + } | ||
| 157 | + | ||
| 158 | + fn pipeline(stage: Option<PipelineStage>) -> Self { | ||
| 159 | + Self::Pipeline { stage } | ||
| 89 | } | 160 | } |
| 90 | } | 161 | } |
| 91 | 162 | ||
| @@ -94,8 +165,10 @@ impl fmt::Display for ServiceError { | |||
| 94 | formatter.write_str(match self { | 165 | formatter.write_str(match self { |
| 95 | Self::InvalidRequest => "memory request is invalid", | 166 | Self::InvalidRequest => "memory request is invalid", |
| 96 | Self::IdempotencyConflict => "idempotency key conflicts with an earlier request", | 167 | Self::IdempotencyConflict => "idempotency key conflicts with an earlier request", |
| 97 | - Self::Pipeline => "memory pipeline failed", | 168 | + Self::Pipeline { .. } => "memory pipeline failed", |
| 169 | + Self::Rerank => "memory rerank failed", | ||
| 98 | Self::Storage => "memory storage failed", | 170 | Self::Storage => "memory storage failed", |
| 171 | + Self::Observed { message, .. } => message, | ||
| 99 | }) | 172 | }) |
| 100 | } | 173 | } |
| 101 | } | 174 | } |
| @@ -188,12 +261,12 @@ where | |||
| 188 | let mut stage_started = Instant::now(); | 261 | let mut stage_started = Instant::now(); |
| 189 | tracing::info!( | 262 | tracing::info!( |
| 190 | event = "ram_a.memory.ingest.stage.started", | 263 | event = "ram_a.memory.ingest.stage.started", |
| 191 | - stage = "validate" | 264 | + stage = "request_validate" |
| 192 | ); | 265 | ); |
| 193 | request.validate().map_err(|_| { | 266 | request.validate().map_err(|_| { |
| 194 | tracing::error!( | 267 | tracing::error!( |
| 195 | event = "ram_a.memory.ingest.stage.failed", | 268 | event = "ram_a.memory.ingest.stage.failed", |
| 196 | - stage = "validate", | 269 | + stage = "request_validate", |
| 197 | error_code = ServiceError::InvalidRequest.code(), | 270 | error_code = ServiceError::InvalidRequest.code(), |
| 198 | retriable = false, | 271 | retriable = false, |
| 199 | elapsed_ms = stage_started.elapsed().as_millis() as u64 | 272 | elapsed_ms = stage_started.elapsed().as_millis() as u64 |
| @@ -202,7 +275,7 @@ where | |||
| 202 | })?; | 275 | })?; |
| 203 | tracing::info!( | 276 | tracing::info!( |
| 204 | event = "ram_a.memory.ingest.stage.completed", | 277 | event = "ram_a.memory.ingest.stage.completed", |
| 205 | - stage = "validate", | 278 | + stage = "request_validate", |
| 206 | elapsed_ms = stage_started.elapsed().as_millis() as u64 | 279 | elapsed_ms = stage_started.elapsed().as_millis() as u64 |
| 207 | ); | 280 | ); |
| 208 | let scope_id = principal.scope_id(); | 281 | let scope_id = principal.scope_id(); |
| @@ -234,12 +307,23 @@ where | |||
| 234 | .reserve(&entries, &proposed_run_id) | 307 | .reserve(&entries, &proposed_run_id) |
| 235 | .await | 308 | .await |
| 236 | .map_err(|error| { | 309 | .map_err(|error| { |
| 237 | - let mapped = map_idempotency_error(error); | 310 | + let mapped = map_idempotency_error(error, "memory_mcp.idempotency.reserve"); |
| 311 | + let (origin_file, origin_line) = | ||
| 312 | + mapped.error_origin().unwrap_or((file!(), line!())); | ||
| 238 | tracing::error!( | 313 | tracing::error!( |
| 239 | event = "ram_a.memory.ingest.stage.failed", | 314 | event = "ram_a.memory.ingest.stage.failed", |
| 240 | stage = "idempotency_reserve", | 315 | stage = "idempotency_reserve", |
| 241 | error_code = mapped.code(), | 316 | error_code = mapped.code(), |
| 242 | retriable = mapped.retriable(), | 317 | retriable = mapped.retriable(), |
| 318 | + error_site = mapped | ||
| 319 | + .error_site() | ||
| 320 | + .unwrap_or("memory_mcp.idempotency.reserve"), | ||
| 321 | + error_origin_file = origin_file, | ||
| 322 | + error_origin_line = origin_line, | ||
| 323 | + source_error_kind = mapped.source_error_kind().unwrap_or("internal"), | ||
| 324 | + source_error_message = mapped | ||
| 325 | + .source_error_message() | ||
| 326 | + .unwrap_or("idempotency reserve failed"), | ||
| 243 | elapsed_ms = stage_started.elapsed().as_millis() as u64 | 327 | elapsed_ms = stage_started.elapsed().as_millis() as u64 |
| 244 | ); | 328 | ); |
| 245 | mapped | 329 | mapped |
| @@ -277,15 +361,24 @@ where | |||
| 277 | None, | 361 | None, |
| 278 | ) | 362 | ) |
| 279 | .await | 363 | .await |
| 280 | - .map_err(|_| { | 364 | + .map_err(|error| { |
| 365 | + let mapped = map_pipeline_error(error); | ||
| 366 | + let (origin_file, origin_line) = | ||
| 367 | + mapped.error_origin().unwrap_or((file!(), line!())); | ||
| 281 | tracing::error!( | 368 | tracing::error!( |
| 282 | event = "ram_a.memory.ingest.stage.failed", | 369 | event = "ram_a.memory.ingest.stage.failed", |
| 283 | stage = "memory_pipeline", | 370 | stage = "memory_pipeline", |
| 284 | - error_code = ServiceError::Pipeline.code(), | 371 | + pipeline_run_id = %pipeline_run_id, |
| 285 | - retriable = true, | 372 | + error_code = mapped.code(), |
| 373 | + retriable = mapped.retriable(), | ||
| 374 | + error_site = mapped.error_site().unwrap_or("memory_pipeline.failed"), | ||
| 375 | + error_origin_file = origin_file, | ||
| 376 | + error_origin_line = origin_line, | ||
| 377 | + source_error_kind = mapped.source_error_kind().unwrap_or("internal"), | ||
| 378 | + source_error_message = mapped.source_error_message().unwrap_or("memory pipeline failed"), | ||
| 286 | elapsed_ms = stage_started.elapsed().as_millis() as u64 | 379 | elapsed_ms = stage_started.elapsed().as_millis() as u64 |
| 287 | ); | 380 | ); |
| 288 | - ServiceError::Pipeline | 381 | + mapped |
| 289 | })?; | 382 | })?; |
| 290 | tracing::info!( | 383 | tracing::info!( |
| 291 | event = "ram_a.memory.ingest.stage.completed", | 384 | event = "ram_a.memory.ingest.stage.completed", |
| @@ -311,15 +404,24 @@ where | |||
| 311 | .manager | 404 | .manager |
| 312 | .add_many(requests) | 405 | .add_many(requests) |
| 313 | .await | 406 | .await |
| 314 | - .map_err(|_| { | 407 | + .map_err(|error| { |
| 408 | + let mapped = map_persist_error(error); | ||
| 409 | + let (origin_file, origin_line) = | ||
| 410 | + mapped.error_origin().unwrap_or((file!(), line!())); | ||
| 315 | tracing::error!( | 411 | tracing::error!( |
| 316 | event = "ram_a.memory.ingest.stage.failed", | 412 | event = "ram_a.memory.ingest.stage.failed", |
| 317 | stage = "vector_persist", | 413 | stage = "vector_persist", |
| 318 | - error_code = ServiceError::Storage.code(), | 414 | + pipeline_run_id = %pipeline_run_id, |
| 319 | - retriable = true, | 415 | + error_code = mapped.code(), |
| 416 | + retriable = mapped.retriable(), | ||
| 417 | + error_site = mapped.error_site().unwrap_or("memory_mcp.ingest.vector_persist"), | ||
| 418 | + error_origin_file = origin_file, | ||
| 419 | + error_origin_line = origin_line, | ||
| 420 | + source_error_kind = mapped.source_error_kind().unwrap_or("internal"), | ||
| 421 | + source_error_message = mapped.source_error_message().unwrap_or("memory vector persistence failed"), | ||
| 320 | elapsed_ms = stage_started.elapsed().as_millis() as u64 | 422 | elapsed_ms = stage_started.elapsed().as_millis() as u64 |
| 321 | ); | 423 | ); |
| 322 | - ServiceError::Storage | 424 | + mapped |
| 323 | })? | 425 | })? |
| 324 | .into_iter() | 426 | .into_iter() |
| 325 | .map(|response| response.id) | 427 | .map(|response| response.id) |
| @@ -379,12 +481,23 @@ where | |||
| 379 | .complete(&pending_entries, &pipeline_run_id, &result) | 481 | .complete(&pending_entries, &pipeline_run_id, &result) |
| 380 | .await | 482 | .await |
| 381 | .map_err(|error| { | 483 | .map_err(|error| { |
| 382 | - let mapped = map_idempotency_error(error); | 484 | + let mapped = map_idempotency_error( |
| 485 | + error, | ||
| 486 | + "memory_mcp.idempotency.complete", | ||
| 487 | + ); | ||
| 488 | + let (origin_file, origin_line) = | ||
| 489 | + mapped.error_origin().unwrap_or((file!(), line!())); | ||
| 383 | tracing::error!( | 490 | tracing::error!( |
| 384 | event = "ram_a.memory.ingest.stage.failed", | 491 | event = "ram_a.memory.ingest.stage.failed", |
| 385 | stage = "idempotency_complete", | 492 | stage = "idempotency_complete", |
| 493 | + pipeline_run_id = %pipeline_run_id, | ||
| 386 | error_code = mapped.code(), | 494 | error_code = mapped.code(), |
| 387 | retriable = mapped.retriable(), | 495 | retriable = mapped.retriable(), |
| 496 | + error_site = mapped.error_site().unwrap_or("memory_mcp.idempotency.complete"), | ||
| 497 | + error_origin_file = origin_file, | ||
| 498 | + error_origin_line = origin_line, | ||
| 499 | + source_error_kind = mapped.source_error_kind().unwrap_or("internal"), | ||
| 500 | + source_error_message = mapped.source_error_message().unwrap_or("idempotency complete failed"), | ||
| 388 | elapsed_ms = stage_started.elapsed().as_millis() as u64 | 501 | elapsed_ms = stage_started.elapsed().as_millis() as u64 |
| 389 | ); | 502 | ); |
| 390 | mapped | 503 | mapped |
| @@ -448,15 +561,25 @@ where | |||
| 448 | graph_target_evidence_speaker: None, | 561 | graph_target_evidence_speaker: None, |
| 449 | }) | 562 | }) |
| 450 | .await | 563 | .await |
| 451 | - .map_err(|_| { | 564 | + .map_err(|error| { |
| 565 | + let mapped = map_search_error(error); | ||
| 566 | + let (origin_file, origin_line) = | ||
| 567 | + mapped.error_origin().unwrap_or((file!(), line!())); | ||
| 452 | tracing::error!( | 568 | tracing::error!( |
| 453 | event = "ram_a.memory.search.stage.failed", | 569 | event = "ram_a.memory.search.stage.failed", |
| 454 | stage = "retrieve", | 570 | stage = "retrieve", |
| 455 | - error_code = ServiceError::Storage.code(), | 571 | + error_code = mapped.code(), |
| 456 | - retriable = true, | 572 | + retriable = mapped.retriable(), |
| 573 | + error_site = mapped.error_site().unwrap_or("memory_mcp.search.retrieve"), | ||
| 574 | + error_origin_file = origin_file, | ||
| 575 | + error_origin_line = origin_line, | ||
| 576 | + source_error_kind = mapped.source_error_kind().unwrap_or("internal"), | ||
| 577 | + source_error_message = mapped | ||
| 578 | + .source_error_message() | ||
| 579 | + .unwrap_or("memory retrieval failed"), | ||
| 457 | elapsed_ms = stage_started.elapsed().as_millis() as u64 | 580 | elapsed_ms = stage_started.elapsed().as_millis() as u64 |
| 458 | ); | 581 | ); |
| 459 | - ServiceError::Storage | 582 | + mapped |
| 460 | })?; | 583 | })?; |
| 461 | let candidate_count = candidates.len(); | 584 | let candidate_count = candidates.len(); |
| 462 | tracing::info!( | 585 | tracing::info!( |
| @@ -555,13 +678,182 @@ fn content_hash(message: &IngestMessage) -> String { | |||
| 555 | format!("{:x}", digest.finalize()) | 678 | format!("{:x}", digest.finalize()) |
| 556 | } | 679 | } |
| 557 | 680 | ||
| 558 | -fn map_idempotency_error(error: IdempotencyError) -> ServiceError { | 681 | +#[track_caller] |
| 682 | +fn map_idempotency_error(error: IdempotencyError, operation: &'static str) -> ServiceError { | ||
| 559 | match error { | 683 | match error { |
| 560 | IdempotencyError::Conflict => ServiceError::IdempotencyConflict, | 684 | IdempotencyError::Conflict => ServiceError::IdempotencyConflict, |
| 561 | - IdempotencyError::Storage => ServiceError::Storage, | 685 | + IdempotencyError::Busy => observed_error( |
| 686 | + "SQLITE_BUSY", | ||
| 687 | + "SQLite database is busy", | ||
| 688 | + true, | ||
| 689 | + None, | ||
| 690 | + operation, | ||
| 691 | + "sqlite_busy", | ||
| 692 | + "SQLite database is busy", | ||
| 693 | + ), | ||
| 694 | + IdempotencyError::ReadOnly => observed_error( | ||
| 695 | + "SQLITE_READONLY", | ||
| 696 | + "SQLite database is read-only", | ||
| 697 | + false, | ||
| 698 | + None, | ||
| 699 | + operation, | ||
| 700 | + "sqlite_readonly", | ||
| 701 | + "SQLite database is read-only", | ||
| 702 | + ), | ||
| 703 | + IdempotencyError::Storage => observed_error( | ||
| 704 | + "IDEMPOTENCY_STORAGE_FAILED", | ||
| 705 | + "idempotency storage failed", | ||
| 706 | + true, | ||
| 707 | + None, | ||
| 708 | + operation, | ||
| 709 | + "sqlite_other", | ||
| 710 | + "idempotency storage operation failed", | ||
| 711 | + ), | ||
| 562 | } | 712 | } |
| 563 | } | 713 | } |
| 564 | 714 | ||
| 715 | + | ||
| 716 | +fn map_pipeline_error(error: PipelineError) -> ServiceError { | ||
| 717 | + let stage = error.stage(); | ||
| 718 | + let source_error_kind = error.source_error_kind(); | ||
| 719 | + let source_error_message = error.safe_summary(); | ||
| 720 | + let origin = error.origin(); | ||
| 721 | + let caller = std::panic::Location::caller(); | ||
| 722 | + ServiceError::Observed { | ||
| 723 | + code: "PIPELINE_FAILED", | ||
| 724 | + message: "memory pipeline failed", | ||
| 725 | + retriable: true, | ||
| 726 | + stage, | ||
| 727 | + error_site: origin | ||
| 728 | + .map(|origin| origin.site) | ||
| 729 | + .unwrap_or("memory_pipeline.failed"), | ||
| 730 | + error_origin_file: origin.map(|origin| origin.file).unwrap_or(caller.file()), | ||
| 731 | + error_origin_line: origin.map(|origin| origin.line).unwrap_or(caller.line()), | ||
| 732 | + source_error_kind, | ||
| 733 | + source_error_message, | ||
| 734 | + } | ||
| 735 | +} | ||
| 736 | + | ||
| 737 | + | ||
| 738 | +fn map_search_error(error: MemoryError) -> ServiceError { | ||
| 739 | + map_memory_error(error, "memory_mcp.search.retrieve", false) | ||
| 740 | +} | ||
| 741 | + | ||
| 742 | + | ||
| 743 | +fn map_persist_error(error: MemoryError) -> ServiceError { | ||
| 744 | + map_memory_error(error, "memory_mcp.ingest.vector_persist", true) | ||
| 745 | +} | ||
| 746 | + | ||
| 747 | + | ||
| 748 | +fn map_memory_error(error: MemoryError, operation: &'static str, persist: bool) -> ServiceError { | ||
| 749 | + match &error { | ||
| 750 | + MemoryError::Embedding { .. } => observed_error( | ||
| 751 | + "EMBEDDING_FAILED", | ||
| 752 | + "embedding provider failed", | ||
| 753 | + true, | ||
| 754 | + None, | ||
| 755 | + operation, | ||
| 756 | + "embedding", | ||
| 757 | + "embedding provider request failed", | ||
| 758 | + ), | ||
| 759 | + MemoryError::Rerank { .. } => observed_error( | ||
| 760 | + "RERANK_FAILED", | ||
| 761 | + "memory rerank failed", | ||
| 762 | + true, | ||
| 763 | + None, | ||
| 764 | + operation, | ||
| 765 | + "rerank", | ||
| 766 | + "memory rerank request failed", | ||
| 767 | + ), | ||
| 768 | + MemoryError::Sqlite(error) if sqlite_error_is_busy(error) => observed_error( | ||
| 769 | + "SQLITE_BUSY", | ||
| 770 | + "SQLite database is busy", | ||
| 771 | + true, | ||
| 772 | + None, | ||
| 773 | + operation, | ||
| 774 | + "sqlite_busy", | ||
| 775 | + "SQLite database is busy", | ||
| 776 | + ), | ||
| 777 | + MemoryError::Sqlite(error) if sqlite_error_is_read_only(error) => observed_error( | ||
| 778 | + "SQLITE_READONLY", | ||
| 779 | + "SQLite database is read-only", | ||
| 780 | + false, | ||
| 781 | + None, | ||
| 782 | + operation, | ||
| 783 | + "sqlite_readonly", | ||
| 784 | + "SQLite database is read-only", | ||
| 785 | + ), | ||
| 786 | + _ if persist => observed_error( | ||
| 787 | + "VECTOR_PERSIST_FAILED", | ||
| 788 | + "memory vector persistence failed", | ||
| 789 | + true, | ||
| 790 | + None, | ||
| 791 | + operation, | ||
| 792 | + "storage", | ||
| 793 | + "memory vector persistence failed", | ||
| 794 | + ), | ||
| 795 | + _ => observed_error( | ||
| 796 | + "STORAGE_FAILED", | ||
| 797 | + "memory storage failed", | ||
| 798 | + true, | ||
| 799 | + None, | ||
| 800 | + operation, | ||
| 801 | + "storage", | ||
| 802 | + "memory storage operation failed", | ||
| 803 | + ), | ||
| 804 | + } | ||
| 805 | +} | ||
| 806 | + | ||
| 807 | + | ||
| 808 | +fn observed_error( | ||
| 809 | + code: &'static str, | ||
| 810 | + message: &'static str, | ||
| 811 | + retriable: bool, | ||
| 812 | + stage: Option<PipelineStage>, | ||
| 813 | + error_site: &'static str, | ||
| 814 | + source_error_kind: &'static str, | ||
| 815 | + source_error_message: &'static str, | ||
| 816 | +) -> ServiceError { | ||
| 817 | + let caller = std::panic::Location::caller(); | ||
| 818 | + ServiceError::Observed { | ||
| 819 | + code, | ||
| 820 | + message, | ||
| 821 | + retriable, | ||
| 822 | + stage, | ||
| 823 | + error_site, | ||
| 824 | + error_origin_file: caller.file(), | ||
| 825 | + error_origin_line: caller.line(), | ||
| 826 | + source_error_kind, | ||
| 827 | + source_error_message, | ||
| 828 | + } | ||
| 829 | +} | ||
| 830 | + | ||
| 831 | +fn sqlite_error_is_busy(error: &rusqlite::Error) -> bool { | ||
| 832 | + matches!( | ||
| 833 | + error, | ||
| 834 | + rusqlite::Error::SqliteFailure( | ||
| 835 | + rusqlite::ffi::Error { | ||
| 836 | + code: rusqlite::ErrorCode::DatabaseBusy | rusqlite::ErrorCode::DatabaseLocked, | ||
| 837 | + .. | ||
| 838 | + }, | ||
| 839 | + _ | ||
| 840 | + ) | ||
| 841 | + ) | ||
| 842 | +} | ||
| 843 | + | ||
| 844 | +fn sqlite_error_is_read_only(error: &rusqlite::Error) -> bool { | ||
| 845 | + matches!( | ||
| 846 | + error, | ||
| 847 | + rusqlite::Error::SqliteFailure( | ||
| 848 | + rusqlite::ffi::Error { | ||
| 849 | + code: rusqlite::ErrorCode::ReadOnly, | ||
| 850 | + .. | ||
| 851 | + }, | ||
| 852 | + _ | ||
| 853 | + ) | ||
| 854 | + ) | ||
| 855 | +} | ||
| 856 | + | ||
| 565 | fn cached_response(results: Vec<Value>) -> Result<IngestResponse, ServiceError> { | 857 | fn cached_response(results: Vec<Value>) -> Result<IngestResponse, ServiceError> { |
| 566 | let mut responses = results | 858 | let mut responses = results |
| 567 | .into_iter() | 859 | .into_iter() |
| @@ -605,23 +897,23 @@ fn stored_memory_requests( | |||
| 605 | let memories = prepared | 897 | let memories = prepared |
| 606 | .get("memories") | 898 | .get("memories") |
| 607 | .and_then(Value::as_array) | 899 | .and_then(Value::as_array) |
| 608 | - .ok_or(ServiceError::Pipeline)?; | 900 | + .ok_or_else(|| ServiceError::pipeline(Some(PipelineStage::Aggregate)))?; |
| 609 | memories | 901 | memories |
| 610 | .iter() | 902 | .iter() |
| 611 | .map(|memory| { | 903 | .map(|memory| { |
| 612 | let id = memory | 904 | let id = memory |
| 613 | .get("id") | 905 | .get("id") |
| 614 | .and_then(Value::as_str) | 906 | .and_then(Value::as_str) |
| 615 | - .ok_or(ServiceError::Pipeline)?; | 907 | + .ok_or_else(|| ServiceError::pipeline(Some(PipelineStage::Aggregate)))?; |
| 616 | let text = memory | 908 | let text = memory |
| 617 | .get("text") | 909 | .get("text") |
| 618 | .and_then(Value::as_str) | 910 | .and_then(Value::as_str) |
| 619 | - .ok_or(ServiceError::Pipeline)?; | 911 | + .ok_or_else(|| ServiceError::pipeline(Some(PipelineStage::Aggregate)))?; |
| 620 | let mut metadata = memory | 912 | let mut metadata = memory |
| 621 | .get("metadata") | 913 | .get("metadata") |
| 622 | .and_then(Value::as_object) | 914 | .and_then(Value::as_object) |
| 623 | .cloned() | 915 | .cloned() |
| 624 | - .ok_or(ServiceError::Pipeline)?; | 916 | + .ok_or_else(|| ServiceError::pipeline(Some(PipelineStage::Aggregate)))?; |
| 625 | metadata.insert("scope_id".to_string(), json!(principal.scope_id())); | 917 | metadata.insert("scope_id".to_string(), json!(principal.scope_id())); |
| 626 | metadata.insert("source_agent_id".to_string(), json!(principal.agent_id)); | 918 | metadata.insert("source_agent_id".to_string(), json!(principal.agent_id)); |
| 627 | metadata.insert("pipeline_run_id".to_string(), json!(pipeline_run_id)); | 919 | metadata.insert("pipeline_run_id".to_string(), json!(pipeline_run_id)); |
| @@ -641,12 +933,15 @@ fn build_graph_add_requests( | |||
| 641 | requests | 933 | requests |
| 642 | .iter() | 934 | .iter() |
| 643 | .map(|request| { | 935 | .map(|request| { |
| 644 | - let id = request.id.as_deref().ok_or(ServiceError::Pipeline)?; | 936 | + let id = request |
| 937 | + .id | ||
| 938 | + .as_deref() | ||
| 939 | + .ok_or_else(|| ServiceError::pipeline(Some(PipelineStage::Aggregate)))?; | ||
| 645 | let mut metadata = request | 940 | let mut metadata = request |
| 646 | .metadata | 941 | .metadata |
| 647 | .as_object() | 942 | .as_object() |
| 648 | .cloned() | 943 | .cloned() |
| 649 | - .ok_or(ServiceError::Pipeline)?; | 944 | + .ok_or_else(|| ServiceError::pipeline(Some(PipelineStage::Aggregate)))?; |
| 650 | metadata.remove("pipeline_run_id"); | 945 | metadata.remove("pipeline_run_id"); |
| 651 | metadata.remove("source_agent_id"); | 946 | metadata.remove("source_agent_id"); |
| 652 | if !metadata.contains_key("graph_source_entity") { | 947 | if !metadata.contains_key("graph_source_entity") { |
| @@ -723,8 +1018,8 @@ async fn build_graph_memories( | |||
| 723 | } | 1018 | } |
| 724 | while let Some(result) = tasks.join_next().await { | 1019 | while let Some(result) = tasks.join_next().await { |
| 725 | result | 1020 | result |
| 726 | - .map_err(|_| ServiceError::Pipeline)? | 1021 | + .map_err(|_| ServiceError::pipeline(None))? |
| 727 | - .map_err(|_| ServiceError::Pipeline)?; | 1022 | + .map_err(|_| ServiceError::pipeline(None))?; |
| 728 | if let Some(request) = requests.next() { | 1023 | if let Some(request) = requests.next() { |
| 729 | spawn_graph_build(&mut tasks, runtime.pipeline.clone(), request); | 1024 | spawn_graph_build(&mut tasks, runtime.pipeline.clone(), request); |
| 730 | } | 1025 | } |
| @@ -887,10 +1182,58 @@ mod tests { | |||
| 887 | use serde_json::json; | 1182 | use serde_json::json; |
| 888 | 1183 | ||
| 889 | use super::{ | 1184 | use super::{ |
| 890 | - bounded_candidate_limit, build_graph_add_requests, build_prepared_input, search_result, | 1185 | + bounded_candidate_limit, build_graph_add_requests, build_prepared_input, map_persist_error, |
| 891 | - stable_source_id, MemoryService, | 1186 | + map_pipeline_error, map_search_error, search_result, stable_source_id, MemoryService, |
| 892 | }; | 1187 | }; |
| 893 | use crate::{IdempotencyRepository, IngestMessage, IngestRequest, Principal}; | 1188 | use crate::{IdempotencyRepository, IngestMessage, IngestRequest, Principal}; |
| 1189 | + use memory_pipeline::error::{PipelineError, PipelineStage}; | ||
| 1190 | + | ||
| 1191 | + | ||
| 1192 | + fn provider_failures_keep_pipeline_stage_and_rerank_classification() { | ||
| 1193 | + for stage in [PipelineStage::Extract, PipelineStage::Ground] { | ||
| 1194 | + let error = PipelineError::Protocol("provider failed".to_string()).at_stage(stage); | ||
| 1195 | + let mapped = map_pipeline_error(error); | ||
| 1196 | + assert_eq!(mapped.code(), "PIPELINE_FAILED"); | ||
| 1197 | + assert_eq!(mapped.stage(), Some(stage.as_str())); | ||
| 1198 | + assert!(mapped.retriable()); | ||
| 1199 | + } | ||
| 1200 | + | ||
| 1201 | + let rerank = map_search_error(memory_core::MemoryError::Rerank { | ||
| 1202 | + message: "reranker unavailable".to_string(), | ||
| 1203 | + }); | ||
| 1204 | + assert_eq!(rerank.code(), "RERANK_FAILED"); | ||
| 1205 | + assert!(rerank.retriable()); | ||
| 1206 | + assert_eq!(rerank.source_error_kind(), Some("rerank")); | ||
| 1207 | + assert!(rerank.error_origin().is_some()); | ||
| 1208 | + } | ||
| 1209 | + | ||
| 1210 | + | ||
| 1211 | + fn storage_failures_map_to_specific_public_codes() { | ||
| 1212 | + let embedding = map_persist_error(memory_core::MemoryError::Embedding { | ||
| 1213 | + message: "PRIVATE_PROVIDER_BODY".to_string(), | ||
| 1214 | + }); | ||
| 1215 | + assert_eq!(embedding.code(), "EMBEDDING_FAILED"); | ||
| 1216 | + assert_eq!(embedding.source_error_kind(), Some("embedding")); | ||
| 1217 | + assert!(!embedding.to_string().contains("PRIVATE_PROVIDER_BODY")); | ||
| 1218 | + | ||
| 1219 | + let vector = map_persist_error(memory_core::MemoryError::StoreBackend { | ||
| 1220 | + message: "PRIVATE_SQL_DETAIL".to_string(), | ||
| 1221 | + }); | ||
| 1222 | + assert_eq!(vector.code(), "VECTOR_PERSIST_FAILED"); | ||
| 1223 | + assert!(!vector.to_string().contains("PRIVATE_SQL_DETAIL")); | ||
| 1224 | + | ||
| 1225 | + for (sqlite_code, expected, retriable) in [ | ||
| 1226 | + (rusqlite::ffi::SQLITE_BUSY, "SQLITE_BUSY", true), | ||
| 1227 | + (rusqlite::ffi::SQLITE_READONLY, "SQLITE_READONLY", false), | ||
| 1228 | + ] { | ||
| 1229 | + let mapped = map_persist_error(memory_core::MemoryError::Sqlite( | ||
| 1230 | + rusqlite::Error::SqliteFailure(rusqlite::ffi::Error::new(sqlite_code), None), | ||
| 1231 | + )); | ||
| 1232 | + assert_eq!(mapped.code(), expected); | ||
| 1233 | + assert_eq!(mapped.retriable(), retriable); | ||
| 1234 | + assert!(mapped.error_origin().is_some()); | ||
| 1235 | + } | ||
| 1236 | + } | ||
| 894 | 1237 | ||
| 895 | fn principal(user: &str, agent: &str) -> Principal { | 1238 | fn principal(user: &str, agent: &str) -> Principal { |
| 896 | Principal { | 1239 | Principal { |
| @@ -6,6 +6,9 @@ use schemars::JsonSchema; | |||
| 6 | use serde::{Deserialize, Serialize}; | 6 | use serde::{Deserialize, Serialize}; |
| 7 | 7 | ||
| 8 | pub const MAX_INGEST_MESSAGES: usize = 100; | 8 | pub const MAX_INGEST_MESSAGES: usize = 100; |
| 9 | +pub const MAX_CONVERSATION_ID_CHARS: usize = 255; | ||
| 10 | +pub const MAX_MESSAGE_ID_CHARS: usize = 255; | ||
| 11 | +pub const MAX_SPEAKER_CHARS: usize = 255; | ||
| 9 | pub const MAX_MESSAGE_TEXT_CHARS: usize = 32_000; | 12 | pub const MAX_MESSAGE_TEXT_CHARS: usize = 32_000; |
| 10 | pub const MAX_QUERY_CHARS: usize = 32_000; | 13 | pub const MAX_QUERY_CHARS: usize = 32_000; |
| 11 | pub const MAX_TOP_K: usize = 100; | 14 | pub const MAX_TOP_K: usize = 100; |
| @@ -16,6 +19,7 @@ pub const MAX_CASE_FILE_NAME_CHARS: usize = 255; | |||
| 16 | pub const MAX_CASE_DOCUMENT_NAME_CHARS: usize = 512; | 19 | pub const MAX_CASE_DOCUMENT_NAME_CHARS: usize = 512; |
| 17 | pub const MAX_CASE_DIAGNOSIS_CHARS: usize = 8_000; | 20 | pub const MAX_CASE_DIAGNOSIS_CHARS: usize = 8_000; |
| 18 | pub const MAX_CASE_DELETION_REASON_CHARS: usize = 2_000; | 21 | pub const MAX_CASE_DELETION_REASON_CHARS: usize = 2_000; |
| 22 | +pub const MAX_CASE_LIBRARY_CHARS: usize = 255; | ||
| 19 | 23 | ||
| 20 | const ALLOWED_ROLES: [&str; 4] = ["user", "assistant", "system", "tool"]; | 24 | const ALLOWED_ROLES: [&str; 4] = ["user", "assistant", "system", "tool"]; |
| 21 | const ALLOWED_MEMORY_TYPES: [&str; 7] = [ | 25 | const ALLOWED_MEMORY_TYPES: [&str; 7] = [ |
| @@ -31,13 +35,19 @@ const ALLOWED_MEMORY_TYPES: [&str; 7] = [ | |||
| 31 | 35 | ||
| 32 | 36 | ||
| 33 | pub struct IngestRequest { | 37 | pub struct IngestRequest { |
| 38 | + | ||
| 34 | pub conversation_id: String, | 39 | pub conversation_id: String, |
| 40 | + | ||
| 35 | pub messages: Vec<IngestMessage>, | 41 | pub messages: Vec<IngestMessage>, |
| 36 | } | 42 | } |
| 37 | 43 | ||
| 38 | impl IngestRequest { | 44 | impl IngestRequest { |
| 39 | pub fn validate(&self) -> Result<()> { | 45 | pub fn validate(&self) -> Result<()> { |
| 40 | - validate_id("conversation_id", &self.conversation_id)?; | 46 | + validate_bounded_id( |
| 47 | + "conversation_id", | ||
| 48 | + &self.conversation_id, | ||
| 49 | + MAX_CONVERSATION_ID_CHARS, | ||
| 50 | + )?; | ||
| 41 | if self.messages.is_empty() { | 51 | if self.messages.is_empty() { |
| 42 | bail!("messages must not be empty"); | 52 | bail!("messages must not be empty"); |
| 43 | } | 53 | } |
| @@ -47,7 +57,7 @@ impl IngestRequest { | |||
| 47 | 57 | ||
| 48 | let mut message_ids = HashSet::with_capacity(self.messages.len()); | 58 | let mut message_ids = HashSet::with_capacity(self.messages.len()); |
| 49 | for message in &self.messages { | 59 | for message in &self.messages { |
| 50 | - validate_id("message id", &message.id)?; | 60 | + validate_bounded_id("message id", &message.id, MAX_MESSAGE_ID_CHARS)?; |
| 51 | if !message_ids.insert(message.id.as_str()) { | 61 | if !message_ids.insert(message.id.as_str()) { |
| 52 | bail!("message IDs must be unique"); | 62 | bail!("message IDs must be unique"); |
| 53 | } | 63 | } |
| @@ -60,6 +70,9 @@ impl IngestRequest { | |||
| 60 | if !ALLOWED_ROLES.contains(&message.role.as_str()) { | 70 | if !ALLOWED_ROLES.contains(&message.role.as_str()) { |
| 61 | bail!("message role is not allowed"); | 71 | bail!("message role is not allowed"); |
| 62 | } | 72 | } |
| 73 | + if let Some(speaker) = &message.speaker { | ||
| 74 | + validate_bounded_id("message speaker", speaker, MAX_SPEAKER_CHARS)?; | ||
| 75 | + } | ||
| 63 | if let Some(timestamp) = &message.timestamp { | 76 | if let Some(timestamp) = &message.timestamp { |
| 64 | validate_rfc3339("message timestamp", timestamp)?; | 77 | validate_rfc3339("message timestamp", timestamp)?; |
| 65 | } | 78 | } |
| @@ -72,9 +85,12 @@ impl IngestRequest { | |||
| 72 | 85 | ||
| 73 | 86 | ||
| 74 | pub struct IngestMessage { | 87 | pub struct IngestMessage { |
| 88 | + | ||
| 75 | pub id: String, | 89 | pub id: String, |
| 76 | pub role: String, | 90 | pub role: String, |
| 91 | + | ||
| 77 | pub speaker: Option<String>, | 92 | pub speaker: Option<String>, |
| 93 | + | ||
| 78 | pub text: String, | 94 | pub text: String, |
| 79 | pub timestamp: Option<String>, | 95 | pub timestamp: Option<String>, |
| 80 | 96 | ||
| @@ -84,6 +100,7 @@ pub struct IngestMessage { | |||
| 84 | 100 | ||
| 85 | 101 | ||
| 86 | pub struct SearchRequest { | 102 | pub struct SearchRequest { |
| 103 | + | ||
| 87 | pub query: String, | 104 | pub query: String, |
| 88 | 105 | ||
| 89 | pub top_k: usize, | 106 | pub top_k: usize, |
| @@ -125,8 +142,10 @@ impl SearchRequest { | |||
| 125 | 142 | ||
| 126 | 143 | ||
| 127 | pub struct CaseSearchRequest { | 144 | pub struct CaseSearchRequest { |
| 145 | + | ||
| 128 | pub query: String, | 146 | pub query: String, |
| 129 | 147 | ||
| 148 | + | ||
| 130 | pub library: Option<String>, | 149 | pub library: Option<String>, |
| 131 | 150 | ||
| 132 | pub top_k: usize, | 151 | pub top_k: usize, |
| @@ -144,7 +163,7 @@ impl CaseSearchRequest { | |||
| 144 | bail!("top_k must be between 1 and {MAX_CASE_TOP_K}"); | 163 | bail!("top_k must be between 1 and {MAX_CASE_TOP_K}"); |
| 145 | } | 164 | } |
| 146 | if let Some(library) = &self.library { | 165 | if let Some(library) = &self.library { |
| 147 | - validate_id("library", library)?; | 166 | + validate_bounded_id("library", library, MAX_CASE_LIBRARY_CHARS)?; |
| 148 | } | 167 | } |
| 149 | Ok(()) | 168 | Ok(()) |
| 150 | } | 169 | } |
| @@ -346,6 +365,14 @@ fn validate_id(field: &str, value: &str) -> Result<()> { | |||
| 346 | Ok(()) | 365 | Ok(()) |
| 347 | } | 366 | } |
| 348 | 367 | ||
| 368 | +fn validate_bounded_id(field: &str, value: &str, max_chars: usize) -> Result<()> { | ||
| 369 | + validate_id(field, value)?; | ||
| 370 | + if value.chars().count() > max_chars { | ||
| 371 | + bail!("{field} must contain at most {max_chars} characters"); | ||
| 372 | + } | ||
| 373 | + Ok(()) | ||
| 374 | +} | ||
| 375 | + | ||
| 349 | fn validate_rfc3339(field: &str, value: &str) -> Result<()> { | 376 | fn validate_rfc3339(field: &str, value: &str) -> Result<()> { |
| 350 | if DateTime::parse_from_rfc3339(value).is_err() { | 377 | if DateTime::parse_from_rfc3339(value).is_err() { |
| 351 | bail!("{field} must be an RFC3339 timestamp"); | 378 | bail!("{field} must be an RFC3339 timestamp"); |
| @@ -5,7 +5,8 @@ use memory_mcp::{ | |||
| 5 | AuthConfig, CaseDocumentDeleteRequest, CaseDocumentUpdateRequest, CaseDocumentUploadRequest, | 5 | AuthConfig, CaseDocumentDeleteRequest, CaseDocumentUpdateRequest, CaseDocumentUploadRequest, |
| 6 | CaseLibraryConfig, CaseMutationConfirmationRequest, CaseSearchRequest, CaseServiceConfig, | 6 | CaseLibraryConfig, CaseMutationConfirmationRequest, CaseSearchRequest, CaseServiceConfig, |
| 7 | IngestMessage, IngestRequest, Principal, SearchRequest, ServerConfig, TokenAuthenticator, | 7 | IngestMessage, IngestRequest, Principal, SearchRequest, ServerConfig, TokenAuthenticator, |
| 8 | - TokenConfig, | 8 | + TokenConfig, MAX_CASE_LIBRARY_CHARS, MAX_CONVERSATION_ID_CHARS, MAX_MESSAGE_ID_CHARS, |
| 9 | + MAX_SPEAKER_CHARS, | ||
| 9 | }; | 10 | }; |
| 10 | use schemars::schema_for; | 11 | use schemars::schema_for; |
| 11 | use serde_json::json; | 12 | use serde_json::json; |
| @@ -130,6 +131,33 @@ fn ingest_rejects_empty_or_noncanonical_ids() { | |||
| 130 | } | 131 | } |
| 131 | } | 132 | } |
| 132 | 133 | ||
| 134 | + | ||
| 135 | +fn ingest_validates_identifier_and_speaker_character_limits() { | ||
| 136 | + let mut exact_limit = valid_ingest(); | ||
| 137 | + exact_limit.conversation_id = "界".repeat(MAX_CONVERSATION_ID_CHARS); | ||
| 138 | + exact_limit.messages[0].id = "信".repeat(MAX_MESSAGE_ID_CHARS); | ||
| 139 | + exact_limit.messages[0].speaker = Some("人".repeat(MAX_SPEAKER_CHARS)); | ||
| 140 | + assert!(exact_limit.validate().is_ok()); | ||
| 141 | + | ||
| 142 | + let mut conversation_too_long = valid_ingest(); | ||
| 143 | + conversation_too_long.conversation_id = "界".repeat(MAX_CONVERSATION_ID_CHARS + 1); | ||
| 144 | + assert!(conversation_too_long.validate().is_err()); | ||
| 145 | + | ||
| 146 | + let mut message_id_too_long = valid_ingest(); | ||
| 147 | + message_id_too_long.messages[0].id = "信".repeat(MAX_MESSAGE_ID_CHARS + 1); | ||
| 148 | + assert!(message_id_too_long.validate().is_err()); | ||
| 149 | + | ||
| 150 | + let mut speaker_too_long = valid_ingest(); | ||
| 151 | + speaker_too_long.messages[0].speaker = Some("人".repeat(MAX_SPEAKER_CHARS + 1)); | ||
| 152 | + assert!(speaker_too_long.validate().is_err()); | ||
| 153 | + | ||
| 154 | + for speaker in ["", " Alice", "Alice "] { | ||
| 155 | + let mut request = valid_ingest(); | ||
| 156 | + request.messages[0].speaker = Some(speaker.to_owned()); | ||
| 157 | + assert!(request.validate().is_err()); | ||
| 158 | + } | ||
| 159 | +} | ||
| 160 | + | ||
| 133 | 161 | ||
| 134 | fn ingest_rejects_an_empty_message_list_or_duplicate_ids() { | 162 | fn ingest_rejects_an_empty_message_list_or_duplicate_ids() { |
| 135 | let mut empty = valid_ingest(); | 163 | let mut empty = valid_ingest(); |
| @@ -145,14 +173,21 @@ fn ingest_rejects_an_empty_message_list_or_duplicate_ids() { | |||
| 145 | 173 | ||
| 146 | 174 | ||
| 147 | fn ingest_rejects_too_many_messages() { | 175 | fn ingest_rejects_too_many_messages() { |
| 148 | - let request = IngestRequest { | 176 | + let exact_limit = IngestRequest { |
| 177 | + conversation_id: "conversation-1".to_owned(), | ||
| 178 | + messages: (0..100) | ||
| 179 | + .map(|index| valid_message(&format!("message-{index}"))) | ||
| 180 | + .collect(), | ||
| 181 | + }; | ||
| 182 | + let over_limit = IngestRequest { | ||
| 149 | conversation_id: "conversation-1".to_owned(), | 183 | conversation_id: "conversation-1".to_owned(), |
| 150 | messages: (0..101) | 184 | messages: (0..101) |
| 151 | .map(|index| valid_message(&format!("message-{index}"))) | 185 | .map(|index| valid_message(&format!("message-{index}"))) |
| 152 | .collect(), | 186 | .collect(), |
| 153 | }; | 187 | }; |
| 154 | 188 | ||
| 155 | - assert!(request.validate().is_err()); | 189 | + assert!(exact_limit.validate().is_ok()); |
| 190 | + assert!(over_limit.validate().is_err()); | ||
| 156 | } | 191 | } |
| 157 | 192 | ||
| 158 | 193 | ||
| @@ -344,6 +379,48 @@ fn case_document_mutations_accept_safe_text_files_and_reject_unsafe_content() { | |||
| 344 | assert!(unconfirmed.validate().is_err()); | 379 | assert!(unconfirmed.validate().is_err()); |
| 345 | } | 380 | } |
| 346 | 381 | ||
| 382 | + | ||
| 383 | +fn case_search_validates_library_character_limit() { | ||
| 384 | + let mut exact_limit = valid_case_search(); | ||
| 385 | + exact_limit.library = Some("库".repeat(MAX_CASE_LIBRARY_CHARS)); | ||
| 386 | + assert!(exact_limit.validate().is_ok()); | ||
| 387 | + | ||
| 388 | + let mut over_limit = valid_case_search(); | ||
| 389 | + over_limit.library = Some("库".repeat(MAX_CASE_LIBRARY_CHARS + 1)); | ||
| 390 | + assert!(over_limit.validate().is_err()); | ||
| 391 | +} | ||
| 392 | + | ||
| 393 | + | ||
| 394 | +fn tool_schemas_publish_string_and_collection_limits() { | ||
| 395 | + let ingest_schema = serde_json::to_value(schema_for!(IngestRequest)).unwrap(); | ||
| 396 | + assert_eq!( | ||
| 397 | + ingest_schema["properties"]["conversation_id"]["maxLength"], | ||
| 398 | + MAX_CONVERSATION_ID_CHARS | ||
| 399 | + ); | ||
| 400 | + assert_eq!( | ||
| 401 | + ingest_schema["properties"]["messages"]["maxItems"], | ||
| 402 | + memory_mcp::MAX_INGEST_MESSAGES | ||
| 403 | + ); | ||
| 404 | + assert_eq!( | ||
| 405 | + ingest_schema["$defs"]["IngestMessage"]["properties"]["id"]["maxLength"], | ||
| 406 | + MAX_MESSAGE_ID_CHARS | ||
| 407 | + ); | ||
| 408 | + assert_eq!( | ||
| 409 | + ingest_schema["$defs"]["IngestMessage"]["properties"]["speaker"]["maxLength"], | ||
| 410 | + MAX_SPEAKER_CHARS | ||
| 411 | + ); | ||
| 412 | + assert_eq!( | ||
| 413 | + ingest_schema["$defs"]["IngestMessage"]["properties"]["text"]["maxLength"], | ||
| 414 | + memory_mcp::MAX_MESSAGE_TEXT_CHARS | ||
| 415 | + ); | ||
| 416 | + | ||
| 417 | + let case_schema = serde_json::to_value(schema_for!(CaseSearchRequest)).unwrap(); | ||
| 418 | + assert_eq!( | ||
| 419 | + case_schema["properties"]["library"]["maxLength"], | ||
| 420 | + MAX_CASE_LIBRARY_CHARS | ||
| 421 | + ); | ||
| 422 | +} | ||
| 423 | + | ||
| 347 | 424 | ||
| 348 | fn case_service_config_requires_unique_names_private_dataset_mapping_and_default() { | 425 | fn case_service_config_requires_unique_names_private_dataset_mapping_and_default() { |
| 349 | let valid = CaseServiceConfig { | 426 | let valid = CaseServiceConfig { |
| @@ -12,8 +12,9 @@ use memory_core::{HashEmbedding, MemoryManager, SqliteMemoryStore}; | |||
| 12 | use memory_mcp::{ | 12 | use memory_mcp::{ |
| 13 | create_http_router, AuthConfig, CaseLibraryConfig, DynCaseSearchProvider, | 13 | create_http_router, AuthConfig, CaseLibraryConfig, DynCaseSearchProvider, |
| 14 | EmbeddedCaseSearchProvider, EmbeddingProviderKind, FeatureFlags, GraphMemoryRetrievalConfig, | 14 | EmbeddedCaseSearchProvider, EmbeddingProviderKind, FeatureFlags, GraphMemoryRetrievalConfig, |
| 15 | - HttpConfig, HttpRuntime, IdempotencyRepository, LimitsConfig, MemoryService, ProvidersConfig, | 15 | + HttpConfig, HttpRuntime, IdempotencyRepository, LimitsConfig, MemoryService, |
| 16 | - ServerConfig, StorageConfig, TokenAuthenticator, TokenConfig, | 16 | + PipelineServiceConfig, ProvidersConfig, ServerConfig, StorageConfig, TokenAuthenticator, |
| 17 | + TokenConfig, | ||
| 17 | }; | 18 | }; |
| 18 | use memory_pipeline::error::Result as PipelineResult; | 19 | use memory_pipeline::error::Result as PipelineResult; |
| 19 | use memory_pipeline::extraction::{ExtractionBatch, MemoryExtractor, ModelUsage, SCHEMA_VERSION}; | 20 | use memory_pipeline::extraction::{ExtractionBatch, MemoryExtractor, ModelUsage, SCHEMA_VERSION}; |
| @@ -671,16 +672,16 @@ async fn idle_session_expiry_closes_the_session_and_releases_its_slot() { | |||
| 671 | initialize_rate_burst: 100, | 672 | initialize_rate_burst: 100, |
| 672 | max_active_sessions_per_principal: 1, | 673 | max_active_sessions_per_principal: 1, |
| 673 | max_active_sessions_global: 1, | 674 | max_active_sessions_global: 1, |
| 674 | - session_idle_timeout_seconds: 1, | 675 | + session_idle_timeout_seconds: 2, |
| 675 | ..LimitsConfig::default() | 676 | ..LimitsConfig::default() |
| 676 | }; | 677 | }; |
| 677 | let fixture = fixture_router_with_permissions(&["memory:read"], limits).await; | 678 | let fixture = fixture_router_with_permissions(&["memory:read"], limits).await; |
| 678 | let (expired_session_id, _) = initialize(&fixture.app).await; | 679 | let (expired_session_id, _) = initialize(&fixture.app).await; |
| 679 | - tokio::time::advance(Duration::from_secs(2)).await; | 680 | + tokio::time::advance(Duration::from_secs(3)).await; |
| 680 | 681 | ||
| 681 | - let _ = initialize(&fixture.app).await; | ||
| 682 | let expired = fixture | 682 | let expired = fixture |
| 683 | .app | 683 | .app |
| 684 | + .clone() | ||
| 684 | .oneshot(session_request( | 685 | .oneshot(session_request( |
| 685 | &expired_session_id, | 686 | &expired_session_id, |
| 686 | json!({"jsonrpc": "2.0", "id": 102, "method": "tools/list", "params": {}}), | 687 | json!({"jsonrpc": "2.0", "id": 102, "method": "tools/list", "params": {}}), |
| @@ -688,6 +689,61 @@ async fn idle_session_expiry_closes_the_session_and_releases_its_slot() { | |||
| 688 | .await | 689 | .await |
| 689 | .unwrap(); | 690 | .unwrap(); |
| 690 | assert_eq!(expired.status(), StatusCode::NOT_FOUND); | 691 | assert_eq!(expired.status(), StatusCode::NOT_FOUND); |
| 692 | + let body = to_bytes(expired.into_body(), 1024).await.unwrap(); | ||
| 693 | + assert_eq!(body.as_ref(), b"session not found"); | ||
| 694 | + | ||
| 695 | + let _ = initialize(&fixture.app).await; | ||
| 696 | +} | ||
| 697 | + | ||
| 698 | + | ||
| 699 | +async fn session_remains_available_before_the_configured_idle_timeout() { | ||
| 700 | + let limits = LimitsConfig { | ||
| 701 | + initialize_requests_per_second: 100, | ||
| 702 | + initialize_rate_burst: 100, | ||
| 703 | + session_idle_timeout_seconds: 30, | ||
| 704 | + ..LimitsConfig::default() | ||
| 705 | + }; | ||
| 706 | + let fixture = fixture_router_with_permissions(&["memory:read"], limits).await; | ||
| 707 | + let (session_id, _) = initialize(&fixture.app).await; | ||
| 708 | + | ||
| 709 | + tokio::time::advance(Duration::from_secs(3)).await; | ||
| 710 | + | ||
| 711 | + let response = fixture | ||
| 712 | + .app | ||
| 713 | + .oneshot(session_request( | ||
| 714 | + &session_id, | ||
| 715 | + json!({"jsonrpc": "2.0", "id": 106, "method": "tools/list", "params": {}}), | ||
| 716 | + )) | ||
| 717 | + .await | ||
| 718 | + .unwrap(); | ||
| 719 | + assert_eq!(response.status(), StatusCode::OK); | ||
| 720 | + let _ = response_json(response).await; | ||
| 721 | +} | ||
| 722 | + | ||
| 723 | + | ||
| 724 | +async fn configured_idle_timeout_is_applied_to_the_rmcp_session_worker() { | ||
| 725 | + let limits = LimitsConfig { | ||
| 726 | + initialize_requests_per_second: 100, | ||
| 727 | + initialize_rate_burst: 100, | ||
| 728 | + session_idle_timeout_seconds: 600, | ||
| 729 | + ..LimitsConfig::default() | ||
| 730 | + }; | ||
| 731 | + let fixture = fixture_router_with_permissions(&["memory:read"], limits).await; | ||
| 732 | + let (session_id, _) = initialize(&fixture.app).await; | ||
| 733 | + | ||
| 734 | + tokio::time::advance(Duration::from_secs(301)).await; | ||
| 735 | + tokio::task::yield_now().await; | ||
| 736 | + | ||
| 737 | + let response = fixture | ||
| 738 | + .app | ||
| 739 | + .oneshot(session_request( | ||
| 740 | + &session_id, | ||
| 741 | + json!({"jsonrpc": "2.0", "id": 103, "method": "tools/list", "params": {}}), | ||
| 742 | + )) | ||
| 743 | + .await | ||
| 744 | + .unwrap(); | ||
| 745 | + assert_eq!(response.status(), StatusCode::OK); | ||
| 746 | + let _ = response_json(response).await; | ||
| 691 | } | 747 | } |
| 692 | 748 | ||
| 693 | 749 | ||
| @@ -716,6 +772,18 @@ async fn authenticated_session_activity_refreshes_the_idle_deadline() { | |||
| 716 | let _ = response_json(active).await; | 772 | let _ = response_json(active).await; |
| 717 | tokio::time::advance(Duration::from_millis(1_500)).await; | 773 | tokio::time::advance(Duration::from_millis(1_500)).await; |
| 718 | 774 | ||
| 775 | + let still_active = fixture | ||
| 776 | + .app | ||
| 777 | + .clone() | ||
| 778 | + .oneshot(session_request( | ||
| 779 | + &session_id, | ||
| 780 | + json!({"jsonrpc": "2.0", "id": 105, "method": "tools/list", "params": {}}), | ||
| 781 | + )) | ||
| 782 | + .await | ||
| 783 | + .unwrap(); | ||
| 784 | + assert_eq!(still_active.status(), StatusCode::OK); | ||
| 785 | + let _ = response_json(still_active).await; | ||
| 786 | + | ||
| 719 | let still_at_cap = fixture | 787 | let still_at_cap = fixture |
| 720 | .app | 788 | .app |
| 721 | .oneshot(initialize_request(Some(&format!("Bearer {TOKEN}")), None)) | 789 | .oneshot(initialize_request(Some(&format!("Bearer {TOKEN}")), None)) |
| @@ -957,6 +1025,40 @@ async fn memory_search_returns_structured_content_and_json_text_fallback() { | |||
| 957 | assert_eq!(fallback, result["structuredContent"]); | 1025 | assert_eq!(fallback, result["structuredContent"]); |
| 958 | } | 1026 | } |
| 959 | 1027 | ||
| 1028 | + | ||
| 1029 | +async fn memory_search_failure_exposes_the_http_request_id() { | ||
| 1030 | + let fixture = fixture_router().await; | ||
| 1031 | + let (session_id, _) = initialize(&fixture.app).await; | ||
| 1032 | + std::fs::remove_file(&fixture.database_path).expect("remove SQLite database"); | ||
| 1033 | + std::fs::create_dir(&fixture.database_path).expect("replace SQLite file with a directory"); | ||
| 1034 | + let called = call_tool( | ||
| 1035 | + &fixture.app, | ||
| 1036 | + &session_id, | ||
| 1037 | + 204, | ||
| 1038 | + "memory_search", | ||
| 1039 | + json!({"query": "window seat", "top_k": 5}), | ||
| 1040 | + ) | ||
| 1041 | + .await; | ||
| 1042 | + assert_eq!(called.status(), StatusCode::OK); | ||
| 1043 | + let request_id = called | ||
| 1044 | + .headers() | ||
| 1045 | + .get("x-request-id") | ||
| 1046 | + .expect("response request id") | ||
| 1047 | + .to_str() | ||
| 1048 | + .unwrap() | ||
| 1049 | + .to_owned(); | ||
| 1050 | + let called = response_json(called).await; | ||
| 1051 | + assert_eq!(called["result"]["isError"], json!(true)); | ||
| 1052 | + assert_eq!( | ||
| 1053 | + called["result"]["structuredContent"]["code"], | ||
| 1054 | + json!("STORAGE_FAILED") | ||
| 1055 | + ); | ||
| 1056 | + assert_eq!( | ||
| 1057 | + called["result"]["structuredContent"]["request_id"], | ||
| 1058 | + json!(request_id) | ||
| 1059 | + ); | ||
| 1060 | +} | ||
| 1061 | + | ||
| 960 | 1062 | ||
| 961 | async fn invalid_tool_input_is_a_tool_execution_error_not_a_protocol_error() { | 1063 | async fn invalid_tool_input_is_a_tool_execution_error_not_a_protocol_error() { |
| 962 | let fixture = fixture_router().await; | 1064 | let fixture = fixture_router().await; |
| @@ -1057,8 +1159,7 @@ async fn missing_case_search_permission_is_rejected_with_http_forbidden() { | |||
| 1057 | 1159 | ||
| 1058 | 1160 | ||
| 1059 | async fn case_document_mutation_tools_require_cases_write_permission() { | 1161 | async fn case_document_mutation_tools_require_cases_write_permission() { |
| 1060 | - let read_only = | 1162 | + let read_only = fixture_router_with_permissions(&["cases:read"], LimitsConfig::default()).await; |
| 1061 | - fixture_router_with_permissions(&["cases:read"], LimitsConfig::default()).await; | ||
| 1062 | let (read_only_session, _) = initialize(&read_only.app).await; | 1163 | let (read_only_session, _) = initialize(&read_only.app).await; |
| 1063 | let denied = call_tool( | 1164 | let denied = call_tool( |
| 1064 | &read_only.app, | 1165 | &read_only.app, |
| @@ -1264,10 +1365,12 @@ async fn mcp_upload_update_and_delete_flow_reaches_the_embedded_case_library() { | |||
| 1264 | ) | 1365 | ) |
| 1265 | .await; | 1366 | .await; |
| 1266 | let new_search = response_json(new_search).await; | 1367 | let new_search = response_json(new_search).await; |
| 1267 | - assert!(new_search["result"]["structuredContent"]["references"][0]["content"] | 1368 | + assert!( |
| 1268 | - .as_str() | 1369 | + new_search["result"]["structuredContent"]["references"][0]["content"] |
| 1269 | - .unwrap() | 1370 | + .as_str() |
| 1270 | - .contains("mcpnewdnsneedle")); | 1371 | + .unwrap() |
| 1372 | + .contains("mcpnewdnsneedle") | ||
| 1373 | + ); | ||
| 1271 | 1374 | ||
| 1272 | let delete_proposal = call_tool( | 1375 | let delete_proposal = call_tool( |
| 1273 | &fixture.app, | 1376 | &fixture.app, |
| @@ -1687,6 +1790,7 @@ fn production_runtime_config_requires_live_components_and_nonzero_limits() { | |||
| 1687 | features: Default::default(), | 1790 | features: Default::default(), |
| 1688 | http: HttpConfig::default(), | 1791 | http: HttpConfig::default(), |
| 1689 | limits: LimitsConfig::default(), | 1792 | limits: LimitsConfig::default(), |
| 1793 | + pipeline: PipelineServiceConfig::default(), | ||
| 1690 | storage: None, | 1794 | storage: None, |
| 1691 | providers: None, | 1795 | providers: None, |
| 1692 | retrieval: Default::default(), | 1796 | retrieval: Default::default(), |
| @@ -1700,6 +1804,7 @@ fn production_runtime_config_requires_live_components_and_nonzero_limits() { | |||
| 1700 | features: Default::default(), | 1804 | features: Default::default(), |
| 1701 | http: HttpConfig::default(), | 1805 | http: HttpConfig::default(), |
| 1702 | limits: LimitsConfig::default(), | 1806 | limits: LimitsConfig::default(), |
| 1807 | + pipeline: PipelineServiceConfig::default(), | ||
| 1703 | storage: Some(StorageConfig { | 1808 | storage: Some(StorageConfig { |
| 1704 | database_path: "memory.sqlite".into(), | 1809 | database_path: "memory.sqlite".into(), |
| 1705 | }), | 1810 | }), |
| @@ -0,0 +1,42 @@ | |||
| 1 | +use std::process::Command; | ||
| 2 | + | ||
| 3 | +fn run_with(name: &str, value: &str) -> std::process::Output { | ||
| 4 | + let directory = tempfile::tempdir().expect("temporary directory"); | ||
| 5 | + Command::new(env!("CARGO_BIN_EXE_ram-a-mem")) | ||
| 6 | + .current_dir(directory.path()) | ||
| 7 | + .env_remove("RAM_A_LOG_FORMAT") | ||
| 8 | + .env_remove("RAM_A_LOG_SOURCE") | ||
| 9 | + .env(name, value) | ||
| 10 | + .output() | ||
| 11 | + .expect("run ram-a-mem") | ||
| 12 | +} | ||
| 13 | + | ||
| 14 | + | ||
| 15 | +fn invalid_log_format_fails_before_service_configuration() { | ||
| 16 | + for invalid in ["", "JSON", " compact", "text"] { | ||
| 17 | + let output = run_with("RAM_A_LOG_FORMAT", invalid); | ||
| 18 | + assert!(!output.status.success()); | ||
| 19 | + let stderr = String::from_utf8(output.stderr).expect("UTF-8 stderr"); | ||
| 20 | + assert!(stderr.contains("invalid RAM_A_LOG_FORMAT"), "{stderr}"); | ||
| 21 | + assert!(stderr.contains("json, compact"), "{stderr}"); | ||
| 22 | + assert!( | ||
| 23 | + !stderr.contains("RAM-A memory config not found"), | ||
| 24 | + "{stderr}" | ||
| 25 | + ); | ||
| 26 | + } | ||
| 27 | +} | ||
| 28 | + | ||
| 29 | + | ||
| 30 | +fn invalid_log_source_fails_before_service_configuration() { | ||
| 31 | + for invalid in ["", "TRUE", "1", " false"] { | ||
| 32 | + let output = run_with("RAM_A_LOG_SOURCE", invalid); | ||
| 33 | + assert!(!output.status.success()); | ||
| 34 | + let stderr = String::from_utf8(output.stderr).expect("UTF-8 stderr"); | ||
| 35 | + assert!(stderr.contains("invalid RAM_A_LOG_SOURCE"), "{stderr}"); | ||
| 36 | + assert!(stderr.contains("true, false"), "{stderr}"); | ||
| 37 | + assert!( | ||
| 38 | + !stderr.contains("RAM-A memory config not found"), | ||
| 39 | + "{stderr}" | ||
| 40 | + ); | ||
| 41 | + } | ||
| 42 | +} | ||
| @@ -88,7 +88,10 @@ impl MemoryExtractor for PreferenceExtractor { | |||
| 88 | } | 88 | } |
| 89 | } | 89 | } |
| 90 | 90 | ||
| 91 | -struct SupportingVerifier; | 91 | +#[derive(Default)] |
| 92 | +struct SupportingVerifier { | ||
| 93 | + calls: AtomicUsize, | ||
| 94 | +} | ||
| 92 | 95 | ||
| 93 | struct PreferenceGraphExtractor; | 96 | struct PreferenceGraphExtractor; |
| 94 | 97 | ||
| @@ -198,6 +201,7 @@ impl GroundingVerifier for SupportingVerifier { | |||
| 198 | memories: &[AtomicMemory], | 201 | memories: &[AtomicMemory], |
| 199 | _messages: &HashMap<String, NormalizedMessage>, | 202 | _messages: &HashMap<String, NormalizedMessage>, |
| 200 | ) -> PipelineResult<GroundingBatch> { | 203 | ) -> PipelineResult<GroundingBatch> { |
| 204 | + self.calls.fetch_add(1, Ordering::SeqCst); | ||
| 201 | Ok(GroundingBatch { | 205 | Ok(GroundingBatch { |
| 202 | window_id: window.id.clone(), | 206 | window_id: window.id.clone(), |
| 203 | results: memories | 207 | results: memories |
| @@ -218,6 +222,7 @@ struct Fixture { | |||
| 218 | _temp: tempfile::TempDir, | 222 | _temp: tempfile::TempDir, |
| 219 | service: MemoryService<PreferenceExtractor, SupportingVerifier>, | 223 | service: MemoryService<PreferenceExtractor, SupportingVerifier>, |
| 220 | extractor: Arc<PreferenceExtractor>, | 224 | extractor: Arc<PreferenceExtractor>, |
| 225 | + verifier: Arc<SupportingVerifier>, | ||
| 221 | database_path: std::path::PathBuf, | 226 | database_path: std::path::PathBuf, |
| 222 | } | 227 | } |
| 223 | 228 | ||
| @@ -233,15 +238,12 @@ async fn fixture_service_with_extractor(extractor: Arc<PreferenceExtractor>) -> | |||
| 233 | Arc::new(HashEmbedding::new(32)), | 238 | Arc::new(HashEmbedding::new(32)), |
| 234 | )); | 239 | )); |
| 235 | let idempotency = IdempotencyRepository::open(&database_path).await.unwrap(); | 240 | let idempotency = IdempotencyRepository::open(&database_path).await.unwrap(); |
| 241 | + let verifier = Arc::new(SupportingVerifier::default()); | ||
| 236 | Fixture { | 242 | Fixture { |
| 237 | _temp: temp, | 243 | _temp: temp, |
| 238 | - service: MemoryService::new( | 244 | + service: MemoryService::new(manager, idempotency, extractor.clone(), verifier.clone()), |
| 239 | - manager, | ||
| 240 | - idempotency, | ||
| 241 | - extractor.clone(), | ||
| 242 | - Arc::new(SupportingVerifier), | ||
| 243 | - ), | ||
| 244 | extractor, | 245 | extractor, |
| 246 | + verifier, | ||
| 245 | database_path, | 247 | database_path, |
| 246 | } | 248 | } |
| 247 | } | 249 | } |
| @@ -274,16 +276,13 @@ async fn fixture_graph_service_with_extractor(graph_extractor: Arc<dyn GraphExtr | |||
| 274 | )); | 276 | )); |
| 275 | let idempotency = IdempotencyRepository::open(&database_path).await.unwrap(); | 277 | let idempotency = IdempotencyRepository::open(&database_path).await.unwrap(); |
| 276 | let extractor = Arc::new(PreferenceExtractor::default()); | 278 | let extractor = Arc::new(PreferenceExtractor::default()); |
| 279 | + let verifier = Arc::new(SupportingVerifier::default()); | ||
| 277 | Fixture { | 280 | Fixture { |
| 278 | _temp: temp, | 281 | _temp: temp, |
| 279 | - service: MemoryService::new( | 282 | + service: MemoryService::new(manager, idempotency, extractor.clone(), verifier.clone()) |
| 280 | - manager, | 283 | + .with_graph_memory(graph_pipeline, 2), |
| 281 | - idempotency, | ||
| 282 | - extractor.clone(), | ||
| 283 | - Arc::new(SupportingVerifier), | ||
| 284 | - ) | ||
| 285 | - .with_graph_memory(graph_pipeline, 2), | ||
| 286 | extractor, | 284 | extractor, |
| 285 | + verifier, | ||
| 287 | database_path, | 286 | database_path, |
| 288 | } | 287 | } |
| 289 | } | 288 | } |
| @@ -436,7 +435,7 @@ async fn graph_ingest_retries_an_incomplete_build_without_duplicating_memory() { | |||
| 436 | .ingest(&principal, preference_ingest()) | 435 | .ingest(&principal, preference_ingest()) |
| 437 | .await | 436 | .await |
| 438 | .unwrap_err(), | 437 | .unwrap_err(), |
| 439 | - ServiceError::Pipeline | 438 | + ServiceError::Pipeline { stage: None } |
| 440 | ); | 439 | ); |
| 441 | let retry = fixture | 440 | let retry = fixture |
| 442 | .service | 441 | .service |
| @@ -473,7 +472,7 @@ async fn graph_ingest_retry_is_stable_when_another_agent_resumes_the_request() { | |||
| 473 | .ingest(&first_agent, preference_ingest()) | 472 | .ingest(&first_agent, preference_ingest()) |
| 474 | .await | 473 | .await |
| 475 | .unwrap_err(), | 474 | .unwrap_err(), |
| 476 | - ServiceError::Pipeline | 475 | + ServiceError::Pipeline { stage: None } |
| 477 | ); | 476 | ); |
| 478 | let retry = fixture | 477 | let retry = fixture |
| 479 | .service | 478 | .service |
| @@ -514,6 +513,35 @@ async fn repeated_message_id_reuses_successful_ingest() { | |||
| 514 | assert_eq!(first.memory_ids, second.memory_ids); | 513 | assert_eq!(first.memory_ids, second.memory_ids); |
| 515 | } | 514 | } |
| 516 | 515 | ||
| 516 | + | ||
| 517 | +async fn context_only_ingest_returns_empty_without_model_calls_or_idempotency_rows() { | ||
| 518 | + let fixture = fixture_service().await; | ||
| 519 | + let mut request = preference_ingest(); | ||
| 520 | + request.messages[0].candidate = false; | ||
| 521 | + | ||
| 522 | + let response = fixture | ||
| 523 | + .service | ||
| 524 | + .ingest(&principal("t", "u", "agent-a"), request) | ||
| 525 | + .await | ||
| 526 | + .unwrap(); | ||
| 527 | + | ||
| 528 | + assert_eq!(response.accepted_count, 0); | ||
| 529 | + assert_eq!(response.rejected_count, 0); | ||
| 530 | + assert_eq!(response.quarantined_count, 0); | ||
| 531 | + assert!(response.memory_ids.is_empty()); | ||
| 532 | + assert!(!response.idempotency_hit); | ||
| 533 | + assert_eq!(fixture.extractor.calls.load(Ordering::SeqCst), 0); | ||
| 534 | + assert_eq!(fixture.verifier.calls.load(Ordering::SeqCst), 0); | ||
| 535 | + | ||
| 536 | + let connection = rusqlite::Connection::open(&fixture.database_path).unwrap(); | ||
| 537 | + let row_count: i64 = connection | ||
| 538 | + .query_row("SELECT COUNT(*) FROM mcp_ingest_idempotency", [], |row| { | ||
| 539 | + row.get(0) | ||
| 540 | + }) | ||
| 541 | + .unwrap(); | ||
| 542 | + assert_eq!(row_count, 0); | ||
| 543 | +} | ||
| 544 | + | ||
| 517 | 545 | ||
| 518 | async fn same_message_key_with_different_content_is_rejected_before_pipeline() { | 546 | async fn same_message_key_with_different_content_is_rejected_before_pipeline() { |
| 519 | let fixture = fixture_service().await; | 547 | let fixture = fixture_service().await; |
| @@ -538,6 +566,85 @@ async fn same_message_key_with_different_content_is_rejected_before_pipeline() { | |||
| 538 | assert!(!rendered.contains("user-secret")); | 566 | assert!(!rendered.contains("user-secret")); |
| 539 | } | 567 | } |
| 540 | 568 | ||
| 569 | + | ||
| 570 | +async fn changing_hashed_message_metadata_returns_conflict_before_pipeline() { | ||
| 571 | + let fixture = fixture_service().await; | ||
| 572 | + let principal = principal("t", "u", "agent-a"); | ||
| 573 | + fixture | ||
| 574 | + .service | ||
| 575 | + .ingest(&principal, preference_ingest()) | ||
| 576 | + .await | ||
| 577 | + .unwrap(); | ||
| 578 | + | ||
| 579 | + let variants = [ | ||
| 580 | + { | ||
| 581 | + let mut request = preference_ingest(); | ||
| 582 | + request.messages[0].role = "assistant".to_string(); | ||
| 583 | + request | ||
| 584 | + }, | ||
| 585 | + { | ||
| 586 | + let mut request = preference_ingest(); | ||
| 587 | + request.messages[0].speaker = Some("Bob".to_string()); | ||
| 588 | + request | ||
| 589 | + }, | ||
| 590 | + { | ||
| 591 | + let mut request = preference_ingest(); | ||
| 592 | + request.messages[0].timestamp = Some("2026-07-22T10:01:00Z".to_string()); | ||
| 593 | + request | ||
| 594 | + }, | ||
| 595 | + ]; | ||
| 596 | + | ||
| 597 | + for request in variants { | ||
| 598 | + let error = fixture | ||
| 599 | + .service | ||
| 600 | + .ingest(&principal, request) | ||
| 601 | + .await | ||
| 602 | + .unwrap_err(); | ||
| 603 | + assert_eq!(error, ServiceError::IdempotencyConflict); | ||
| 604 | + } | ||
| 605 | + assert_eq!(fixture.extractor.calls.load(Ordering::SeqCst), 1); | ||
| 606 | + assert_eq!(fixture.verifier.calls.load(Ordering::SeqCst), 1); | ||
| 607 | +} | ||
| 608 | + | ||
| 609 | + | ||
| 610 | +async fn idempotency_key_is_scoped_by_principal_and_conversation() { | ||
| 611 | + let fixture = fixture_service().await; | ||
| 612 | + let base = preference_ingest(); | ||
| 613 | + fixture | ||
| 614 | + .service | ||
| 615 | + .ingest(&principal("tenant-a", "user-a", "agent-a"), base.clone()) | ||
| 616 | + .await | ||
| 617 | + .unwrap(); | ||
| 618 | + | ||
| 619 | + let mut other_conversation = base.clone(); | ||
| 620 | + other_conversation.conversation_id = "conversation-2".to_string(); | ||
| 621 | + let conversation_response = fixture | ||
| 622 | + .service | ||
| 623 | + .ingest( | ||
| 624 | + &principal("tenant-a", "user-a", "agent-a"), | ||
| 625 | + other_conversation, | ||
| 626 | + ) | ||
| 627 | + .await | ||
| 628 | + .unwrap(); | ||
| 629 | + let scope_response = fixture | ||
| 630 | + .service | ||
| 631 | + .ingest(&principal("tenant-a", "user-b", "agent-a"), base) | ||
| 632 | + .await | ||
| 633 | + .unwrap(); | ||
| 634 | + | ||
| 635 | + assert!(!conversation_response.idempotency_hit); | ||
| 636 | + assert!(!scope_response.idempotency_hit); | ||
| 637 | + assert_eq!(fixture.extractor.calls.load(Ordering::SeqCst), 3); | ||
| 638 | + | ||
| 639 | + let connection = rusqlite::Connection::open(&fixture.database_path).unwrap(); | ||
| 640 | + let row_count: i64 = connection | ||
| 641 | + .query_row("SELECT COUNT(*) FROM mcp_ingest_idempotency", [], |row| { | ||
| 642 | + row.get(0) | ||
| 643 | + }) | ||
| 644 | + .unwrap(); | ||
| 645 | + assert_eq!(row_count, 3); | ||
| 646 | +} | ||
| 647 | + | ||
| 541 | 648 | ||
| 542 | async fn concurrent_identical_ingests_run_pipeline_once() { | 649 | async fn concurrent_identical_ingests_run_pipeline_once() { |
| 543 | let extractor = Arc::new(PreferenceExtractor { | 650 | let extractor = Arc::new(PreferenceExtractor { |
| @@ -608,8 +715,8 @@ async fn search_filters_type_and_event_time_after_scoped_retrieval() { | |||
| 608 | query: "window trip".to_string(), | 715 | query: "window trip".to_string(), |
| 609 | top_k: 10, | 716 | top_k: 10, |
| 610 | memory_types: vec!["event".to_string()], | 717 | memory_types: vec!["event".to_string()], |
| 611 | - event_time_from: Some("2026-07-31T00:00:00Z".to_string()), | 718 | + event_time_from: Some("2026-08-01T00:00:00Z".to_string()), |
| 612 | - event_time_to: Some("2026-08-02T00:00:00Z".to_string()), | 719 | + event_time_to: Some("2026-08-01T00:00:00Z".to_string()), |
| 613 | }; | 720 | }; |
| 614 | let result = fixture.service.search(&principal, request).await.unwrap(); | 721 | let result = fixture.service.search(&principal, request).await.unwrap(); |
| 615 | assert_eq!(result.memories.len(), 1); | 722 | assert_eq!(result.memories.len(), 1); |
| @@ -21,3 +21,4 @@ tracing.workspace = true | |||
| 21 | [dev-dependencies] | 21 | [dev-dependencies] |
| 22 | pretty_assertions = "1.4" | 22 | pretty_assertions = "1.4" |
| 23 | tempfile = "3.9" | 23 | tempfile = "3.9" |
| 24 | +tracing-subscriber.workspace = true | ||
| @@ -83,7 +83,13 @@ impl OpenAiCompatibleClient { | |||
| 83 | last_error = format!("chat completion body read failed: {error}"); | 83 | last_error = format!("chat completion body read failed: {error}"); |
| 84 | if attempt + 1 < self.max_retries.max(1) { | 84 | if attempt + 1 < self.max_retries.max(1) { |
| 85 | let backoff = Duration::from_secs((1u64 << attempt.min(6)).min(64)); | 85 | let backoff = Duration::from_secs((1u64 << attempt.min(6)).min(64)); |
| 86 | - log_retry(model, attempt, self.max_retries, backoff, "read"); | 86 | + log_retry( |
| 87 | + model, | ||
| 88 | + attempt, | ||
| 89 | + self.max_retries, | ||
| 90 | + backoff, | ||
| 91 | + "response_read", | ||
| 92 | + ); | ||
| 87 | tokio::time::sleep(backoff).await; | 93 | tokio::time::sleep(backoff).await; |
| 88 | } | 94 | } |
| 89 | continue; | 95 | continue; |
| @@ -95,7 +101,13 @@ impl OpenAiCompatibleClient { | |||
| 95 | last_error = format!("chat completion returned invalid JSON: {error}"); | 101 | last_error = format!("chat completion returned invalid JSON: {error}"); |
| 96 | if attempt + 1 < self.max_retries.max(1) { | 102 | if attempt + 1 < self.max_retries.max(1) { |
| 97 | let backoff = Duration::from_secs((1u64 << attempt.min(6)).min(64)); | 103 | let backoff = Duration::from_secs((1u64 << attempt.min(6)).min(64)); |
| 98 | - log_retry(model, attempt, self.max_retries, backoff, "decode"); | 104 | + log_retry( |
| 105 | + model, | ||
| 106 | + attempt, | ||
| 107 | + self.max_retries, | ||
| 108 | + backoff, | ||
| 109 | + "invalid_json", | ||
| 110 | + ); | ||
| 99 | tokio::time::sleep(backoff).await; | 111 | tokio::time::sleep(backoff).await; |
| 100 | } | 112 | } |
| 101 | continue; | 113 | continue; |
| @@ -167,7 +179,8 @@ impl OpenAiCompatibleClient { | |||
| 167 | ); | 179 | ); |
| 168 | Err(PipelineError::Protocol(format!( | 180 | Err(PipelineError::Protocol(format!( |
| 169 | "chat completion failed after retries: {last_error}" | 181 | "chat completion failed after retries: {last_error}" |
| 170 | - ))) | 182 | + )) |
| 183 | + .at_site("memory_pipeline.provider.chat_completion")) | ||
| 171 | } | 184 | } |
| 172 | } | 185 | } |
| 173 | 186 | ||
| @@ -187,16 +200,22 @@ fn log_retry(model: &str, attempt: usize, max_retries: usize, backoff: Duration, | |||
| 187 | 200 | ||
| 188 | fn llm_error_kind(message: &str) -> &'static str { | 201 | fn llm_error_kind(message: &str) -> &'static str { |
| 189 | let lower = message.to_ascii_lowercase(); | 202 | let lower = message.to_ascii_lowercase(); |
| 190 | - if lower.contains("429") { | 203 | + if lower.contains("empty content") { |
| 204 | + "empty_content" | ||
| 205 | + } else if lower.contains("429") { | ||
| 191 | "http_429" | 206 | "http_429" |
| 192 | } else if lower.contains("503") { | 207 | } else if lower.contains("503") { |
| 193 | "http_503" | 208 | "http_503" |
| 194 | } else if lower.contains("invalid json") { | 209 | } else if lower.contains("invalid json") { |
| 195 | - "decode" | 210 | + "invalid_json" |
| 196 | } else if lower.contains("body read") { | 211 | } else if lower.contains("body read") { |
| 197 | - "read" | 212 | + "response_read" |
| 198 | } else if lower.contains("timed out") { | 213 | } else if lower.contains("timed out") { |
| 199 | "timeout" | 214 | "timeout" |
| 215 | + } else if lower.contains("http ") { | ||
| 216 | + "http_status" | ||
| 217 | + } else if lower.contains("connect") { | ||
| 218 | + "connect" | ||
| 200 | } else { | 219 | } else { |
| 201 | "request" | 220 | "request" |
| 202 | } | 221 | } |
| @@ -226,6 +245,21 @@ mod tests { | |||
| 226 | assert!(OpenAiCompatibleClient::new(" ", "http://localhost", 1, 1).is_err()); | 245 | assert!(OpenAiCompatibleClient::new(" ", "http://localhost", 1, 1).is_err()); |
| 227 | } | 246 | } |
| 228 | 247 | ||
| 248 | + | ||
| 249 | + fn model_failures_have_stable_safe_categories() { | ||
| 250 | + for (message, expected) in [ | ||
| 251 | + ("chat completion returned empty content", "empty_content"), | ||
| 252 | + ("chat completion returned invalid JSON", "invalid_json"), | ||
| 253 | + ("chat completion body read failed", "response_read"), | ||
| 254 | + ("request timed out", "timeout"), | ||
| 255 | + ("HTTP 429 Too Many Requests", "http_429"), | ||
| 256 | + ("HTTP 400 Bad Request", "http_status"), | ||
| 257 | + ("connection refused", "connect"), | ||
| 258 | + ] { | ||
| 259 | + assert_eq!(llm_error_kind(message), expected); | ||
| 260 | + } | ||
| 261 | + } | ||
| 262 | + | ||
| 229 | 263 | ||
| 230 | async fn retries_invalid_success_json() { | 264 | async fn retries_invalid_success_json() { |
| 231 | let listener = TcpListener::bind("127.0.0.1:0").unwrap(); | 265 | let listener = TcpListener::bind("127.0.0.1:0").unwrap(); |
| @@ -1,5 +1,49 @@ | |||
| 1 | use thiserror::Error; | 1 | use thiserror::Error; |
| 2 | 2 | ||
| 3 | + | ||
| 4 | +pub struct ErrorOrigin { | ||
| 5 | + pub site: &'static str, | ||
| 6 | + pub file: &'static str, | ||
| 7 | + pub line: u32, | ||
| 8 | +} | ||
| 9 | + | ||
| 10 | +impl ErrorOrigin { | ||
| 11 | + | ||
| 12 | + pub fn capture(site: &'static str) -> Self { | ||
| 13 | + let caller = std::panic::Location::caller(); | ||
| 14 | + Self { | ||
| 15 | + site, | ||
| 16 | + file: caller.file(), | ||
| 17 | + line: caller.line(), | ||
| 18 | + } | ||
| 19 | + } | ||
| 20 | +} | ||
| 21 | + | ||
| 22 | + | ||
| 23 | +pub enum PipelineStage { | ||
| 24 | + Normalize, | ||
| 25 | + Episode, | ||
| 26 | + Window, | ||
| 27 | + Extract, | ||
| 28 | + Validate, | ||
| 29 | + Ground, | ||
| 30 | + Aggregate, | ||
| 31 | +} | ||
| 32 | + | ||
| 33 | +impl PipelineStage { | ||
| 34 | + pub fn as_str(self) -> &'static str { | ||
| 35 | + match self { | ||
| 36 | + Self::Normalize => "normalize", | ||
| 37 | + Self::Episode => "episode", | ||
| 38 | + Self::Window => "window", | ||
| 39 | + Self::Extract => "extract", | ||
| 40 | + Self::Validate => "validate", | ||
| 41 | + Self::Ground => "ground", | ||
| 42 | + Self::Aggregate => "aggregate", | ||
| 43 | + } | ||
| 44 | + } | ||
| 45 | +} | ||
| 46 | + | ||
| 3 | 47 | ||
| 4 | pub enum PipelineError { | 48 | pub enum PipelineError { |
| 5 | 49 | ||
| @@ -10,6 +54,145 @@ pub enum PipelineError { | |||
| 10 | Json( serde_json::Error), | 54 | Json( serde_json::Error), |
| 11 | 55 | ||
| 12 | Io( std::io::Error), | 56 | Io( std::io::Error), |
| 57 | + | ||
| 58 | + Located { | ||
| 59 | + origin: ErrorOrigin, | ||
| 60 | + | ||
| 61 | + source: Box<PipelineError>, | ||
| 62 | + }, | ||
| 63 | + | ||
| 64 | + Stage { | ||
| 65 | + stage: PipelineStage, | ||
| 66 | + origin: ErrorOrigin, | ||
| 67 | + | ||
| 68 | + source: Box<PipelineError>, | ||
| 69 | + }, | ||
| 70 | +} | ||
| 71 | + | ||
| 72 | +impl PipelineError { | ||
| 73 | + | ||
| 74 | + pub fn at_site(self, site: &'static str) -> Self { | ||
| 75 | + match self { | ||
| 76 | + Self::Located { .. } => self, | ||
| 77 | + source => Self::Located { | ||
| 78 | + origin: ErrorOrigin::capture(site), | ||
| 79 | + source: Box::new(source), | ||
| 80 | + }, | ||
| 81 | + } | ||
| 82 | + } | ||
| 83 | + | ||
| 84 | + | ||
| 85 | + pub fn at_stage(self, stage: PipelineStage) -> Self { | ||
| 86 | + match self { | ||
| 87 | + Self::Stage { .. } => self, | ||
| 88 | + source => Self::Stage { | ||
| 89 | + stage, | ||
| 90 | + origin: ErrorOrigin::capture(stage.site()), | ||
| 91 | + source: Box::new(source), | ||
| 92 | + }, | ||
| 93 | + } | ||
| 94 | + } | ||
| 95 | + | ||
| 96 | + pub fn stage(&self) -> Option<PipelineStage> { | ||
| 97 | + match self { | ||
| 98 | + Self::Stage { stage, .. } => Some(*stage), | ||
| 99 | + Self::Located { source, .. } => source.stage(), | ||
| 100 | + _ => None, | ||
| 101 | + } | ||
| 102 | + } | ||
| 103 | + | ||
| 104 | + pub fn origin(&self) -> Option<ErrorOrigin> { | ||
| 105 | + match self { | ||
| 106 | + Self::Stage { origin, source, .. } => source.origin().or(Some(*origin)), | ||
| 107 | + Self::Located { origin, .. } => Some(*origin), | ||
| 108 | + _ => None, | ||
| 109 | + } | ||
| 110 | + } | ||
| 111 | + | ||
| 112 | + pub fn source_error_kind(&self) -> &'static str { | ||
| 113 | + match self.root() { | ||
| 114 | + Self::InvalidInput(_) => "invalid_input", | ||
| 115 | + Self::Json(_) => "invalid_json", | ||
| 116 | + Self::Io(_) => "io", | ||
| 117 | + Self::Protocol(message) if message.contains("empty content") => "empty_content", | ||
| 118 | + Self::Protocol(message) | ||
| 119 | + if message.contains("valid JSON") || message.contains("invalid grounding JSON") => | ||
| 120 | + { | ||
| 121 | + "invalid_json" | ||
| 122 | + } | ||
| 123 | + Self::Protocol(message) | ||
| 124 | + if message.contains("schema_version") | ||
| 125 | + || message.contains("must be a list") | ||
| 126 | + || message.contains("must be an object") | ||
| 127 | + || message.contains("fields are invalid") | ||
| 128 | + || message.contains("duplicate") | ||
| 129 | + || message.contains("omitted memory") => | ||
| 130 | + { | ||
| 131 | + "schema_invalid" | ||
| 132 | + } | ||
| 133 | + Self::Protocol(message) if message.contains("timed out") => "timeout", | ||
| 134 | + Self::Protocol(message) if message.contains("HTTP ") => "http_status", | ||
| 135 | + Self::Protocol(_) => "protocol", | ||
| 136 | + Self::Located { .. } | Self::Stage { .. } => { | ||
| 137 | + unreachable!("root removes diagnostic wrappers") | ||
| 138 | + } | ||
| 139 | + } | ||
| 140 | + } | ||
| 141 | + | ||
| 142 | + pub fn safe_summary(&self) -> &'static str { | ||
| 143 | + match self.source_error_kind() { | ||
| 144 | + "empty_content" => "model returned empty content", | ||
| 145 | + "invalid_json" => "model returned invalid JSON", | ||
| 146 | + "schema_invalid" => "model response did not match the required schema", | ||
| 147 | + "timeout" => "model request timed out", | ||
| 148 | + "http_status" => "model provider returned an HTTP error", | ||
| 149 | + "invalid_input" => "pipeline input validation failed", | ||
| 150 | + "io" => "pipeline I/O operation failed", | ||
| 151 | + _ => "memory pipeline failed", | ||
| 152 | + } | ||
| 153 | + } | ||
| 154 | + | ||
| 155 | + fn root(&self) -> &Self { | ||
| 156 | + match self { | ||
| 157 | + Self::Located { source, .. } | Self::Stage { source, .. } => source.root(), | ||
| 158 | + error => error, | ||
| 159 | + } | ||
| 160 | + } | ||
| 161 | +} | ||
| 162 | + | ||
| 163 | +impl PipelineStage { | ||
| 164 | + pub const fn site(self) -> &'static str { | ||
| 165 | + match self { | ||
| 166 | + Self::Normalize => "memory_pipeline.normalize", | ||
| 167 | + Self::Episode => "memory_pipeline.episode", | ||
| 168 | + Self::Window => "memory_pipeline.window", | ||
| 169 | + Self::Extract => "memory_pipeline.extract", | ||
| 170 | + Self::Validate => "memory_pipeline.validate", | ||
| 171 | + Self::Ground => "memory_pipeline.ground", | ||
| 172 | + Self::Aggregate => "memory_pipeline.aggregate", | ||
| 173 | + } | ||
| 174 | + } | ||
| 13 | } | 175 | } |
| 14 | 176 | ||
| 15 | pub type Result<T> = std::result::Result<T, PipelineError>; | 177 | pub type Result<T> = std::result::Result<T, PipelineError>; |
| 178 | + | ||
| 179 | + | ||
| 180 | +mod tests { | ||
| 181 | + use super::{PipelineError, PipelineStage}; | ||
| 182 | + | ||
| 183 | + | ||
| 184 | + fn stage_wrapping_preserves_the_first_error_origin() { | ||
| 185 | + let error = PipelineError::Protocol("extractor returned empty content".into()) | ||
| 186 | + .at_site("memory_pipeline.extract.parse_response"); | ||
| 187 | + let origin = error.origin().expect("located error origin"); | ||
| 188 | + let wrapped = error.at_stage(PipelineStage::Extract); | ||
| 189 | + | ||
| 190 | + assert_eq!(wrapped.stage(), Some(PipelineStage::Extract)); | ||
| 191 | + assert_eq!(wrapped.origin(), Some(origin)); | ||
| 192 | + assert_eq!( | ||
| 193 | + wrapped.origin().unwrap().site, | ||
| 194 | + "memory_pipeline.extract.parse_response" | ||
| 195 | + ); | ||
| 196 | + assert_eq!(wrapped.source_error_kind(), "empty_content"); | ||
| 197 | + } | ||
| 198 | +} | ||
| @@ -99,8 +99,10 @@ impl MemoryExtractor for LlmMemoryExtractor { | |||
| 99 | json!({"role": "system", "content": "You are a source-faithful long-term-memory extractor. Output only the requested JSON object. Never invent evidence identifiers."}), | 99 | json!({"role": "system", "content": "You are a source-faithful long-term-memory extractor. Output only the requested JSON object. Never invent evidence identifiers."}), |
| 100 | json!({"role": "user", "content": prompt}), | 100 | json!({"role": "user", "content": prompt}), |
| 101 | ], self.max_output_tokens).await?; | 101 | ], self.max_output_tokens).await?; |
| 102 | - let payload = parse_extraction_json(&result.content)?; | 102 | + let payload = parse_extraction_json(&result.content) |
| 103 | - let mut batch = batch_from_payload(&window.id, &payload, &result.content)?; | 103 | + .map_err(|error| error.at_site("memory_pipeline.extract.parse_response"))?; |
| 104 | + let mut batch = batch_from_payload(&window.id, &payload, &result.content) | ||
| 105 | + .map_err(|error| error.at_site("memory_pipeline.extract.validate_response"))?; | ||
| 104 | batch.usage = result.usage; | 106 | batch.usage = result.usage; |
| 105 | Ok(batch) | 107 | Ok(batch) |
| 106 | } | 108 | } |
| @@ -104,10 +104,12 @@ impl GroundingVerifier for LlmGroundingVerifier { | |||
| 104 | serde_json::json!({"role": "user", "content": prompt}), | 104 | serde_json::json!({"role": "user", "content": prompt}), |
| 105 | ], self.max_output_tokens).await?; | 105 | ], self.max_output_tokens).await?; |
| 106 | let payload = parse_extraction_json(&result.content) | 106 | let payload = parse_extraction_json(&result.content) |
| 107 | - .map_err(|error| PipelineError::Protocol(format!("invalid grounding JSON: {error}")))?; | 107 | + .map_err(|error| PipelineError::Protocol(format!("invalid grounding JSON: {error}"))) |
| 108 | + .map_err(|error| error.at_site("memory_pipeline.ground.parse_response"))?; | ||
| 108 | Ok(GroundingBatch { | 109 | Ok(GroundingBatch { |
| 109 | window_id: window.id.clone(), | 110 | window_id: window.id.clone(), |
| 110 | - results: parse_grounding_results(&payload, memories)?, | 111 | + results: parse_grounding_results(&payload, memories) |
| 112 | + .map_err(|error| error.at_site("memory_pipeline.ground.validate_response"))?, | ||
| 111 | usage: result.usage, | 113 | usage: result.usage, |
| 112 | raw_response: result.content, | 114 | raw_response: result.content, |
| 113 | }) | 115 | }) |
| @@ -8,7 +8,7 @@ use serde_json::{json, Map, Value}; | |||
| 8 | use crate::cache::JsonCache; | 8 | use crate::cache::JsonCache; |
| 9 | use crate::canonical::stable_hash; | 9 | use crate::canonical::stable_hash; |
| 10 | use crate::episode::{build_episodes, EpisodeConfig}; | 10 | use crate::episode::{build_episodes, EpisodeConfig}; |
| 11 | -use crate::error::{PipelineError, Result}; | 11 | +use crate::error::{PipelineError, PipelineStage, Result}; |
| 12 | use crate::extraction::{component_identity, ExtractionBatch, MemoryExtractor, SCHEMA_VERSION}; | 12 | use crate::extraction::{component_identity, ExtractionBatch, MemoryExtractor, SCHEMA_VERSION}; |
| 13 | use crate::grounding::{GroundingBatch, GroundingVerifier}; | 13 | use crate::grounding::{GroundingBatch, GroundingVerifier}; |
| 14 | use crate::models::{ | 14 | use crate::models::{ |
| @@ -62,17 +62,26 @@ pub async fn run_memory_pipeline<E: MemoryExtractor + ?Sized, V: GroundingVerifi | |||
| 62 | cache: Option<&JsonCache>, | 62 | cache: Option<&JsonCache>, |
| 63 | ) -> Result<PipelineRun> { | 63 | ) -> Result<PipelineRun> { |
| 64 | if config.validation.max_memory_chars == 0 { | 64 | if config.validation.max_memory_chars == 0 { |
| 65 | - return Err(PipelineError::InvalidInput( | 65 | + tracing::error!( |
| 66 | - "max_memory_chars must be positive".into(), | 66 | + event = "ram_a.memory.ingest.stage.failed", |
| 67 | - )); | 67 | + stage = "validate", |
| 68 | + error_code = "PIPELINE_VALIDATE_FAILED", | ||
| 69 | + retriable = false, | ||
| 70 | + reason = "invalid_config" | ||
| 71 | + ); | ||
| 72 | + return Err( | ||
| 73 | + PipelineError::InvalidInput("max_memory_chars must be positive".into()) | ||
| 74 | + .at_stage(PipelineStage::Validate), | ||
| 75 | + ); | ||
| 68 | } | 76 | } |
| 69 | let mut stage_started = Instant::now(); | 77 | let mut stage_started = Instant::now(); |
| 70 | tracing::info!( | 78 | tracing::info!( |
| 71 | event = "ram_a.memory.ingest.stage.started", | 79 | event = "ram_a.memory.ingest.stage.started", |
| 72 | stage = "normalize" | 80 | stage = "normalize" |
| 73 | ); | 81 | ); |
| 74 | - let (messages, normalization_issues) = | 82 | + let (messages, normalization_issues) = normalize_prepared_memories(prepared) |
| 75 | - normalize_prepared_memories(prepared).inspect_err(|_error| { | 83 | + .map_err(|error| error.at_stage(PipelineStage::Normalize)) |
| 84 | + .inspect_err(|_error| { | ||
| 76 | tracing::error!( | 85 | tracing::error!( |
| 77 | event = "ram_a.memory.ingest.stage.failed", | 86 | event = "ram_a.memory.ingest.stage.failed", |
| 78 | stage = "normalize", | 87 | stage = "normalize", |
| @@ -96,42 +105,46 @@ pub async fn run_memory_pipeline<E: MemoryExtractor + ?Sized, V: GroundingVerifi | |||
| 96 | stage_started = Instant::now(); | 105 | stage_started = Instant::now(); |
| 97 | tracing::info!( | 106 | tracing::info!( |
| 98 | event = "ram_a.memory.ingest.stage.started", | 107 | event = "ram_a.memory.ingest.stage.started", |
| 99 | - stage = "episode_build", | 108 | + stage = "episode", |
| 100 | message_count = messages.len() | 109 | message_count = messages.len() |
| 101 | ); | 110 | ); |
| 102 | - let episodes = build_episodes(&messages, &config.episode).inspect_err(|_error| { | 111 | + let episodes = build_episodes(&messages, &config.episode) |
| 103 | - tracing::error!( | 112 | + .map_err(|error| error.at_stage(PipelineStage::Episode)) |
| 104 | - event = "ram_a.memory.ingest.stage.failed", | 113 | + .inspect_err(|_error| { |
| 105 | - stage = "episode_build", | 114 | + tracing::error!( |
| 106 | - error_code = "PIPELINE_EPISODE_FAILED", | 115 | + event = "ram_a.memory.ingest.stage.failed", |
| 107 | - retriable = false, | 116 | + stage = "episode", |
| 108 | - elapsed_ms = stage_started.elapsed().as_millis() as u64 | 117 | + error_code = "PIPELINE_EPISODE_FAILED", |
| 109 | - ); | 118 | + retriable = false, |
| 110 | - })?; | 119 | + elapsed_ms = stage_started.elapsed().as_millis() as u64 |
| 120 | + ); | ||
| 121 | + })?; | ||
| 111 | tracing::info!( | 122 | tracing::info!( |
| 112 | event = "ram_a.memory.ingest.stage.completed", | 123 | event = "ram_a.memory.ingest.stage.completed", |
| 113 | - stage = "episode_build", | 124 | + stage = "episode", |
| 114 | episode_count = episodes.len(), | 125 | episode_count = episodes.len(), |
| 115 | elapsed_ms = stage_started.elapsed().as_millis() as u64 | 126 | elapsed_ms = stage_started.elapsed().as_millis() as u64 |
| 116 | ); | 127 | ); |
| 117 | stage_started = Instant::now(); | 128 | stage_started = Instant::now(); |
| 118 | tracing::info!( | 129 | tracing::info!( |
| 119 | event = "ram_a.memory.ingest.stage.started", | 130 | event = "ram_a.memory.ingest.stage.started", |
| 120 | - stage = "window_build", | 131 | + stage = "window", |
| 121 | episode_count = episodes.len() | 132 | episode_count = episodes.len() |
| 122 | ); | 133 | ); |
| 123 | - let windows = build_windows(&episodes, &lookup, &config.window).inspect_err(|_error| { | 134 | + let windows = build_windows(&episodes, &lookup, &config.window) |
| 124 | - tracing::error!( | 135 | + .map_err(|error| error.at_stage(PipelineStage::Window)) |
| 125 | - event = "ram_a.memory.ingest.stage.failed", | 136 | + .inspect_err(|_error| { |
| 126 | - stage = "window_build", | 137 | + tracing::error!( |
| 127 | - error_code = "PIPELINE_WINDOW_FAILED", | 138 | + event = "ram_a.memory.ingest.stage.failed", |
| 128 | - retriable = false, | 139 | + stage = "window", |
| 129 | - elapsed_ms = stage_started.elapsed().as_millis() as u64 | 140 | + error_code = "PIPELINE_WINDOW_FAILED", |
| 130 | - ); | 141 | + retriable = false, |
| 131 | - })?; | 142 | + elapsed_ms = stage_started.elapsed().as_millis() as u64 |
| 143 | + ); | ||
| 144 | + })?; | ||
| 132 | tracing::info!( | 145 | tracing::info!( |
| 133 | event = "ram_a.memory.ingest.stage.completed", | 146 | event = "ram_a.memory.ingest.stage.completed", |
| 134 | - stage = "window_build", | 147 | + stage = "window", |
| 135 | window_count = windows.len(), | 148 | window_count = windows.len(), |
| 136 | elapsed_ms = stage_started.elapsed().as_millis() as u64 | 149 | elapsed_ms = stage_started.elapsed().as_millis() as u64 |
| 137 | ); | 150 | ); |
| @@ -171,7 +184,7 @@ pub async fn run_memory_pipeline<E: MemoryExtractor + ?Sized, V: GroundingVerifi | |||
| 171 | total_units = windows.len(), | 184 | total_units = windows.len(), |
| 172 | elapsed_ms = unit_started.elapsed().as_millis() as u64 | 185 | elapsed_ms = unit_started.elapsed().as_millis() as u64 |
| 173 | ); | 186 | ); |
| 174 | - return Err(error); | 187 | + return Err(error.at_stage(PipelineStage::Extract)); |
| 175 | } | 188 | } |
| 176 | Err(error) => { | 189 | Err(error) => { |
| 177 | tracing::warn!( | 190 | tracing::warn!( |
| @@ -211,7 +224,7 @@ pub async fn run_memory_pipeline<E: MemoryExtractor + ?Sized, V: GroundingVerifi | |||
| 211 | let validation_started = Instant::now(); | 224 | let validation_started = Instant::now(); |
| 212 | tracing::info!( | 225 | tracing::info!( |
| 213 | event = "ram_a.memory.ingest.stage.started", | 226 | event = "ram_a.memory.ingest.stage.started", |
| 214 | - stage = "extraction_validate", | 227 | + stage = "validate", |
| 215 | candidate_count = batch.raw_memories.len(), | 228 | candidate_count = batch.raw_memories.len(), |
| 216 | completed_units = window_index, | 229 | completed_units = window_index, |
| 217 | total_units = windows.len() | 230 | total_units = windows.len() |
| @@ -220,7 +233,7 @@ pub async fn run_memory_pipeline<E: MemoryExtractor + ?Sized, V: GroundingVerifi | |||
| 220 | validate_extraction(&batch.raw_memories, window, &lookup, &config.validation); | 233 | validate_extraction(&batch.raw_memories, window, &lookup, &config.validation); |
| 221 | tracing::info!( | 234 | tracing::info!( |
| 222 | event = "ram_a.memory.ingest.stage.completed", | 235 | event = "ram_a.memory.ingest.stage.completed", |
| 223 | - stage = "extraction_validate", | 236 | + stage = "validate", |
| 224 | valid_count = validation.valid.len(), | 237 | valid_count = validation.valid.len(), |
| 225 | rejected_count = validation.rejected.len(), | 238 | rejected_count = validation.rejected.len(), |
| 226 | quarantined_count = validation.quarantined.len(), | 239 | quarantined_count = validation.quarantined.len(), |
| @@ -236,7 +249,7 @@ pub async fn run_memory_pipeline<E: MemoryExtractor + ?Sized, V: GroundingVerifi | |||
| 236 | let verify_started = Instant::now(); | 249 | let verify_started = Instant::now(); |
| 237 | tracing::info!( | 250 | tracing::info!( |
| 238 | event = "ram_a.memory.ingest.stage.started", | 251 | event = "ram_a.memory.ingest.stage.started", |
| 239 | - stage = "verify", | 252 | + stage = "ground", |
| 240 | candidate_count = validation.valid.len(), | 253 | candidate_count = validation.valid.len(), |
| 241 | completed_units = window_index, | 254 | completed_units = window_index, |
| 242 | total_units = windows.len() | 255 | total_units = windows.len() |
| @@ -247,14 +260,14 @@ pub async fn run_memory_pipeline<E: MemoryExtractor + ?Sized, V: GroundingVerifi | |||
| 247 | Err(error) if config.fail_fast => { | 260 | Err(error) if config.fail_fast => { |
| 248 | tracing::error!( | 261 | tracing::error!( |
| 249 | event = "ram_a.memory.ingest.stage.failed", | 262 | event = "ram_a.memory.ingest.stage.failed", |
| 250 | - stage = "verify", | 263 | + stage = "ground", |
| 251 | - error_code = "PIPELINE_VERIFY_FAILED", | 264 | + error_code = "PIPELINE_GROUND_FAILED", |
| 252 | retriable = true, | 265 | retriable = true, |
| 253 | completed_units = window_index, | 266 | completed_units = window_index, |
| 254 | total_units = windows.len(), | 267 | total_units = windows.len(), |
| 255 | elapsed_ms = verify_started.elapsed().as_millis() as u64 | 268 | elapsed_ms = verify_started.elapsed().as_millis() as u64 |
| 256 | ); | 269 | ); |
| 257 | - return Err(error); | 270 | + return Err(error.at_stage(PipelineStage::Ground)); |
| 258 | } | 271 | } |
| 259 | Err(error) => { | 272 | Err(error) => { |
| 260 | tracing::warn!( | 273 | tracing::warn!( |
| @@ -277,7 +290,7 @@ pub async fn run_memory_pipeline<E: MemoryExtractor + ?Sized, V: GroundingVerifi | |||
| 277 | }; | 290 | }; |
| 278 | tracing::info!( | 291 | tracing::info!( |
| 279 | event = "ram_a.memory.ingest.stage.completed", | 292 | event = "ram_a.memory.ingest.stage.completed", |
| 280 | - stage = "verify", | 293 | + stage = "ground", |
| 281 | result_count = grounding.results.len(), | 294 | result_count = grounding.results.len(), |
| 282 | cache_hit = cached, | 295 | cache_hit = cached, |
| 283 | completed_units = window_index + 1, | 296 | completed_units = window_index + 1, |
| @@ -297,9 +310,12 @@ pub async fn run_memory_pipeline<E: MemoryExtractor + ?Sized, V: GroundingVerifi | |||
| 297 | .map(|result| (result.memory_id.clone(), result)) | 310 | .map(|result| (result.memory_id.clone(), result)) |
| 298 | .collect::<HashMap<_, _>>(); | 311 | .collect::<HashMap<_, _>>(); |
| 299 | for memory in validation.valid { | 312 | for memory in validation.valid { |
| 300 | - let result = results.get(&memory.id).ok_or_else(|| { | 313 | + let result = results |
| 301 | - PipelineError::Protocol(format!("verifier omitted memory {}", memory.id)) | 314 | + .get(&memory.id) |
| 302 | - })?; | 315 | + .ok_or_else(|| { |
| 316 | + PipelineError::Protocol(format!("verifier omitted memory {}", memory.id)) | ||
| 317 | + }) | ||
| 318 | + .map_err(|error| error.at_stage(PipelineStage::Ground))?; | ||
| 303 | *grounding_counts.entry(result.status.clone()).or_default() += 1; | 319 | *grounding_counts.entry(result.status.clone()).or_default() += 1; |
| 304 | if result.status == "SUPPORTED" { | 320 | if result.status == "SUPPORTED" { |
| 305 | supported.push(memory) | 321 | supported.push(memory) |
| @@ -329,12 +345,6 @@ pub async fn run_memory_pipeline<E: MemoryExtractor + ?Sized, V: GroundingVerifi | |||
| 329 | ); | 345 | ); |
| 330 | crate::writer::attach_source_observations(&mut supported, &lookup); | 346 | crate::writer::attach_source_observations(&mut supported, &lookup); |
| 331 | let accepted = aggregate_exact_memories(&supported); | 347 | let accepted = aggregate_exact_memories(&supported); |
| 332 | - tracing::info!( | ||
| 333 | - event = "ram_a.memory.ingest.stage.completed", | ||
| 334 | - stage = "aggregate", | ||
| 335 | - accepted_count = accepted.len(), | ||
| 336 | - elapsed_ms = stage_started.elapsed().as_millis() as u64 | ||
| 337 | - ); | ||
| 338 | let (coverage, duplication) = candidate_span_metrics(&messages, &windows); | 348 | let (coverage, duplication) = candidate_span_metrics(&messages, &windows); |
| 339 | let source_memory_counts = source_counts(&lookup, &accepted, true); | 349 | let source_memory_counts = source_counts(&lookup, &accepted, true); |
| 340 | let source_evidence_counts = source_counts(&lookup, &accepted, false); | 350 | let source_evidence_counts = source_counts(&lookup, &accepted, false); |
| @@ -368,7 +378,23 @@ pub async fn run_memory_pipeline<E: MemoryExtractor + ?Sized, V: GroundingVerifi | |||
| 368 | "source_turn_memory_counts": source_memory_counts, | 378 | "source_turn_memory_counts": source_memory_counts, |
| 369 | "source_turn_evidence_ref_counts": source_evidence_counts, | 379 | "source_turn_evidence_ref_counts": source_evidence_counts, |
| 370 | }); | 380 | }); |
| 371 | - let output = make_prepared_output(prepared, &accepted, &run_metadata)?; | 381 | + let output = make_prepared_output(prepared, &accepted, &run_metadata) |
| 382 | + .map_err(|error| error.at_stage(PipelineStage::Aggregate)) | ||
| 383 | + .inspect_err(|_error| { | ||
| 384 | + tracing::error!( | ||
| 385 | + event = "ram_a.memory.ingest.stage.failed", | ||
| 386 | + stage = "aggregate", | ||
| 387 | + error_code = "PIPELINE_AGGREGATE_FAILED", | ||
| 388 | + retriable = false, | ||
| 389 | + elapsed_ms = stage_started.elapsed().as_millis() as u64 | ||
| 390 | + ); | ||
| 391 | + })?; | ||
| 392 | + tracing::info!( | ||
| 393 | + event = "ram_a.memory.ingest.stage.completed", | ||
| 394 | + stage = "aggregate", | ||
| 395 | + accepted_count = accepted.len(), | ||
| 396 | + elapsed_ms = stage_started.elapsed().as_millis() as u64 | ||
| 397 | + ); | ||
| 372 | Ok(PipelineRun { | 398 | Ok(PipelineRun { |
| 373 | prepared: output, | 399 | prepared: output, |
| 374 | normalized_messages: messages, | 400 | normalized_messages: messages, |
| @@ -204,6 +204,81 @@ fn asserted_plan_is_quarantined() { | |||
| 204 | assert_eq!(batch.quarantined[0].code, "suspicious_modality"); | 204 | assert_eq!(batch.quarantined[0].code, "suspicious_modality"); |
| 205 | } | 205 | } |
| 206 | 206 | ||
| 207 | + | ||
| 208 | +fn memory_text_limit_accepts_500_unicode_characters_and_quarantines_501() { | ||
| 209 | + let (window, lookup) = setup(); | ||
| 210 | + let mut exact = raw_memory("planned"); | ||
| 211 | + exact["text"] = json!("界".repeat(500)); | ||
| 212 | + let exact_batch = validate_extraction(&[exact], &window, &lookup, &ValidationConfig::default()); | ||
| 213 | + assert_eq!(exact_batch.valid.len(), 1); | ||
| 214 | + assert!(exact_batch.quarantined.is_empty()); | ||
| 215 | + | ||
| 216 | + let mut over = raw_memory("planned"); | ||
| 217 | + over["text"] = json!("界".repeat(501)); | ||
| 218 | + let over_batch = validate_extraction(&[over], &window, &lookup, &ValidationConfig::default()); | ||
| 219 | + assert!(over_batch.valid.is_empty()); | ||
| 220 | + assert_eq!(over_batch.quarantined[0].code, "memory_text_too_long"); | ||
| 221 | +} | ||
| 222 | + | ||
| 223 | + | ||
| 224 | +fn unknown_memory_type_or_modality_is_rejected() { | ||
| 225 | + let (window, lookup) = setup(); | ||
| 226 | + for (field, value) in [("memory_type", "unknown"), ("modality", "unknown")] { | ||
| 227 | + let mut raw = raw_memory("planned"); | ||
| 228 | + raw[field] = json!(value); | ||
| 229 | + let batch = validate_extraction(&[raw], &window, &lookup, &ValidationConfig::default()); | ||
| 230 | + assert!(batch.valid.is_empty()); | ||
| 231 | + assert_eq!(batch.rejected[0].code, "unknown_enum"); | ||
| 232 | + } | ||
| 233 | +} | ||
| 234 | + | ||
| 235 | + | ||
| 236 | +fn all_documented_memory_types_and_modalities_are_accepted() { | ||
| 237 | + let (window, lookup) = setup(); | ||
| 238 | + for memory_type in [ | ||
| 239 | + "fact", | ||
| 240 | + "preference", | ||
| 241 | + "relationship", | ||
| 242 | + "event", | ||
| 243 | + "state", | ||
| 244 | + "procedure", | ||
| 245 | + "other", | ||
| 246 | + ] { | ||
| 247 | + let mut raw = raw_memory("planned"); | ||
| 248 | + raw["memory_type"] = json!(memory_type); | ||
| 249 | + let batch = validate_extraction(&[raw], &window, &lookup, &ValidationConfig::default()); | ||
| 250 | + assert_eq!(batch.valid.len(), 1, "memory type {memory_type}"); | ||
| 251 | + } | ||
| 252 | + | ||
| 253 | + for modality in [ | ||
| 254 | + "asserted", | ||
| 255 | + "negated", | ||
| 256 | + "possible", | ||
| 257 | + "planned", | ||
| 258 | + "conditional", | ||
| 259 | + "reported", | ||
| 260 | + ] { | ||
| 261 | + let mut raw = raw_memory(modality); | ||
| 262 | + if modality == "asserted" { | ||
| 263 | + raw["evidence"][0]["quote"] = json!("去杭州"); | ||
| 264 | + } | ||
| 265 | + let batch = validate_extraction(&[raw], &window, &lookup, &ValidationConfig::default()); | ||
| 266 | + assert_eq!(batch.valid.len(), 1, "modality {modality}"); | ||
| 267 | + } | ||
| 268 | +} | ||
| 269 | + | ||
| 270 | + | ||
| 271 | +fn evidence_quote_must_match_the_referenced_span_exactly() { | ||
| 272 | + let (window, lookup) = setup(); | ||
| 273 | + let mut raw = raw_memory("planned"); | ||
| 274 | + raw["evidence"][0]["quote"] = json!("计划 去杭州"); | ||
| 275 | + | ||
| 276 | + let batch = validate_extraction(&[raw], &window, &lookup, &ValidationConfig::default()); | ||
| 277 | + | ||
| 278 | + assert!(batch.valid.is_empty()); | ||
| 279 | + assert_eq!(batch.quarantined[0].code, "evidence_quote_not_found"); | ||
| 280 | +} | ||
| 281 | + | ||
| 207 | 282 | ||
| 208 | fn empty_event_time_becomes_null_and_integer_confidence_is_preserved() { | 283 | fn empty_event_time_becomes_null_and_integer_confidence_is_preserved() { |
| 209 | let (window, lookup) = setup(); | 284 | let (window, lookup) = setup(); |
| @@ -2,6 +2,7 @@ use std::collections::HashMap; | |||
| 2 | 2 | ||
| 3 | use memory_pipeline::cache::JsonCache; | 3 | use memory_pipeline::cache::JsonCache; |
| 4 | use memory_pipeline::episode::build_episodes; | 4 | use memory_pipeline::episode::build_episodes; |
| 5 | +use memory_pipeline::error::PipelineStage; | ||
| 5 | use memory_pipeline::extraction::StaticMemoryExtractor; | 6 | use memory_pipeline::extraction::StaticMemoryExtractor; |
| 6 | use memory_pipeline::grounding::StaticGroundingVerifier; | 7 | use memory_pipeline::grounding::StaticGroundingVerifier; |
| 7 | use memory_pipeline::normalize::normalize_prepared_memories; | 8 | use memory_pipeline::normalize::normalize_prepared_memories; |
| @@ -188,3 +189,111 @@ async fn candidate_coverage_excludes_context_only_sources() { | |||
| 188 | 189 | ||
| 189 | assert_eq!(run.stats["candidate_source_coverage"], 1.0); | 190 | assert_eq!(run.stats["candidate_source_coverage"], 1.0); |
| 190 | } | 191 | } |
| 192 | + | ||
| 193 | + | ||
| 194 | +async fn fail_fast_controls_extraction_and_grounding_failures() { | ||
| 195 | + let source = prepared(); | ||
| 196 | + let config = PipelineConfig::default(); | ||
| 197 | + let (messages, _) = normalize_prepared_memories(&source).unwrap(); | ||
| 198 | + let episodes = build_episodes(&messages, &config.episode).unwrap(); | ||
| 199 | + let lookup = messages | ||
| 200 | + .into_iter() | ||
| 201 | + .map(|message| (message.id.clone(), message)) | ||
| 202 | + .collect::<HashMap<_, _>>(); | ||
| 203 | + let window = build_windows(&episodes, &lookup, &config.window) | ||
| 204 | + .unwrap() | ||
| 205 | + .remove(0); | ||
| 206 | + | ||
| 207 | + let missing_extraction = StaticMemoryExtractor::new(HashMap::new()); | ||
| 208 | + let empty_verifier = StaticGroundingVerifier::new(HashMap::new()); | ||
| 209 | + let extraction_error = | ||
| 210 | + run_memory_pipeline(&source, &config, &missing_extraction, &empty_verifier, None) | ||
| 211 | + .await | ||
| 212 | + .unwrap_err(); | ||
| 213 | + assert_eq!(extraction_error.stage(), Some(PipelineStage::Extract)); | ||
| 214 | + | ||
| 215 | + let best_effort = PipelineConfig { | ||
| 216 | + fail_fast: false, | ||
| 217 | + ..config.clone() | ||
| 218 | + }; | ||
| 219 | + let extraction_run = run_memory_pipeline( | ||
| 220 | + &source, | ||
| 221 | + &best_effort, | ||
| 222 | + &missing_extraction, | ||
| 223 | + &empty_verifier, | ||
| 224 | + None, | ||
| 225 | + ) | ||
| 226 | + .await | ||
| 227 | + .unwrap(); | ||
| 228 | + assert_eq!(extraction_run.rejected[0].stage, "extract"); | ||
| 229 | + | ||
| 230 | + let extractor = StaticMemoryExtractor::new(HashMap::from([( | ||
| 231 | + window.id, | ||
| 232 | + json!({"schema_version": "atomic_memory_v1", "memories": [raw_memory()]}), | ||
| 233 | + )])); | ||
| 234 | + let grounding_error = run_memory_pipeline(&source, &config, &extractor, &empty_verifier, None) | ||
| 235 | + .await | ||
| 236 | + .unwrap_err(); | ||
| 237 | + assert_eq!(grounding_error.stage(), Some(PipelineStage::Ground)); | ||
| 238 | + | ||
| 239 | + let grounding_run = | ||
| 240 | + run_memory_pipeline(&source, &best_effort, &extractor, &empty_verifier, None) | ||
| 241 | + .await | ||
| 242 | + .unwrap(); | ||
| 243 | + assert_eq!(grounding_run.quarantined[0].stage, "grounding"); | ||
| 244 | + assert!(grounding_run.accepted_memories.is_empty()); | ||
| 245 | +} | ||
| 246 | + | ||
| 247 | + | ||
| 248 | +async fn only_supported_grounding_results_are_accepted() { | ||
| 249 | + for status in ["PARTIALLY_SUPPORTED", "UNSUPPORTED", "UNCERTAIN"] { | ||
| 250 | + let source = prepared(); | ||
| 251 | + let config = PipelineConfig::default(); | ||
| 252 | + let (messages, _) = normalize_prepared_memories(&source).unwrap(); | ||
| 253 | + let episodes = build_episodes(&messages, &config.episode).unwrap(); | ||
| 254 | + let lookup = messages | ||
| 255 | + .into_iter() | ||
| 256 | + .map(|message| (message.id.clone(), message)) | ||
| 257 | + .collect::<HashMap<_, _>>(); | ||
| 258 | + let window = build_windows(&episodes, &lookup, &config.window) | ||
| 259 | + .unwrap() | ||
| 260 | + .remove(0); | ||
| 261 | + let candidate = validate_extraction( | ||
| 262 | + &[raw_memory()], | ||
| 263 | + &window, | ||
| 264 | + &lookup, | ||
| 265 | + &ValidationConfig::default(), | ||
| 266 | + ) | ||
| 267 | + .valid | ||
| 268 | + .remove(0); | ||
| 269 | + let extractor = StaticMemoryExtractor::new(HashMap::from([( | ||
| 270 | + window.id, | ||
| 271 | + json!({"schema_version": "atomic_memory_v1", "memories": [raw_memory()]}), | ||
| 272 | + )])); | ||
| 273 | + let verifier = StaticGroundingVerifier::new(HashMap::from([(candidate.id, json!(status))])); | ||
| 274 | + | ||
| 275 | + let run = run_memory_pipeline(&source, &config, &extractor, &verifier, None) | ||
| 276 | + .await | ||
| 277 | + .unwrap(); | ||
| 278 | + | ||
| 279 | + assert!(run.accepted_memories.is_empty()); | ||
| 280 | + assert_eq!(run.quarantined.len(), 1); | ||
| 281 | + assert_eq!(run.quarantined[0].details["status"], status); | ||
| 282 | + } | ||
| 283 | +} | ||
| 284 | + | ||
| 285 | + | ||
| 286 | +async fn zero_max_memory_chars_is_rejected() { | ||
| 287 | + let mut config = PipelineConfig::default(); | ||
| 288 | + config.validation.max_memory_chars = 0; | ||
| 289 | + let extractor = StaticMemoryExtractor::new(HashMap::new()); | ||
| 290 | + let verifier = StaticGroundingVerifier::new(HashMap::new()); | ||
| 291 | + | ||
| 292 | + let error = run_memory_pipeline(&prepared(), &config, &extractor, &verifier, None) | ||
| 293 | + .await | ||
| 294 | + .unwrap_err(); | ||
| 295 | + | ||
| 296 | + assert!(error | ||
| 297 | + .to_string() | ||
| 298 | + .contains("max_memory_chars must be positive")); | ||
| 299 | +} | ||
| @@ -0,0 +1,284 @@ | |||
| 1 | +use std::collections::HashMap; | ||
| 2 | +use std::io::{self, Write}; | ||
| 3 | +use std::sync::{Arc, Mutex}; | ||
| 4 | + | ||
| 5 | +use memory_pipeline::episode::build_episodes; | ||
| 6 | +use memory_pipeline::error::PipelineStage; | ||
| 7 | +use memory_pipeline::extraction::StaticMemoryExtractor; | ||
| 8 | +use memory_pipeline::grounding::StaticGroundingVerifier; | ||
| 9 | +use memory_pipeline::normalize::normalize_prepared_memories; | ||
| 10 | +use memory_pipeline::pipeline::{run_memory_pipeline, PipelineConfig}; | ||
| 11 | +use memory_pipeline::validation::{validate_extraction, ValidationConfig}; | ||
| 12 | +use memory_pipeline::window::build_windows; | ||
| 13 | +use serde_json::{json, Value}; | ||
| 14 | +use tracing_subscriber::fmt::MakeWriter; | ||
| 15 | + | ||
| 16 | + | ||
| 17 | +struct LogBuffer(Arc<Mutex<Vec<u8>>>); | ||
| 18 | + | ||
| 19 | +struct LogWriter(Arc<Mutex<Vec<u8>>>); | ||
| 20 | + | ||
| 21 | +impl Write for LogWriter { | ||
| 22 | + fn write(&mut self, bytes: &[u8]) -> io::Result<usize> { | ||
| 23 | + self.0 | ||
| 24 | + .lock() | ||
| 25 | + .expect("log buffer lock") | ||
| 26 | + .extend_from_slice(bytes); | ||
| 27 | + Ok(bytes.len()) | ||
| 28 | + } | ||
| 29 | + | ||
| 30 | + fn flush(&mut self) -> io::Result<()> { | ||
| 31 | + Ok(()) | ||
| 32 | + } | ||
| 33 | +} | ||
| 34 | + | ||
| 35 | +impl<'a> MakeWriter<'a> for LogBuffer { | ||
| 36 | + type Writer = LogWriter; | ||
| 37 | + | ||
| 38 | + fn make_writer(&'a self) -> Self::Writer { | ||
| 39 | + LogWriter(self.0.clone()) | ||
| 40 | + } | ||
| 41 | +} | ||
| 42 | + | ||
| 43 | +impl LogBuffer { | ||
| 44 | + fn records(&self) -> Vec<Value> { | ||
| 45 | + let bytes = self.0.lock().expect("log buffer lock").clone(); | ||
| 46 | + String::from_utf8(bytes) | ||
| 47 | + .expect("UTF-8 tracing output") | ||
| 48 | + .lines() | ||
| 49 | + .map(|line| serde_json::from_str(line).expect("JSON tracing record")) | ||
| 50 | + .collect() | ||
| 51 | + } | ||
| 52 | + | ||
| 53 | + fn text(&self) -> String { | ||
| 54 | + String::from_utf8(self.0.lock().expect("log buffer lock").clone()) | ||
| 55 | + .expect("UTF-8 tracing output") | ||
| 56 | + } | ||
| 57 | +} | ||
| 58 | + | ||
| 59 | +fn prepared() -> Value { | ||
| 60 | + json!({ | ||
| 61 | + "schema_version": "benchmark-prepared-v1", | ||
| 62 | + "dataset": {"name": "logging-test"}, | ||
| 63 | + "memories": [{ | ||
| 64 | + "id": "message-1", | ||
| 65 | + "text": "PRIVATE_PIPELINE_TEST Alice prefers tea.", | ||
| 66 | + "metadata": { | ||
| 67 | + "scope_id": "scope-1", | ||
| 68 | + "session_id": "conversation-1", | ||
| 69 | + "role": "user", | ||
| 70 | + "speaker": "Alice", | ||
| 71 | + "timestamp": "2026-08-17T10:00:00Z", | ||
| 72 | + "memory_candidate": true | ||
| 73 | + } | ||
| 74 | + }] | ||
| 75 | + }) | ||
| 76 | +} | ||
| 77 | + | ||
| 78 | +fn raw_memory() -> Value { | ||
| 79 | + json!({ | ||
| 80 | + "text": "Alice prefers tea.", | ||
| 81 | + "memory_type": "preference", | ||
| 82 | + "subject": {"name": "Alice", "source_speaker": "Alice"}, | ||
| 83 | + "predicate": "prefers", | ||
| 84 | + "object": {"name": "tea", "type": "drink"}, | ||
| 85 | + "modality": "asserted", | ||
| 86 | + "event_time": null, | ||
| 87 | + "attributes": {}, | ||
| 88 | + "evidence": [{ | ||
| 89 | + "message_id": "message-1", | ||
| 90 | + "quote": "Alice prefers tea.", | ||
| 91 | + "evidence_role": "primary" | ||
| 92 | + }], | ||
| 93 | + "model_confidence": 0.95 | ||
| 94 | + }) | ||
| 95 | +} | ||
| 96 | + | ||
| 97 | +fn successful_components( | ||
| 98 | + source: &Value, | ||
| 99 | + config: &PipelineConfig, | ||
| 100 | +) -> (StaticMemoryExtractor, StaticGroundingVerifier) { | ||
| 101 | + let (messages, _) = normalize_prepared_memories(source).expect("normalize fixture"); | ||
| 102 | + let episodes = build_episodes(&messages, &config.episode).expect("build episodes"); | ||
| 103 | + let lookup = messages | ||
| 104 | + .into_iter() | ||
| 105 | + .map(|message| (message.id.clone(), message)) | ||
| 106 | + .collect::<HashMap<_, _>>(); | ||
| 107 | + let window = build_windows(&episodes, &lookup, &config.window) | ||
| 108 | + .expect("build windows") | ||
| 109 | + .remove(0); | ||
| 110 | + let candidate = validate_extraction( | ||
| 111 | + &[raw_memory()], | ||
| 112 | + &window, | ||
| 113 | + &lookup, | ||
| 114 | + &ValidationConfig::default(), | ||
| 115 | + ) | ||
| 116 | + .valid | ||
| 117 | + .remove(0); | ||
| 118 | + ( | ||
| 119 | + StaticMemoryExtractor::new(HashMap::from([( | ||
| 120 | + window.id, | ||
| 121 | + json!({"schema_version": "atomic_memory_v1", "memories": [raw_memory()]}), | ||
| 122 | + )])), | ||
| 123 | + StaticGroundingVerifier::new(HashMap::from([(candidate.id, json!("SUPPORTED"))])), | ||
| 124 | + ) | ||
| 125 | +} | ||
| 126 | + | ||
| 127 | +fn event_stages(records: &[Value], event: &str) -> Vec<String> { | ||
| 128 | + records | ||
| 129 | + .iter() | ||
| 130 | + .filter_map(|record| { | ||
| 131 | + let fields = record.get("fields")?; | ||
| 132 | + (fields.get("event")?.as_str()? == event) | ||
| 133 | + .then(|| fields.get("stage")?.as_str().map(str::to_owned)) | ||
| 134 | + .flatten() | ||
| 135 | + }) | ||
| 136 | + .collect() | ||
| 137 | +} | ||
| 138 | + | ||
| 139 | +fn failure_codes(records: &[Value]) -> HashMap<String, String> { | ||
| 140 | + records | ||
| 141 | + .iter() | ||
| 142 | + .filter_map(|record| { | ||
| 143 | + let fields = record.get("fields")?; | ||
| 144 | + (fields.get("event")?.as_str()? == "ram_a.memory.ingest.stage.failed") | ||
| 145 | + .then(|| { | ||
| 146 | + Some(( | ||
| 147 | + fields.get("stage")?.as_str()?.to_owned(), | ||
| 148 | + fields.get("error_code")?.as_str()?.to_owned(), | ||
| 149 | + )) | ||
| 150 | + }) | ||
| 151 | + .flatten() | ||
| 152 | + }) | ||
| 153 | + .collect() | ||
| 154 | +} | ||
| 155 | + | ||
| 156 | + | ||
| 157 | +async fn successful_pipeline_logs_all_seven_stages_without_memory_content() { | ||
| 158 | + let logs = LogBuffer::default(); | ||
| 159 | + let subscriber = tracing_subscriber::fmt() | ||
| 160 | + .json() | ||
| 161 | + .without_time() | ||
| 162 | + .with_max_level(tracing::Level::INFO) | ||
| 163 | + .with_writer(logs.clone()) | ||
| 164 | + .finish(); | ||
| 165 | + let _subscriber = tracing::subscriber::set_default(subscriber); | ||
| 166 | + let source = prepared(); | ||
| 167 | + let config = PipelineConfig::default(); | ||
| 168 | + let (extractor, verifier) = successful_components(&source, &config); | ||
| 169 | + | ||
| 170 | + run_memory_pipeline(&source, &config, &extractor, &verifier, None) | ||
| 171 | + .await | ||
| 172 | + .expect("successful pipeline"); | ||
| 173 | + | ||
| 174 | + let records = logs.records(); | ||
| 175 | + let expected = vec![ | ||
| 176 | + "normalize", | ||
| 177 | + "episode", | ||
| 178 | + "window", | ||
| 179 | + "extract", | ||
| 180 | + "validate", | ||
| 181 | + "ground", | ||
| 182 | + "aggregate", | ||
| 183 | + ]; | ||
| 184 | + assert_eq!( | ||
| 185 | + event_stages(&records, "ram_a.memory.ingest.stage.started"), | ||
| 186 | + expected | ||
| 187 | + ); | ||
| 188 | + assert_eq!( | ||
| 189 | + event_stages(&records, "ram_a.memory.ingest.stage.completed"), | ||
| 190 | + expected | ||
| 191 | + ); | ||
| 192 | + assert!(!logs.text().contains("PRIVATE_PIPELINE_TEST")); | ||
| 193 | + assert!(!logs.text().contains("Alice prefers tea.")); | ||
| 194 | +} | ||
| 195 | + | ||
| 196 | + | ||
| 197 | +async fn pipeline_logs_every_reachable_fatal_stage_failure() { | ||
| 198 | + let logs = LogBuffer::default(); | ||
| 199 | + let subscriber = tracing_subscriber::fmt() | ||
| 200 | + .json() | ||
| 201 | + .without_time() | ||
| 202 | + .with_max_level(tracing::Level::INFO) | ||
| 203 | + .with_writer(logs.clone()) | ||
| 204 | + .finish(); | ||
| 205 | + let _subscriber = tracing::subscriber::set_default(subscriber); | ||
| 206 | + let source = prepared(); | ||
| 207 | + let empty_extractor = StaticMemoryExtractor::new(HashMap::new()); | ||
| 208 | + let empty_verifier = StaticGroundingVerifier::new(HashMap::new()); | ||
| 209 | + | ||
| 210 | + let mut invalid_validation = PipelineConfig::default(); | ||
| 211 | + invalid_validation.validation.max_memory_chars = 0; | ||
| 212 | + let error = run_memory_pipeline( | ||
| 213 | + &source, | ||
| 214 | + &invalid_validation, | ||
| 215 | + &empty_extractor, | ||
| 216 | + &empty_verifier, | ||
| 217 | + None, | ||
| 218 | + ) | ||
| 219 | + .await | ||
| 220 | + .expect_err("invalid validation config"); | ||
| 221 | + assert_eq!(error.stage(), Some(PipelineStage::Validate)); | ||
| 222 | + | ||
| 223 | + let error = run_memory_pipeline( | ||
| 224 | + &json!({"schema_version": "wrong"}), | ||
| 225 | + &PipelineConfig::default(), | ||
| 226 | + &empty_extractor, | ||
| 227 | + &empty_verifier, | ||
| 228 | + None, | ||
| 229 | + ) | ||
| 230 | + .await | ||
| 231 | + .expect_err("invalid normalize input"); | ||
| 232 | + assert_eq!(error.stage(), Some(PipelineStage::Normalize)); | ||
| 233 | + | ||
| 234 | + let mut invalid_episode = PipelineConfig::default(); | ||
| 235 | + invalid_episode.episode.max_time_gap_minutes = Some(-1); | ||
| 236 | + let error = run_memory_pipeline( | ||
| 237 | + &source, | ||
| 238 | + &invalid_episode, | ||
| 239 | + &empty_extractor, | ||
| 240 | + &empty_verifier, | ||
| 241 | + None, | ||
| 242 | + ) | ||
| 243 | + .await | ||
| 244 | + .expect_err("invalid episode config"); | ||
| 245 | + assert_eq!(error.stage(), Some(PipelineStage::Episode)); | ||
| 246 | + | ||
| 247 | + let mut invalid_window = PipelineConfig::default(); | ||
| 248 | + invalid_window.window.max_candidate_tokens = 0; | ||
| 249 | + let error = run_memory_pipeline( | ||
| 250 | + &source, | ||
| 251 | + &invalid_window, | ||
| 252 | + &empty_extractor, | ||
| 253 | + &empty_verifier, | ||
| 254 | + None, | ||
| 255 | + ) | ||
| 256 | + .await | ||
| 257 | + .expect_err("invalid window config"); | ||
| 258 | + assert_eq!(error.stage(), Some(PipelineStage::Window)); | ||
| 259 | + | ||
| 260 | + let config = PipelineConfig::default(); | ||
| 261 | + let error = run_memory_pipeline(&source, &config, &empty_extractor, &empty_verifier, None) | ||
| 262 | + .await | ||
| 263 | + .expect_err("extractor failure"); | ||
| 264 | + assert_eq!(error.stage(), Some(PipelineStage::Extract)); | ||
| 265 | + | ||
| 266 | + let (extractor, _) = successful_components(&source, &config); | ||
| 267 | + let error = run_memory_pipeline(&source, &config, &extractor, &empty_verifier, None) | ||
| 268 | + .await | ||
| 269 | + .expect_err("grounding failure"); | ||
| 270 | + assert_eq!(error.stage(), Some(PipelineStage::Ground)); | ||
| 271 | + | ||
| 272 | + let failures = failure_codes(&logs.records()); | ||
| 273 | + for (stage, code) in [ | ||
| 274 | + ("validate", "PIPELINE_VALIDATE_FAILED"), | ||
| 275 | + ("normalize", "PIPELINE_NORMALIZE_FAILED"), | ||
| 276 | + ("episode", "PIPELINE_EPISODE_FAILED"), | ||
| 277 | + ("window", "PIPELINE_WINDOW_FAILED"), | ||
| 278 | + ("extract", "PIPELINE_EXTRACT_FAILED"), | ||
| 279 | + ("ground", "PIPELINE_GROUND_FAILED"), | ||
| 280 | + ] { | ||
| 281 | + assert_eq!(failures.get(stage).map(String::as_str), Some(code)); | ||
| 282 | + } | ||
| 283 | + assert!(!failures.contains_key("aggregate")); | ||
| 284 | +} | ||
| @@ -29,6 +29,12 @@ repository. | |||
| 29 | - [guides/memory-cases-qa-evaluation.md](guides/memory-cases-qa-evaluation.md): QA eval | 29 | - [guides/memory-cases-qa-evaluation.md](guides/memory-cases-qa-evaluation.md): QA eval |
| 30 | test flow, case schema, coverage, strengths and limitations, and comparison with | 30 | test flow, case schema, coverage, strengths and limitations, and comparison with |
| 31 | external retrieval tests. | 31 | external retrieval tests. |
| 32 | +- [guides/ram-a-mem-configuration-and-pipeline.zh-CN.md](guides/ram-a-mem-configuration-and-pipeline.zh-CN.md): | ||
| 33 | + RAM-A-MEM 完整配置、摄入七阶段、完整摄入链和检索链的数据契约。 | ||
| 34 | +- [guides/ram-a-mem-configuration-reference.zh-CN.md](guides/ram-a-mem-configuration-reference.zh-CN.md): | ||
| 35 | + RAM-A-MEM 全量配置字段、默认值、推荐值、条件依赖和测试要求。 | ||
| 36 | +- [guides/ram-a-mem-rpm-agent-self-test.zh-CN.md](guides/ram-a-mem-rpm-agent-self-test.zh-CN.md): | ||
| 37 | + 在 openEuler 容器中安装 RAM-A RPM,并通过裸 MCP 和 xiaoO 验证摄入、检索与持久化。 | ||
| 32 | - [guides/xiaoo-case-library-integration.md](guides/xiaoo-case-library-integration.md): | 38 | - [guides/xiaoo-case-library-integration.md](guides/xiaoo-case-library-integration.md): |
| 33 | `memory_case_search` deployment, authorization, tool selection, and xiaoO | 39 | `memory_case_search` deployment, authorization, tool selection, and xiaoO |
| 34 | integration boundary. | 40 | integration boundary. |
| @@ -0,0 +1,551 @@ | |||
| 1 | +# RAM-A-MEM 日志与错误可观测性设计 | ||
| 2 | + | ||
| 3 | +状态:核心日志格式与错误分类已实现,剩余验收项见第 11 节 | ||
| 4 | + | ||
| 5 | +本文定义 RAM-A-MEM 的日志输出、请求关联、错误分类和脱敏规则,并记录当前工作树的 | ||
| 6 | +实现及自动化覆盖。未标注“已覆盖”的验收项仍不能作为已验证能力使用。 | ||
| 7 | + | ||
| 8 | +## 1. 背景与现状 | ||
| 9 | + | ||
| 10 | +当前日志和错误处理存在以下问题: | ||
| 11 | + | ||
| 12 | +1. `init_tracing()` 固定输出 JSON,不能在日志平台采集和终端排障之间切换。 | ||
| 13 | +2. 终端直接查看 JSON 日志时单条记录过长,不利于使用 `tail`、`journalctl` 等工具。 | ||
| 14 | +3. 日志默认不包含源码文件名和行号,现场排障后仍需二次搜索代码。 | ||
| 15 | +4. MCP 工具错误响应未返回 `request_id`,调用方难以将失败响应与服务端日志关联。 | ||
| 16 | +5. Pipeline 错误已经保留 `stage`,但根因在映射为 `PIPELINE_FAILED` 时仍会丢失; | ||
| 17 | + `STORAGE_FAILED` 仍同时覆盖 embedding、幂等表和记忆存储错误。 | ||
| 18 | +6. Extract 处理多消息或复杂窗口失败时,当前信息不足以区分网络错误、空模型输出、 | ||
| 19 | + Extract JSON 非法、Extract schema 不匹配等情况;Provider 日志中的 | ||
| 20 | + `error_kind=request` 仍然过粗。 | ||
| 21 | + | ||
| 22 | +当前未提交代码已经形成以下基线,本设计在此基础上增量完善: | ||
| 23 | + | ||
| 24 | +| 能力 | 当前状态 | | ||
| 25 | +| --- | --- | | ||
| 26 | +| JSON 结构化输出 | 已支持,可通过环境变量选择。 | | ||
| 27 | +| 七阶段 started/completed/failed 事件 | 已支持,并有 `pipeline_logging.rs` 自动化用例。 | | ||
| 28 | +| Pipeline 失败阶段透传 | 已支持,MCP 错误可返回 `stage=extract/ground` 等值。 | | ||
| 29 | +| LLM/Rerank retry 和 failed 事件 | 已支持;LLM 可区分空内容、非法 JSON、响应读取、超时、HTTP 状态和连接错误。 | | ||
| 30 | +| Rerank 独立错误 | 已支持 `RERANK_FAILED`。 | | ||
| 31 | +| request span | 已支持,日志可继承 `request_id` 和 `scope_id_hash`。 | | ||
| 32 | +| Compact、源码位置、错误响应 request_id | 已支持;格式器、非法配置和 MCP 失败响应均有自动化用例。 | | ||
| 33 | +| 细粒度 Extract/Storage 根因 | 已支持主要类别;未知存储错误仍保留 `STORAGE_FAILED` 兜底。 | | ||
| 34 | + | ||
| 35 | +## 2. 目标与非目标 | ||
| 36 | + | ||
| 37 | +### 2.1 目标 | ||
| 38 | + | ||
| 39 | +- 通过环境变量在 `json` 和 `compact` 两种格式之间切换。 | ||
| 40 | +- 可选择输出源码文件名和行号。 | ||
| 41 | +- 使用稳定业务字段关联 HTTP 请求、MCP 工具调用和 Pipeline 执行。 | ||
| 42 | +- 在不记录用户内容和凭据的前提下,区分主要失败组件和失败类型。 | ||
| 43 | +- 为格式、字段、错误映射、脱敏和行为不变性提供自动化测试。 | ||
| 44 | + | ||
| 45 | +### 2.2 非目标 | ||
| 46 | + | ||
| 47 | +- 不在 RAM-A-MEM 进程内实现日志文件落盘、轮转、压缩和保留。 | ||
| 48 | +- 不把源码文件名、行号或自然语言错误摘要定义为稳定告警接口。 | ||
| 49 | +- 不记录模型原始请求、模型原始响应、消息正文或记忆正文来辅助排障。 | ||
| 50 | +- 本设计不引入分布式追踪系统;`request_id` 和 `pipeline_run_id` 是本阶段的关联手段。 | ||
| 51 | +- 本设计不改变记忆抽取、校验、Grounding、持久化或检索算法。 | ||
| 52 | + | ||
| 53 | +## 3. 配置设计 | ||
| 54 | + | ||
| 55 | +日志渲染配置只从环境变量读取。目标状态下,日志级别由 `RUST_LOG` 控制,未设置时 | ||
| 56 | +默认为 `info`。现有版本若已提供 `logging.level`,应在迁移时标记为弃用;不能为了 | ||
| 57 | +读取该字段而把日志初始化推迟到完整服务配置加载之后。 | ||
| 58 | + | ||
| 59 | +| 环境变量 | 合法值 | 默认值 | 作用 | | ||
| 60 | +| --- | --- | --- | --- | | ||
| 61 | +| `RAM_A_LOG_FORMAT` | `json`、`compact` | `json` | 选择结构化采集格式或终端紧凑格式。 | | ||
| 62 | +| `RAM_A_LOG_SOURCE` | `true`、`false` | `false` | 是否为全部日志增加事件打印位置;不控制失败日志必备的错误起点。 | | ||
| 63 | + | ||
| 64 | +解析规则: | ||
| 65 | + | ||
| 66 | +1. 环境变量未设置时使用默认值。 | ||
| 67 | +2. 值按区分大小写的精确字符串解析;空字符串、前后空格、`JSON`、`1`、`yes` 等 | ||
| 68 | + 均视为非法值。 | ||
| 69 | +3. 任一值非法时,进程必须在创建 HTTP listener、打开数据库和构造外部 Provider | ||
| 70 | + 之前退出,退出码非 0。 | ||
| 71 | +4. 由于此时日志系统尚未初始化,启动错误直接向 stderr 输出单行安全文本,文本包含 | ||
| 72 | + 环境变量名、非法值类别和合法值列表,但不得打印其他环境变量。 | ||
| 73 | + | ||
| 74 | +示例: | ||
| 75 | + | ||
| 76 | +```text | ||
| 77 | +invalid RAM_A_LOG_FORMAT; expected one of: json, compact | ||
| 78 | +``` | ||
| 79 | + | ||
| 80 | +建议新增独立的 `LogSettings::from_env()`,并在 `main` 的业务配置加载和运行时资源 | ||
| 81 | +初始化之前完成解析与 subscriber 初始化。格式和源码位置选项不加入 | ||
| 82 | +`ram-a-mem.json`,避免服务配置文件和环境变量形成两套优先级。 | ||
| 83 | + | ||
| 84 | +## 4. 输出目标 | ||
| 85 | + | ||
| 86 | +RAM-A-MEM 应将应用日志写入 stderr,由运行环境负责收集: | ||
| 87 | + | ||
| 88 | +- systemd:journald; | ||
| 89 | +- 容器:容器日志驱动; | ||
| 90 | +- 文件部署:由 shell 重定向或日志采集代理落盘; | ||
| 91 | +- 文件轮转和保留:由 journald、容器运行时或 logrotate 管理。 | ||
| 92 | + | ||
| 93 | +目标实现不直接创建 `/var/log/ram-a`,也不持有日志文件句柄,不新增 | ||
| 94 | +`logging.directory` 等进程内文件输出配置。 | ||
| 95 | + | ||
| 96 | +## 5. 日志格式 | ||
| 97 | + | ||
| 98 | +### 5.1 JSON 格式 | ||
| 99 | + | ||
| 100 | +`RAM_A_LOG_FORMAT=json` 时,每条日志为一个完整 JSON 对象,并以换行分隔。RAM-A | ||
| 101 | +自身产生的业务日志至少满足以下结构: | ||
| 102 | + | ||
| 103 | +```json | ||
| 104 | +{ | ||
| 105 | + "timestamp": "2026-08-20T08:30:00.123456Z", | ||
| 106 | + "level": "ERROR", | ||
| 107 | + "target": "memory_mcp::service", | ||
| 108 | + "fields": { | ||
| 109 | + "event": "ram_a.memory.ingest.failed", | ||
| 110 | + "message": "memory ingest failed", | ||
| 111 | + "operation": "memory_ingest", | ||
| 112 | + "request_id": "4f65c9ad-7db2-4dbe-a836-3bce6adbd736", | ||
| 113 | + "pipeline_run_id": "run-b4a31b0c-2c5d-47e3-b895-8df12e90323a", | ||
| 114 | + "stage": "vector_persist", | ||
| 115 | + "error_code": "EMBEDDING_FAILED", | ||
| 116 | + "source_error_kind": "timeout", | ||
| 117 | + "source_error_message": "embedding provider timed out", | ||
| 118 | + "error_site": "memory_mcp.ingest.vector_persist", | ||
| 119 | + "error_origin_file": "crates/memory-core/src/embedding.rs", | ||
| 120 | + "error_origin_line": 153 | ||
| 121 | + }, | ||
| 122 | + "filename": "crates/memory-mcp/src/service.rs", | ||
| 123 | + "line_number": 298 | ||
| 124 | +} | ||
| 125 | +``` | ||
| 126 | + | ||
| 127 | +字段要求: | ||
| 128 | + | ||
| 129 | +| 字段 | 要求 | | ||
| 130 | +| --- | --- | | ||
| 131 | +| `timestamp` | 必须存在,使用 UTC RFC3339 时间。 | | ||
| 132 | +| `level` | 必须存在,取 `TRACE/DEBUG/INFO/WARN/ERROR`。 | | ||
| 133 | +| `target` | 必须存在,用于区分模块;不得关闭 target 输出。 | | ||
| 134 | +| `fields.event` | RAM-A 自有业务事件必须存在,使用稳定事件名。 | | ||
| 135 | +| `fields.message` | RAM-A 自有事件必须存在,提供简短、可读且不含用户内容的说明。 | | ||
| 136 | +| `fields.operation` | 工具、Pipeline、Provider 和存储事件必须存在,例如 `memory_ingest`、`chat_completion`。 | | ||
| 137 | +| `fields.request_id` | 进入 HTTP middleware 后产生的日志必须尽可能携带。 | | ||
| 138 | +| `fields.pipeline_run_id` | 已创建 Pipeline run 后的摄入日志必须携带。 | | ||
| 139 | +| `fields.stage` | Pipeline 阶段和服务内部阶段事件必须携带。 | | ||
| 140 | +| `fields.error_code` | 对外或内部失败事件必须携带。 | | ||
| 141 | +| `fields.error_site` | RAM-A 自有失败事件必须存在,标识稳定的逻辑错误点。 | | ||
| 142 | +| `filename`、`line_number` | 日志打印位置,仅在 `RAM_A_LOG_SOURCE=true` 时存在。 | | ||
| 143 | +| `fields.error_origin_file`、`fields.error_origin_line` | 错误首次在 RAM-A 中形成或被分类的位置;RAM-A 自有失败日志必须存在,不受 source 开关影响。 | | ||
| 144 | + | ||
| 145 | +`rmcp`、`hyper` 等第三方 crate 的日志可能没有 `fields.event`。该要求只约束 RAM-A | ||
| 146 | +自有事件;告警规则应通过 `target` 或 `fields.event` 过滤掉不相关的第三方日志。 | ||
| 147 | + | ||
| 148 | +### 5.2 Compact 格式 | ||
| 149 | + | ||
| 150 | +`RAM_A_LOG_FORMAT=compact` 面向 `tail`、`journalctl` 和容器手工测试。它不是简单地 | ||
| 151 | +把 JSON 字段全部摊平成一行,而是使用固定视觉顺序,将最重要的信息放在前面。 | ||
| 152 | + | ||
| 153 | +普通事件和失败事件分别使用以下固定格式: | ||
| 154 | + | ||
| 155 | +```text | ||
| 156 | +普通事件:[<timestamp>] [<level>] [<target>] [<operation>/<stage>] <message> | <context fields> | ||
| 157 | +失败事件:[<timestamp>] [<level>] [<error origin>] [<operation>/<stage>] <error_code>: <error summary> | <diagnostic fields> | ||
| 158 | +``` | ||
| 159 | + | ||
| 160 | +失败事件把真正的 RAM-A 错误起点放在视觉前部,不要求排障人员先扫到行尾。字段顺序 | ||
| 161 | +固定为: | ||
| 162 | + | ||
| 163 | +1. `timestamp`:UTC RFC3339,精确到毫秒。 | ||
| 164 | +2. `level`:`TRACE/DEBUG/INFO/WARN/ERROR`。 | ||
| 165 | +3. 普通事件显示 `target`;失败事件显示 `error_origin_file:error_origin_line`。 | ||
| 166 | +4. `operation/stage`:例如 `memory_ingest/extract`;没有 stage 时只显示 operation。 | ||
| 167 | +5. 正常事件使用安全的 `message`;失败事件使用 | ||
| 168 | + `<error_code>: <source_error_message>`。 | ||
| 169 | +6. 后置字段:失败事件先输出 `target` 和 `error_site`,随后按约定顺序输出 request_id、 | ||
| 170 | + pipeline_run_id、retriable、进度和耗时等 `key=value` 字段。 | ||
| 171 | + | ||
| 172 | +失败示例: | ||
| 173 | + | ||
| 174 | +```text | ||
| 175 | +[2026-08-20T08:30:00.123Z] [ERROR] [crates/memory-core/src/embedding.rs:153] [memory_ingest/vector_persist] EMBEDDING_FAILED: embedding provider timed out | target=memory_mcp::service site=memory_mcp.ingest.vector_persist request_id=4f65c9ad-7db2-4dbe-a836-3bce6adbd736 pipeline_run_id=run-b4a31b0c-2c5d-47e3-b895-8df12e90323a retriable=true | ||
| 176 | +``` | ||
| 177 | + | ||
| 178 | +阶段成功示例: | ||
| 179 | + | ||
| 180 | +```text | ||
| 181 | +[2026-08-20T08:30:00.456Z] [INFO] [memory_pipeline::pipeline] [memory_ingest/extract] extraction completed | request_id=4f65c9ad-7db2-4dbe-a836-3bce6adbd736 pipeline_run_id=run-b4a31b0c-2c5d-47e3-b895-8df12e90323a completed_units=1 total_units=1 elapsed_ms=328 | ||
| 182 | +``` | ||
| 183 | + | ||
| 184 | +Provider 重试示例: | ||
| 185 | + | ||
| 186 | +```text | ||
| 187 | +[2026-08-20T08:30:01.000Z] [WARN] [memory_pipeline::client] [provider/chat_completion] retrying LLM request after timeout | request_id=4f65c9ad-7db2-4dbe-a836-3bce6adbd736 model=glm-5.1 attempt=2 max_attempts=3 backoff_ms=2000 | ||
| 188 | +``` | ||
| 189 | + | ||
| 190 | +渲染规则: | ||
| 191 | + | ||
| 192 | +- 每条事件严格单行;换行、回车和制表符转义为 `\n`、`\r`、`\t`。 | ||
| 193 | +- 字符串含空格、引号或 `=` 时使用 JSON 字符串转义,避免字段边界不清。 | ||
| 194 | +- 已知诊断字段按上面的顺序输出,其余字段按字段名排序,保证同类日志布局稳定。 | ||
| 195 | +- 不重复输出已经进入固定前缀的 `timestamp`、`level`、`target`、operation 和 stage。 | ||
| 196 | +- 失败日志始终包含 origin;source=true 时额外增加 | ||
| 197 | + `log_at=<filename>:<line_number>`,用于定位日志打印行。 | ||
| 198 | +- 不输出 ANSI 颜色码到非终端输出;是否在交互式 TTY 中按级别着色属于实现细节, | ||
| 199 | + 不能影响文本内容和自动化断言。 | ||
| 200 | + | ||
| 201 | +Compact 格式只改变渲染方式,不改变底层结构化事件字段。实现上应提供 RAM-A 自定义 | ||
| 202 | +`FormatEvent`,不能直接依赖 tracing 默认 Compact 的字段顺序。 | ||
| 203 | + | ||
| 204 | +这一设计采用主流日志系统的共同结构:OpenTelemetry 将 timestamp、severity、可读 | ||
| 205 | +Body、EventName 和 Attributes 分开;Go `slog` 的文本格式固定输出 time、level、msg | ||
| 206 | +后再跟结构化属性;Log4j Pattern Layout 使用日期、级别、logger、message 的固定模式; | ||
| 207 | +systemd `journalctl short-iso` 使用 RFC3339 单行展示。参考: | ||
| 208 | + | ||
| 209 | +- [OpenTelemetry Logs Data Model](https://opentelemetry.io/docs/specs/otel/logs/data-model/) | ||
| 210 | +- [Go structured logging with slog](https://go.dev/blog/slog) | ||
| 211 | +- [Apache Log4j Pattern Layout](https://logging.apache.org/log4j/2.x/manual/pattern-layout.html) | ||
| 212 | +- [tracing-subscriber fmt formatters](https://docs.rs/tracing-subscriber/latest/tracing_subscriber/fmt/index.html) | ||
| 213 | +- [journalctl output formats](https://www.freedesktop.org/software/systemd/man/latest/journalctl.html) | ||
| 214 | + | ||
| 215 | +### 5.3 源码位置 | ||
| 216 | + | ||
| 217 | +单独启用 tracing subscriber 的源码字段还不足以定位根因。它只说明 `error!()` 在哪一 | ||
| 218 | +行执行;如果日志在 `service.rs` 的 `map_err` 中产生,只能定位到错误映射代码,不能 | ||
| 219 | +定位到 `embedding.rs`、Extractor JSON 解析或 SQLite 操作的实际失败边界。 | ||
| 220 | + | ||
| 221 | +因此错误起点不能依赖 `RAM_A_LOG_SOURCE`。两类位置的输出条件如下: | ||
| 222 | + | ||
| 223 | +| 字段 | 含义 | | ||
| 224 | +| --- | --- | | ||
| 225 | +| `filename`、`line_number` | 日志事件的打印位置,由 tracing subscriber 提供;source=true 时为全部日志输出。 | | ||
| 226 | +| `error_origin_file`、`error_origin_line` | 错误首次在 RAM-A 代码中创建,或第三方错误首次被 RAM-A 分类的位置;RAM-A 自有失败日志始终输出。 | | ||
| 227 | +| `error_site` | 不依赖物理行号的稳定逻辑位置,例如 `memory_pipeline.extract.parse_response`;RAM-A 自有失败日志始终输出。 | | ||
| 228 | + | ||
| 229 | +具体行为: | ||
| 230 | + | ||
| 231 | +| 日志类型 | source=false | source=true | | ||
| 232 | +| --- | --- | --- | | ||
| 233 | +| 成功、启动、阶段开始等普通日志 | 不输出物理源码位置。 | 输出 `filename/line_number`。 | | ||
| 234 | +| RAM-A 自有失败日志 | 输出 `error_site` 和 `error_origin_file/error_origin_line`。 | 在左侧字段基础上,额外输出日志打印位置 `filename/line_number`。 | | ||
| 235 | +| 第三方 crate 日志 | 不保证源码位置。 | 尽可能输出 `filename/line_number`;不要求有 RAM-A error origin。 | | ||
| 236 | + | ||
| 237 | +错误起点遵循“第一次记录,后续只传递”的原则: | ||
| 238 | + | ||
| 239 | +1. RAM-A 自身发现非法 JSON、schema 错误、维度不匹配等问题时,在创建结构化错误的 | ||
| 240 | + 同一行记录 origin。 | ||
| 241 | +2. reqwest、rusqlite 等第三方错误没有可依赖的 RAM-A 源码行;在它们第一次转换成 | ||
| 242 | + RAM-A 错误的边界记录 origin,例如 embedding HTTP 调用、幂等 reserve SQL 或 | ||
| 243 | + vector store 写入位置。 | ||
| 244 | +3. `PipelineError::at_stage`、`ServiceError` 映射和 MCP 响应包装必须保留已有 origin, | ||
| 245 | + 不得用外层 `map_err` 行号覆盖。 | ||
| 246 | +4. Provider 服务端内部哪一行失败无法由 RAM-A 获得;日志只能精确到 RAM-A 的调用 | ||
| 247 | + 边界,并结合 HTTP 状态和 `source_error_kind` 判断远端原因。 | ||
| 248 | + | ||
| 249 | +建议错误类型携带以下上下文: | ||
| 250 | + | ||
| 251 | +```rust | ||
| 252 | +pub struct ErrorOrigin { | ||
| 253 | + pub site: &'static str, | ||
| 254 | + pub file: &'static str, | ||
| 255 | + pub line: u32, | ||
| 256 | +} | ||
| 257 | +``` | ||
| 258 | + | ||
| 259 | +通过带 `#[track_caller]` 的错误构造函数或统一宏读取 | ||
| 260 | +`std::panic::Location::caller()`。不建议默认打印 Backtrace:它体积大、依赖调试符号, | ||
| 261 | +还可能暴露构建路径;Backtrace 可作为开发期临时诊断手段,但不属于稳定日志协议。 | ||
| 262 | + | ||
| 263 | +`filename/line_number` 和 `error_origin_file/error_origin_line` 都会随源码重构改变,不能 | ||
| 264 | +用于稳定告警。日志检索先使用 `error_site`、`event`、`stage` 和 `error_code`,需要查看 | ||
| 265 | +具体实现时再使用文件名和行号。默认 source=false 不影响失败日志定位,只减少普通日志 | ||
| 266 | +中的物理源码字段。 | ||
| 267 | + | ||
| 268 | +## 6. 关联标识与业务字段 | ||
| 269 | + | ||
| 270 | +### 6.1 标识传播 | ||
| 271 | + | ||
| 272 | +1. HTTP middleware 为每个请求生成 `request_id`,继续通过 `x-request-id` 响应头 | ||
| 273 | + 返回。 | ||
| 274 | +2. `request_id` 必须写入请求 extension/span,使 MCP handler、MemoryService、 | ||
| 275 | + Pipeline、Provider 和持久化日志继承同一个值。 | ||
| 276 | +3. `memory_ingest` 在幂等 reserve 确定 run 后,将 `pipeline_run_id` 加入后续阶段日志。 | ||
| 277 | +4. MCP 工具错误的 `structuredContent` 返回 `request_id`,并与 HTTP 响应头一致。 | ||
| 278 | +5. 将 `pipeline_run_id` 同时返回到失败响应仍是后续项;当前调用方可先通过 | ||
| 279 | + `request_id` 关联包含 `pipeline_run_id` 的服务端失败日志。 | ||
| 280 | + | ||
| 281 | +目标工具错误响应示例(当前实现尚不返回其中的 `pipeline_run_id`): | ||
| 282 | + | ||
| 283 | +```json | ||
| 284 | +{ | ||
| 285 | + "code": "PIPELINE_FAILED", | ||
| 286 | + "message": "memory pipeline failed", | ||
| 287 | + "retriable": true, | ||
| 288 | + "request_id": "4f65c9ad-7db2-4dbe-a836-3bce6adbd736", | ||
| 289 | + "pipeline_run_id": "run-b4a31b0c-2c5d-47e3-b895-8df12e90323a", | ||
| 290 | + "stage": "extract" | ||
| 291 | +} | ||
| 292 | +``` | ||
| 293 | + | ||
| 294 | +响应不返回 `source_error_message`,避免向客户端暴露内部实现、Provider 返回内容或 | ||
| 295 | +存储路径。调用方使用 `request_id` 查询服务端日志。 | ||
| 296 | + | ||
| 297 | +### 6.2 安全业务字段 | ||
| 298 | + | ||
| 299 | +日志可记录以下统计或派生字段: | ||
| 300 | + | ||
| 301 | +- `scope_id_hash`、`conversation_id_hash`:带版本前缀的 SHA-256 摘要前 16 个小写 | ||
| 302 | + 十六进制字符,仅用于日志关联; | ||
| 303 | +- `message_count`、`candidate_message_count`; | ||
| 304 | +- `window_count`、`window_message_count`、`window_candidate_count`; | ||
| 305 | +- `accepted_count`、`rejected_count`、`quarantined_count`; | ||
| 306 | +- `record_count`、`completed_units`、`total_units`; | ||
| 307 | +- Provider 类型、模型名、attempt、max_attempts、backoff_ms 和 HTTP 状态码。 | ||
| 308 | + | ||
| 309 | +不得记录原始 `scope_id`、`conversation_id`、`message_id` 或用未加版本前缀的裸 | ||
| 310 | +摘要代替上述 hash。日志 hash 只用于关联,不能参与鉴权、幂等或数据查询。 | ||
| 311 | + | ||
| 312 | +## 7. 事件模型 | ||
| 313 | + | ||
| 314 | +事件名使用小写点分层命名,字段值使用 `snake_case`。第一阶段至少覆盖下列事件: | ||
| 315 | + | ||
| 316 | +| event | level | 必要业务字段 | | ||
| 317 | +| --- | --- | --- | | ||
| 318 | +| `ram_a.service.started` | INFO | `message`、`log_format`、`log_source` | | ||
| 319 | +| `ram_a.http.request.completed` | INFO | `request_id`、`operation`、`status`、`duration_ms` | | ||
| 320 | +| `ram_a.memory.ingest.started` | INFO | `request_id`、两个业务 hash、消息数量 | | ||
| 321 | +| `ram_a.memory.ingest.stage.started` | INFO | `request_id`、`pipeline_run_id`、`stage`、进度字段 | | ||
| 322 | +| `ram_a.memory.ingest.stage.completed` | INFO | 上述字段、`elapsed_ms` | | ||
| 323 | +| `ram_a.memory.ingest.stage.window_skipped` | WARN | 当前窗口失败但 `fail_fast=false` 继续执行时,记录阶段、窗口 hash 和错误诊断字段 | | ||
| 324 | +| `ram_a.memory.ingest.stage.failed` | ERROR | 上述字段、错误诊断字段 | | ||
| 325 | +| `ram_a.memory.ingest.completed` | INFO | 两个关联 ID、accepted/rejected/quarantined 数量、`latency_ms` | | ||
| 326 | +| `ram_a.memory.ingest.failed` | ERROR | 可用的关联 ID、`stage`、错误诊断字段、`latency_ms` | | ||
| 327 | +| `ram_a.memory.search.started` | INFO | `request_id`、`scope_id_hash`、`query_hash`、`top_k`、检索模式 | | ||
| 328 | +| `ram_a.memory.search.completed` | INFO | `request_id`、返回数量、检索模式、`latency_ms` | | ||
| 329 | +| `ram_a.memory.search.failed` | ERROR | `request_id`、`stage`、错误诊断字段、`latency_ms` | | ||
| 330 | +| `ram_a.provider.retry` | WARN | 关联 ID、`component`、Provider、model、attempt、backoff、错误分类 | | ||
| 331 | +| `ram_a.provider.failed` | ERROR | 关联 ID、`component`、Provider、model、attempts、错误分类 | | ||
| 332 | +| `ram_a.storage.operation.failed` | ERROR | 关联 ID、`component`、`storage_operation`、错误分类 | | ||
| 333 | + | ||
| 334 | +七阶段 Pipeline 的 `stage` 固定为: | ||
| 335 | + | ||
| 336 | +```text | ||
| 337 | +normalize, episode, window, extract, validate, ground, aggregate | ||
| 338 | +``` | ||
| 339 | + | ||
| 340 | +服务编排阶段可额外使用: | ||
| 341 | + | ||
| 342 | +```text | ||
| 343 | +request_validate, idempotency_reserve, vector_persist, idempotency_complete, | ||
| 344 | +search_embedding, dense_recall, bm25_recall, hybrid_fuse, rerank, post_filter | ||
| 345 | +``` | ||
| 346 | + | ||
| 347 | +这些 stage 值可用于日志检索和测试,但增加新阶段是允许的。重命名或改变已有阶段语义 | ||
| 348 | +需要同步更新文档和测试。 | ||
| 349 | + | ||
| 350 | +`stage.failed` 只表示该失败终止了本次 Pipeline。`fail_fast=false` 时,Extract 或 | ||
| 351 | +Verify 的单个窗口失败使用 `stage.window_skipped`,随后仍应产生后续阶段事件和最终 | ||
| 352 | +`ingest.completed`;不得同时将这次请求记录成 `ingest.failed`。 | ||
| 353 | + | ||
| 354 | +## 8. 错误模型 | ||
| 355 | + | ||
| 356 | +### 8.1 分层原则 | ||
| 357 | + | ||
| 358 | +错误分为四层: | ||
| 359 | + | ||
| 360 | +1. 对外 `error_code`:供 MCP 调用方决定是否重试和如何提示。 | ||
| 361 | +2. `stage`、`component`、`storage_operation`:指出失败发生的位置。 | ||
| 362 | +3. 所有 RAM-A 自有失败日志中的 `error_site` 和 origin:定位到逻辑错误点和源码行。 | ||
| 363 | +4. 仅日志可见的 `source_error_kind`、`source_error_message`:说明底层失败类别。 | ||
| 364 | + | ||
| 365 | +服务内部错误类型必须同时保留 cause 和最初的 `ErrorOrigin`,不能在 `map_err` 时立即 | ||
| 366 | +压缩为无上下文的枚举值。建议将 `ServiceError` 改为携带 code、stage、retriable、 | ||
| 367 | +origin 和 source 的结构,或为每个错误变体保留 `#[source]` 和 origin。 | ||
| 368 | + | ||
| 369 | +### 8.2 对外错误码 | ||
| 370 | + | ||
| 371 | +| error_code | 适用范围 | retriable | | ||
| 372 | +| --- | --- | --- | | ||
| 373 | +| `INVALID_REQUEST` | 请求字段或业务边界校验失败。 | `false` | | ||
| 374 | +| `IDEMPOTENCY_CONFLICT` | 同一幂等消息的 content hash 改变。 | `false` | | ||
| 375 | +| `PIPELINE_FAILED` | Normalize 至 Aggregate 阶段失败;用 `stage` 区分。 | 由根因决定 | | ||
| 376 | +| `RERANK_FAILED` | Rerank 已启用且调用失败,同时 `fail_open=false`。 | `true` | | ||
| 377 | +| `EMBEDDING_FAILED` | 摄入或检索调用 embedding Provider 失败或返回无效向量。 | 由根因决定 | | ||
| 378 | +| `IDEMPOTENCY_STORAGE_FAILED` | 幂等 reserve/complete 失败,且未命中更具体的 SQLite 错误。 | `true` | | ||
| 379 | +| `SQLITE_BUSY` | 任意 SQLite 操作返回 busy/locked。 | `true` | | ||
| 380 | +| `SQLITE_READONLY` | SQLite 数据库或目录不可写。 | `false` | | ||
| 381 | +| `VECTOR_PERSIST_FAILED` | embedding 已完成,但记忆记录或向量写入失败,且未命中具体 SQLite 错误。 | `true` | | ||
| 382 | +| `STORAGE_FAILED` | 无法归入上述类别的兼容性兜底。 | `true` | | ||
| 383 | + | ||
| 384 | +Provider 根因的 `retriable` 规则:连接失败、超时、HTTP 408/425/429/5xx 为 `true`; | ||
| 385 | +其他明确的 HTTP 4xx、配置错误和 embedding 维度不匹配为 `false`。模型返回空内容、 | ||
| 386 | +非法 JSON 或 schema 错误在内部重试耗尽后仍映射为 `PIPELINE_FAILED`;由于下一次模型 | ||
| 387 | +生成可能不同,第一阶段将其标记为 `true`。 | ||
| 388 | + | ||
| 389 | +错误码选择优先级如下: | ||
| 390 | + | ||
| 391 | +```text | ||
| 392 | +SQLITE_BUSY / SQLITE_READONLY | ||
| 393 | + > EMBEDDING_FAILED | ||
| 394 | + > IDEMPOTENCY_STORAGE_FAILED / VECTOR_PERSIST_FAILED | ||
| 395 | + > STORAGE_FAILED | ||
| 396 | +``` | ||
| 397 | + | ||
| 398 | +例如,幂等 reserve 遇到 SQLite busy 时返回 `SQLITE_BUSY`,同时日志中的 | ||
| 399 | +`component=idempotency`、`storage_operation=reserve` 表明具体位置,不再同时返回 | ||
| 400 | +`IDEMPOTENCY_STORAGE_FAILED`。 | ||
| 401 | + | ||
| 402 | +### 8.3 内部错误分类 | ||
| 403 | + | ||
| 404 | +`source_error_kind` 使用受控枚举,不直接使用底层错误字符串: | ||
| 405 | + | ||
| 406 | +```text | ||
| 407 | +timeout, connect, http_status, response_read, empty_content, invalid_json, | ||
| 408 | +schema_invalid, embedding_invalid_response, embedding_dimension_mismatch, | ||
| 409 | +sqlite_busy, sqlite_readonly, sqlite_other, io, cancelled, internal | ||
| 410 | +``` | ||
| 411 | + | ||
| 412 | +模型输出 schema 问题可额外记录不含原文的 `schema_issue`: | ||
| 413 | + | ||
| 414 | +```text | ||
| 415 | +root_not_object, schema_version_mismatch, memories_not_array, | ||
| 416 | +memory_not_object, duplicate_memory_id, results_not_array, | ||
| 417 | +missing_result, duplicate_result, unexpected_result, invalid_enum | ||
| 418 | +``` | ||
| 419 | + | ||
| 420 | +`source_error_message` 必须由本地受控模板生成,而不是直接记录 HTTP body、模型输出或 | ||
| 421 | +任意 `Display` 链。它必须满足: | ||
| 422 | + | ||
| 423 | +- 单行; | ||
| 424 | +- 最多 512 个 Unicode 字符; | ||
| 425 | +- 不包含 URL query、请求/响应 body、文件中的业务数据或环境变量值; | ||
| 426 | +- 对 Authorization、Bearer、token、api_key 等模式执行二次脱敏; | ||
| 427 | +- 可用于人工阅读,但不作为告警条件。 | ||
| 428 | + | ||
| 429 | +### 8.4 多消息 Extract 诊断 | ||
| 430 | + | ||
| 431 | +多消息 Extract 失败时,日志必须能够回答“哪个窗口、失败在哪一类”,但不能泄露窗口 | ||
| 432 | +内容。失败事件至少记录: | ||
| 433 | + | ||
| 434 | +```text | ||
| 435 | +request_id, pipeline_run_id, stage=extract, window_id_hash, | ||
| 436 | +window_message_count, window_candidate_count, model, attempts, | ||
| 437 | +error_code=PIPELINE_FAILED, source_error_kind | ||
| 438 | +``` | ||
| 439 | + | ||
| 440 | +按失败情况补充: | ||
| 441 | + | ||
| 442 | +| 情况 | source_error_kind | 可选字段 | | ||
| 443 | +| --- | --- | --- | | ||
| 444 | +| 连接失败或超时 | `connect` / `timeout` | `attempts` | | ||
| 445 | +| 非成功 HTTP | `http_status` | `http_status`、`attempts` | | ||
| 446 | +| HTTP 200 但 content 为空 | `empty_content` | `completion_tokens`(可用时) | | ||
| 447 | +| content 不是合法 JSON | `invalid_json` | 不记录 content | | ||
| 448 | +| JSON 合法但 Extract schema 不匹配 | `schema_invalid` | `schema_issue` | | ||
| 449 | + | ||
| 450 | +这样可以定位此前多消息请求的失败类别,但不会将消息正文或模型原始输出写入日志。 | ||
| 451 | + | ||
| 452 | +## 9. 安全与隐私约束 | ||
| 453 | + | ||
| 454 | +所有日志格式和级别均不得记录: | ||
| 455 | + | ||
| 456 | +- Bearer Token、API Key、Authorization header 或密钥环境变量值; | ||
| 457 | +- 消息正文、query 原文、Evidence quote; | ||
| 458 | +- 完整记忆正文、模型 prompt、模型原始 response; | ||
| 459 | +- 未脱敏的 Provider 响应 body; | ||
| 460 | +- 原始 tenant_id、user_id、agent_id、scope_id、conversation_id、message_id。 | ||
| 461 | + | ||
| 462 | +允许记录 query、scope、conversation、window 等内容的版本化摘要和长度/数量。即使在 | ||
| 463 | +`RUST_LOG=trace`、`RAM_A_LOG_SOURCE=true` 时,上述禁止项仍不得出现。 | ||
| 464 | + | ||
| 465 | +## 10. 实现建议 | ||
| 466 | + | ||
| 467 | +建议按以下边界拆分代码: | ||
| 468 | + | ||
| 469 | +1. `memory-mcp::observability`:环境变量解析、自定义 Compact `FormatEvent`、subscriber | ||
| 470 | + 构建、日志 hash 和安全摘要。 | ||
| 471 | +2. HTTP middleware:创建 request span,写入 `request_id` 和安全 principal hash。 | ||
| 472 | +3. MCP handler:将 `request_id` 传入工具错误响应。 | ||
| 473 | +4. 公共错误模块:提供 `ErrorOrigin`、`#[track_caller]` 构造函数和首次 origin 保留规则。 | ||
| 474 | +5. MemoryService:保留内部错误 cause/origin,增加 service stage 和 `pipeline_run_id` span。 | ||
| 475 | +6. `memory-pipeline` client/extractor/verifier:返回结构化错误类别和 origin,并记录 Provider retry。 | ||
| 476 | +7. `memory-core`:保留 `MemoryError` 变体和 origin,在 MCP 边界映射 embedding、SQLite 和 | ||
| 477 | + vector persist 错误。 | ||
| 478 | + | ||
| 479 | +日志事件应在错误最终归属层记录一次。底层 Provider 可以记录 retry;最终失败由 | ||
| 480 | +Pipeline 或 Service 记录。避免同一错误在每层打印完整 ERROR,造成一条失败出现多条 | ||
| 481 | +无法区分的重复日志。 | ||
| 482 | + | ||
| 483 | +## 11. 自动化验收用例 | ||
| 484 | + | ||
| 485 | +建议新增或扩展以下测试。日志 subscriber 是进程级全局状态,格式和非法环境变量测试 | ||
| 486 | +应优先通过启动子进程执行,避免测试间互相污染。 | ||
| 487 | + | ||
| 488 | +当前自动化已经覆盖:格式配置严格解析、JSON/Compact 渲染、失败 origin 与日志打印点 | ||
| 489 | +区分、Pipeline 七阶段日志、模型错误分类、主要存储错误映射,以及 HTTP 响应头与 | ||
| 490 | +`structuredContent.request_id` 一致。以下项目仍未完整覆盖:带有效服务配置启动进程后 | ||
| 491 | +验证四种格式/source 组合、所有敏感字段的端到端哨兵测试、多消息窗口 hash 与 | ||
| 492 | +`schema_issue` 诊断、失败响应返回 `pipeline_run_id`。这些项目不能作为当前已验证能力。 | ||
| 493 | + | ||
| 494 | +| 用例 | 建议位置 | 操作 | 预期 | | ||
| 495 | +| --- | --- | --- | --- | | ||
| 496 | +| 默认 JSON | `crates/memory-mcp/tests/logging.rs` | 不设置两个环境变量,启动测试进程并触发普通事件和失败事件。 | 每行可解析为 JSON;普通事件无源码字段,失败事件有 error origin。 | | ||
| 497 | +| Compact 切换 | 同上 | 设置 `RAM_A_LOG_FORMAT=compact`。 | 日志严格符合固定前缀顺序,错误摘要位于诊断字段之前。 | | ||
| 498 | +| Compact 转义与顺序 | 同上 | message 和属性包含空格、引号、`=`、换行及乱序字段。 | 输出保持单行、可区分字段边界,已知字段和剩余字段顺序稳定。 | | ||
| 499 | +| JSON 源码位置 | 同上 | 设置 `json` 和 `RAM_A_LOG_SOURCE=true`。 | `filename` 为字符串,`line_number` 为正整数。 | | ||
| 500 | +| Compact 源码位置 | 同上 | 设置 `compact` 和 source=true。 | 同一行含源码文件名和行号。 | | ||
| 501 | +| 错误起点 | `crates/memory-mcp/tests/logging.rs` 和各错误模块单元测试 | 分别在 source=false/true 下,于已知行通过 `#[track_caller]` 构造错误,再经过 Pipeline、Service、MCP 多层包装。 | 两种配置均输出 origin;origin 始终等于首次构造位置,不等于外层日志行;`error_site` 保持不变。 | | ||
| 502 | +| 第三方错误边界 | 同上 | 注入 reqwest/rusqlite/Store 错误。 | origin 指向第一次转换为 RAM-A 错误的位置,日志不声称能定位远端服务内部行。 | | ||
| 503 | +| 非法 format | 同上 | 分别设置空值、`JSON`、`text`。 | 进程非 0 退出;stderr 给出变量名和合法值。 | | ||
| 504 | +| 非法 source | 同上 | 分别设置空值、`TRUE`、`1`。 | 进程非 0 退出;未打开 listener 或数据库。 | | ||
| 505 | +| JSON 必要字段 | 同上 | 触发 ingest 成功、stage 失败和 search 成功事件。 | 按第 5 节逐项断言字段。 | | ||
| 506 | +| 关联 ID 贯通 | `crates/memory-mcp/tests/http_mcp.rs` | 发起失败的 `memory_ingest`。 | 响应头、structuredContent 和服务日志中的 request_id 相同;run 创建后 pipeline_run_id 相同。 | | ||
| 507 | +| 格式不改变行为 | 同上 | 使用静态 Extractor/Verifier 分别在 json、compact 下执行相同请求。 | 工具响应、错误码、accepted/quarantine 结果一致。 | | ||
| 508 | +| Extract 空 content | `crates/memory-pipeline/src/client.rs` 与 MCP 集成测试 | mock Provider 返回 HTTP 200 且 content 为空。 | 重试后日志分类为 `empty_content`,不含响应 body。 | | ||
| 509 | +| Extract 非法 JSON | `crates/memory-pipeline/tests/` | mock content 为非法 JSON。 | `stage=extract`、`source_error_kind=invalid_json`。 | | ||
| 510 | +| Extract schema 错误 | 同上 | 返回错误 schema_version 或非数组 memories。 | `schema_invalid` 和对应 `schema_issue`。 | | ||
| 511 | +| 多消息 Extract 失败 | MCP 集成测试 | 构造多个 candidate,mock Extractor 在单个窗口返回结构化错误。 | 记录消息/窗口数量和 hash,不记录消息正文。 | | ||
| 512 | +| Embedding 失败 | `crates/memory-mcp/tests/service.rs` | 注入 `MemoryError::Embedding`。 | 响应 `EMBEDDING_FAILED`,stage 为 vector_persist 或 search_embedding。 | | ||
| 513 | +| 幂等存储失败 | 同上 | 注入 reserve/complete 非 SQLite 特定错误。 | `IDEMPOTENCY_STORAGE_FAILED` 和对应 operation。 | | ||
| 514 | +| SQLite busy | 同上 | 使用可控错误注入返回 busy/locked。 | `SQLITE_BUSY`、retriable=true。 | | ||
| 515 | +| SQLite readonly | 同上 | 使用可控错误注入返回 readonly。 | `SQLITE_READONLY`、retriable=false。 | | ||
| 516 | +| Vector persist 失败 | 同上 | embedding 成功,store 写入返回非特定错误。 | `VECTOR_PERSIST_FAILED`。 | | ||
| 517 | +| STORAGE 兜底 | 同上 | 注入无法分类的存储错误。 | 保留 `STORAGE_FAILED`,日志分类为 internal。 | | ||
| 518 | +| 敏感信息不落日志 | `crates/memory-mcp/tests/logging.rs` | 使用唯一哨兵作为 token、API key、消息、query、quote 和记忆正文,覆盖成功与失败路径。 | 捕获的 json/compact/trace 日志均不含任一哨兵。 | | ||
| 519 | +| 第三方日志兼容 | 同上 | 触发一条 rmcp 日志。 | 日志可输出;不强制存在 fields.event,不影响 RAM-A 事件断言。 | | ||
| 520 | + | ||
| 521 | +SQLite busy/readonly 用例优先使用错误注入或受控测试 Store,不依赖宿主文件权限和时序, | ||
| 522 | +以保证 CI 稳定。另增加少量真实 SQLite 集成测试验证底层错误到分类器的映射即可。 | ||
| 523 | + | ||
| 524 | +## 12. 验收标准 | ||
| 525 | + | ||
| 526 | +满足以下条件后可认为本设计落地: | ||
| 527 | + | ||
| 528 | +1. 两种格式和 source 开关的四种组合均通过自动化测试。 | ||
| 529 | +2. 非法环境变量均在业务资源初始化前导致启动失败。 | ||
| 530 | +3. RAM-A 自有 JSON 事件满足字段约定;Compact 可单行直接阅读。 | ||
| 531 | +4. source 开关的任意取值下,RAM-A 自有失败日志都包含首次错误起点和稳定 | ||
| 532 | + `error_site`,多层包装不覆盖 origin;source=true 时额外包含日志打印位置。 | ||
| 533 | +5. MCP 工具错误可通过 `request_id` 与日志唯一关联。 | ||
| 534 | +6. Extract 空内容、非法 JSON、schema 错误可以只根据日志分类区分。 | ||
| 535 | +7. embedding、幂等存储、SQLite busy/readonly 和向量持久化错误映射符合第 8 节。 | ||
| 536 | +8. 两种日志格式下,同一测试输入的服务响应和 Pipeline 结果一致。 | ||
| 537 | +9. 敏感信息哨兵测试在 json、compact 和 trace 级别全部通过。 | ||
| 538 | +10. 服务自身不创建、轮转或保留日志文件。 | ||
| 539 | + | ||
| 540 | +## 13. 实施顺序与优先级 | ||
| 541 | + | ||
| 542 | +建议优先级为中等。该问题不阻断记忆摄入和检索,但直接影响容器验证、并发故障定位 | ||
| 543 | +和生产日志平台接入。 | ||
| 544 | + | ||
| 545 | +建议分两步实施: | ||
| 546 | + | ||
| 547 | +1. 先实现环境变量、两种渲染格式、source 开关、事件字段、关联 ID 和脱敏测试。 | ||
| 548 | +2. 再保留底层 cause,完成细粒度错误码映射和 Extract/存储错误分类。 | ||
| 549 | + | ||
| 550 | +第二步会扩展 MCP 对外错误码,提交前需要同步接口规格和调用方容错说明;调用方应以 | ||
| 551 | +`retriable` 决定自动重试,并允许出现新增错误码。 | ||
| @@ -123,6 +123,21 @@ pub trait Reranker: Send + Sync { | |||
| 123 | OpenRouter client 对网络错误、HTTP 429/5xx 和可重试服务错误做有限重试;认证、 | 123 | OpenRouter client 对网络错误、HTTP 429/5xx 和可重试服务错误做有限重试;认证、 |
| 124 | 额度和响应格式错误会直接返回错误。 | 124 | 额度和响应格式错误会直接返回错误。 |
| 125 | 125 | ||
| 126 | +在 `ram-a-mem` 服务中,配置入口为顶层配置文件的 `retrieval.rerank`,不是 | ||
| 127 | +`memory_search` 的请求参数。`enabled=false` 时 `fail_open` 不生效;开启 rerank | ||
| 128 | +还要求 `retrieval.mode=hybrid`。最终错误是指 reranker 返回的任意错误,包括重试耗尽 | ||
| 129 | +的传输或超时错误、非成功 HTTP 状态以及响应格式或索引校验错误。 | ||
| 130 | + | ||
| 131 | +传输错误、响应读取错误、超时及 HTTP 408/425/429/500/502/503/504 最多尝试 8 次, | ||
| 132 | +重试等待为 1/2/4/8/16/32/64 秒。其他 HTTP 状态和响应校验错误不重试。 | ||
| 133 | + | ||
| 134 | +`fail_open=false` 的最终错误通过 MCP 工具错误返回:`code=RERANK_FAILED`、 | ||
| 135 | +`message="memory rerank failed"`、`retriable=true`。`fail_open=true` 返回 rerank 前的 | ||
| 136 | +hybrid 顺序并截断至 `top_k`,同时记录 `ram_a.memory.search.degraded` 日志。 | ||
| 137 | + | ||
| 138 | +MCP 服务的 `timeout_ms` 默认值为 30000;启用 rerank 时必须位于 1..=120000。 | ||
| 139 | +超时按单次 provider 请求计算,最终 fail-open/fail-closed 判定发生在重试结束后。 | ||
| 140 | + | ||
| 126 | ## 6. CLI 使用 | 141 | ## 6. CLI 使用 |
| 127 | 142 | ||
| 128 | `memory-bench` 默认不启用 rerank。开启方式: | 143 | `memory-bench` 默认不启用 rerank。开启方式: |
| @@ -29,6 +29,11 @@ delete confirmation tools, `memory_search`, and `memory_ingest` are MCP tools on | |||
| 29 | 29 | ||
| 30 | Create `config/ram-a-mem.json`: | 30 | Create `config/ram-a-mem.json`: |
| 31 | 31 | ||
| 32 | +The checked-in [`plugins/mcp/ram-a-mem.json`](../../plugins/mcp/ram-a-mem.json) explicitly | ||
| 33 | +lists every current server configuration field. For a field-by-field explanation and the | ||
| 34 | +seven-stage ingest data contract, see the | ||
| 35 | +[Chinese configuration and pipeline reference](ram-a-mem-configuration-and-pipeline.zh-CN.md). | ||
| 36 | + | ||
| 32 | ```json | 37 | ```json |
| 33 | { | 38 | { |
| 34 | "auth": { | 39 | "auth": { |
| @@ -60,7 +65,7 @@ Create `config/ram-a-mem.json`: | |||
| 60 | "allowed_hosts": ["127.0.0.1:18081"] | 65 | "allowed_hosts": ["127.0.0.1:18081"] |
| 61 | }, | 66 | }, |
| 62 | "limits": { | 67 | "limits": { |
| 63 | - "max_body_bytes": 1048576, | 68 | + "max_body_bytes": 16777216, |
| 64 | "requests_per_second": 20, | 69 | "requests_per_second": 20, |
| 65 | "rate_burst": 40, | 70 | "rate_burst": 40, |
| 66 | "max_in_flight_per_principal_tool": 4, | 71 | "max_in_flight_per_principal_tool": 4, |
| @@ -70,6 +75,10 @@ Create `config/ram-a-mem.json`: | |||
| 70 | "max_active_sessions_global": 256, | 75 | "max_active_sessions_global": 256, |
| 71 | "session_idle_timeout_seconds": 1800 | 76 | "session_idle_timeout_seconds": 1800 |
| 72 | }, | 77 | }, |
| 78 | + "pipeline": { | ||
| 79 | + "fail_fast": true, | ||
| 80 | + "max_memory_chars": 500 | ||
| 81 | + }, | ||
| 73 | "storage": { | 82 | "storage": { |
| 74 | "database_path": "data/ram-a-memory.sqlite" | 83 | "database_path": "data/ram-a-memory.sqlite" |
| 75 | }, | 84 | }, |
| @@ -122,6 +131,35 @@ Create `config/ram-a-mem.json`: | |||
| 122 | } | 131 | } |
| 123 | ``` | 132 | ``` |
| 124 | 133 | ||
| 134 | +The `limits` values are service-level controls and are not MCP tool arguments: | ||
| 135 | + | ||
| 136 | +| Field | Default | Accepted range | | ||
| 137 | +| --- | ---: | ---: | | ||
| 138 | +| `max_body_bytes` | 16777216 | 1..=67108864 | | ||
| 139 | +| `requests_per_second` | 20 | 1..=10000 | | ||
| 140 | +| `rate_burst` | 40 | 1..=100000 | | ||
| 141 | +| `max_in_flight_per_principal_tool` | 4 | 1..=1024 | | ||
| 142 | +| `initialize_requests_per_second` | 4 | 1..=1000 | | ||
| 143 | +| `initialize_rate_burst` | 8 | 1..=10000 | | ||
| 144 | +| `max_active_sessions_per_principal` | 8 | 1..=1024 | | ||
| 145 | +| `max_active_sessions_global` | 256 | 1..=100000 | | ||
| 146 | +| `session_idle_timeout_seconds` | 1800 | 1..=86400; shared by RAM-A session admission and the underlying rmcp session worker | | ||
| 147 | + | ||
| 148 | +`max_active_sessions_global` must be greater than or equal to | ||
| 149 | +`max_active_sessions_per_principal`. Tool rate and concurrency limits are keyed by authenticated | ||
| 150 | +`scope_id + agent_id + tool name`; initialize and per-principal session limits are keyed by | ||
| 151 | +`scope_id + agent_id`. Concurrency excess is rejected without queueing. A rejected request returns | ||
| 152 | +HTTP 429, `Retry-After: 1`, and an `x-ram-a-limit-reason` value of `tool_rate_limit`, | ||
| 153 | +`tool_concurrency`, `initialize_rate_limit`, or `session_admission`. | ||
| 154 | + | ||
| 155 | +The `pipeline` object controls ingest processing for the whole service. `fail_fast` defaults to | ||
| 156 | +`true`; an Extract or Ground provider failure terminates the request. The MCP tool error uses | ||
| 157 | +`code=PIPELINE_FAILED`, `retriable=true`, and `stage=extract` or `stage=ground`. With | ||
| 158 | +`fail_fast=false`, a failed Extract window is counted as rejected and a failed Ground window is | ||
| 159 | +counted as quarantined; remaining windows continue. `max_memory_chars` defaults to 500 and accepts | ||
| 160 | +1..=32000 Unicode characters. A longer extracted memory is quarantined rather than truncating or | ||
| 161 | +failing the request. | ||
| 162 | + | ||
| 125 | Set secrets in the environment, not in config files: | 163 | Set secrets in the environment, not in config files: |
| 126 | 164 | ||
| 127 | ```bash | 165 | ```bash |
| @@ -322,7 +360,31 @@ pre-rerank hybrid order and emits a `ram_a.memory.search.degraded` event. | |||
| 322 | 360 | ||
| 323 | ## Structured progress logs | 361 | ## Structured progress logs |
| 324 | 362 | ||
| 325 | -`ram-a-mem` writes one-line JSON logs. Use `RUST_LOG` to select the level; the default is `info`. | 363 | +`ram-a-mem` writes logs to stderr. Use `RUST_LOG` to select the level; the default is `info`. |
| 364 | +The rendering format and source location are configured before the service configuration is loaded: | ||
| 365 | + | ||
| 366 | +```bash | ||
| 367 | +# Production/log collector (defaults shown explicitly) | ||
| 368 | +export RAM_A_LOG_FORMAT=json | ||
| 369 | +export RAM_A_LOG_SOURCE=false | ||
| 370 | + | ||
| 371 | +# Terminal debugging | ||
| 372 | +export RAM_A_LOG_FORMAT=compact | ||
| 373 | +export RAM_A_LOG_SOURCE=true | ||
| 374 | +``` | ||
| 375 | + | ||
| 376 | +`RAM_A_LOG_FORMAT` accepts only `json` or `compact`. `RAM_A_LOG_SOURCE` accepts only `true` | ||
| 377 | +or `false`. Invalid, differently cased, or whitespace-padded values make startup fail before the | ||
| 378 | +listener and storage are initialized. JSON is one object per line. Compact output has this shape: | ||
| 379 | + | ||
| 380 | +```text | ||
| 381 | +[2026-08-20T08:30:00.123Z] [ERROR] [crates/memory-pipeline/src/extraction.rs:91] [memory_ingest/extract] PIPELINE_FAILED: model returned invalid JSON | request_id=... pipeline_run_id=... error_site=memory_pipeline.extract.parse_response source_error_kind=invalid_json | ||
| 382 | +``` | ||
| 383 | + | ||
| 384 | +RAM-A failure events include the error origin regardless of `RAM_A_LOG_SOURCE`. Enabling source | ||
| 385 | +adds the location of each tracing call as `filename`/`line_number` in JSON or `log_at` in compact | ||
| 386 | +output; it is useful for debug sessions but is not a stable alerting field. | ||
| 387 | + | ||
| 326 | Every MCP tool call carries the HTTP `request_id` in its tracing span. Memory ingest emits stage | 388 | Every MCP tool call carries the HTTP `request_id` in its tracing span. Memory ingest emits stage |
| 327 | events for validation, idempotency, normalization, episode/window construction, extraction, | 389 | events for validation, idempotency, normalization, episode/window construction, extraction, |
| 328 | verification, vector persistence, optional graph build, and completion. Hybrid search emits | 390 | verification, vector persistence, optional graph build, and completion. Hybrid search emits |
| @@ -343,6 +405,8 @@ memory text, or provider response bodies. Successful ingest events include gener | |||
| 343 | case task events include `task_id`, `dataset_id`, and `document_id` for operational correlation. | 405 | case task events include `task_id`, `dataset_id`, and `document_id` for operational correlation. |
| 344 | `window_skipped` is emitted only for fail-open extraction or verification errors where the | 406 | `window_skipped` is emitted only for fail-open extraction or verification errors where the |
| 345 | pipeline continues; `failed` means the current ingest operation stops. | 407 | pipeline continues; `failed` means the current ingest operation stops. |
| 408 | +The process does not create or rotate log files; journald, the container runtime, or an external | ||
| 409 | +collector owns persistence and retention. | ||
| 346 | 410 | ||
| 347 | ## Storage boundary | 411 | ## Storage boundary |
| 348 | 412 | ||
| @@ -0,0 +1,825 @@ | |||
| 1 | +# RAM-A-MEM 完整配置与记忆管线 | ||
| 2 | + | ||
| 3 | +本文以 `ram-a-mem` HTTP MCP 服务的当前实现为准。仓库中的完整配置文件是 | ||
| 4 | +[`plugins/mcp/ram-a-mem.json`](../../plugins/mcp/ram-a-mem.json)。该文件显式列出了 | ||
| 5 | +`ServerConfig` 的全部配置模块和字段;密钥只写环境变量名称,不写密钥值。 | ||
| 6 | +逐字段的默认值、推荐值、约束、生效条件和测试要求见 | ||
| 7 | +[`ram-a-mem-configuration-reference.zh-CN.md`](ram-a-mem-configuration-reference.zh-CN.md)。 | ||
| 8 | +RPM 和 xiaoO 的可执行环境验收步骤见 | ||
| 9 | +[`ram-a-mem-rpm-agent-self-test.zh-CN.md`](ram-a-mem-rpm-agent-self-test.zh-CN.md)。 | ||
| 10 | + | ||
| 11 | +## 配置生效规则 | ||
| 12 | + | ||
| 13 | +- 配置文件采用 JSON,并拒绝未知字段。 | ||
| 14 | +- `auth`、持久化 `storage` 和 `providers` 是生产启动必需项。 | ||
| 15 | +- `features.*.enabled` 决定对应模块是否对外提供能力;配置对象存在不代表功能已开启。 | ||
| 16 | +- `graph_memory` 仅在 `features.graph_memory.enabled=true` 时参与摄入和检索。 | ||
| 17 | +- `case_library` 仅在案例库功能开启时构建案例库服务。 | ||
| 18 | +- `retrieval.rerank` 仅在 `enabled=true` 时调用 Rerank 服务。 | ||
| 19 | +- 配置中的 `*_env` 是环境变量名称。实际 Token/API Key 必须通过进程环境注入。 | ||
| 20 | + | ||
| 21 | +## 配置模块说明 | ||
| 22 | + | ||
| 23 | +### `auth` | ||
| 24 | + | ||
| 25 | +`tokens` 至少包含一个主体绑定。每项中的 `token_env` 指向 Bearer Token 环境变量; | ||
| 26 | +`tenant_id + user_id` 用于派生数据隔离 `scope_id`,`agent_id` 用于可选的 | ||
| 27 | +`x-agent-id` 一致性校验。`permissions` 当前支持 `memory:read`、`memory:write`、 | ||
| 28 | +`cases:read` 和 `cases:write`。不同配置项不能解析为相同 Token。 | ||
| 29 | + | ||
| 30 | +### `features` | ||
| 31 | + | ||
| 32 | +- `memory.enabled`:控制 `memory_ingest` 和 `memory_search`。 | ||
| 33 | +- `case_library.enabled`:控制案例检索和案例变更工具;显式开启时必须提供 | ||
| 34 | + `case_library`。 | ||
| 35 | +- `graph_memory.enabled`:在原子记忆基础上启用图构建和图检索;必须同时开启 | ||
| 36 | + `memory` 并提供 `graph_memory`。 | ||
| 37 | + | ||
| 38 | +### `http` | ||
| 39 | + | ||
| 40 | +- `bind_address`、`port`:监听地址和端口。 | ||
| 41 | +- `allowed_origins`:允许浏览器跨域访问的 Origin;空数组表示不授予跨域来源。 | ||
| 42 | +- `allowed_hosts`:允许的 HTTP Host,不能为空。 | ||
| 43 | +- `tls_termination_acknowledged`:非 loopback 地址监听时必须为 `true`,表示部署者已在 | ||
| 44 | + 服务前配置 TLS termination;该字段本身不会启用 TLS。 | ||
| 45 | + | ||
| 46 | +### `limits` | ||
| 47 | + | ||
| 48 | +这些限制作用于整个服务,不是 MCP Tool 参数。 | ||
| 49 | + | ||
| 50 | +| 字段 | 默认值 | 有效范围 | 作用 | | ||
| 51 | +| --- | ---: | ---: | --- | | ||
| 52 | +| `max_body_bytes` | 16777216 | 1..=67108864 | 单个 MCP HTTP 请求体上限 | | ||
| 53 | +| `requests_per_second` | 20 | 1..=10000 | 每主体、每 Tool 的持续速率 | | ||
| 54 | +| `rate_burst` | 40 | 1..=100000 | 每主体、每 Tool 的突发容量 | | ||
| 55 | +| `max_in_flight_per_principal_tool` | 4 | 1..=1024 | 每主体、每 Tool 的并发请求数 | | ||
| 56 | +| `initialize_requests_per_second` | 4 | 1..=1000 | MCP initialize 持续速率 | | ||
| 57 | +| `initialize_rate_burst` | 8 | 1..=10000 | MCP initialize 突发容量 | | ||
| 58 | +| `max_active_sessions_per_principal` | 8 | 1..=1024 | 每主体的活动 Session 数 | | ||
| 59 | +| `max_active_sessions_global` | 256 | 1..=100000 | 进程内活动 Session 总数 | | ||
| 60 | +| `session_idle_timeout_seconds` | 1800 | 1..=86400 | MCP Session 空闲回收时间,同时作用于 RAM-A Admission 和底层 `rmcp` Session Worker | | ||
| 61 | + | ||
| 62 | +全局 Session 上限必须不小于单主体上限。并发超限不会排队,直接返回 HTTP 429。 | ||
| 63 | + | ||
| 64 | +### `pipeline` | ||
| 65 | + | ||
| 66 | +- `fail_fast`:默认 `true`,只控制 Extract 和 Ground 的模型调用/协议错误。为 | ||
| 67 | + `true` 时终止整个摄入,本次管线结果不写入正式记忆;为 `false` 时跳过失败窗口, | ||
| 68 | + 继续处理其他窗口。 | ||
| 69 | +- `max_memory_chars`:默认 500,有效范围 1..=32000,按 Unicode 字符计数。超长抽取 | ||
| 70 | + 结果进入 quarantine,不截断,也不使请求失败。 | ||
| 71 | + | ||
| 72 | +Episode 和 Window 的参数目前没有暴露为服务配置。服务固定使用代码默认值:Episode | ||
| 73 | +不按时间间隔和 metadata 字段主动切分;Window 的候选预算为 320 个估算 Token、总预算 | ||
| 74 | +为 640、向前取 2 条上下文、向后取 0 条上下文。这里的 Token 是本地启发式估算,不是 | ||
| 75 | +模型 Tokenizer 的精确计数。 | ||
| 76 | + | ||
| 77 | +### `storage` | ||
| 78 | + | ||
| 79 | +`database_path` 是正式记忆、幂等记录和可选图数据使用的持久化 SQLite 文件。生产配置 | ||
| 80 | +不接受空路径或 `:memory:`。同一数据库应只由一个 `ram-a-mem` 进程写入。 | ||
| 81 | + | ||
| 82 | +### `providers` | ||
| 83 | + | ||
| 84 | +- `api_key_env`、`base_url`:Extract 和 Ground 共用的 OpenAI-compatible Chat API。 | ||
| 85 | +- `extractor_model`:Extract 阶段模型名。 | ||
| 86 | +- `verifier_model`:Ground 阶段模型名;可以与 Extract 相同,也可以独立配置。 | ||
| 87 | +- `timeout_seconds`、`max_retries`:上述 Chat API 客户端的单次超时和最大尝试次数配置。 | ||
| 88 | +- `embedding_provider`:`hash` 或 `openai_compatible`。`hash` 仅适合离线测试和演示。 | ||
| 89 | +- `embedding_api_key_env`、`embedding_base_url`:可选;未配置时回退到 `api_key_env` 和 | ||
| 90 | + `base_url`。 | ||
| 91 | +- `embedding_model`、`embedding_dimensions`:正式记忆写入和 dense 检索的向量模型及维度。 | ||
| 92 | + | ||
| 93 | +即使选择 `embedding_provider=hash`,`api_key_env` 仍然必需,因为 Extract 和 Ground | ||
| 94 | +仍调用 Chat 模型。 | ||
| 95 | + | ||
| 96 | +### `retrieval` | ||
| 97 | + | ||
| 98 | +- `mode`:`dense`、`bm25` 或 `hybrid`;MCP 服务不接受独立的 `graph` mode。 | ||
| 99 | +- `embedding_weight`、`bm25_weight`:Hybrid 权重,均为 0..=1 且总和必须为 1。 | ||
| 100 | +- `candidate_k`:可选固定候选数,范围 1..=500;为 `null` 时使用核心检索的动态规则。 | ||
| 101 | +- `rerank.enabled`:是否在 Hybrid 结果上调用 Rerank。 | ||
| 102 | +- `rerank.provider`:当前仅支持配置值 `openrouter`,同时兼容同协议的自托管端点。 | ||
| 103 | +- `rerank.model`、`api_key_env`、`base_url`:Rerank 模型和端点。`api_key_env=null` | ||
| 104 | + 可用于无需认证的本地端点。 | ||
| 105 | +- `rerank.input_k`:送入 Rerank 的结果数,范围 1..=500;运行时至少为 `top_k`。 | ||
| 106 | +- `rerank.timeout_ms`:启用时范围 1..=120000。 | ||
| 107 | +- `rerank.fail_open`:`false` 时 Rerank 异常返回 `RERANK_FAILED`;`true` 时返回 Rerank | ||
| 108 | + 前的 Hybrid 顺序。 | ||
| 109 | + | ||
| 110 | +### `case_library` | ||
| 111 | + | ||
| 112 | +- `rag_store`:案例源数据、任务和 chunk 的 SQLite 文件。 | ||
| 113 | +- `index_store`:案例检索索引 SQLite 文件,必须与 `rag_store` 和个人记忆库不同。 | ||
| 114 | +- `source_dir`:可选的本地案例导入目录。 | ||
| 115 | +- `api_token_env`:可选的案例管理 REST API 独立管理员 Token 环境变量。 | ||
| 116 | +- `ingestion_poll_ms`:内置摄入 Worker 的轮询间隔,必须大于 0。 | ||
| 117 | +- `embedding_*`、`chunk_size`:案例 chunk 的向量化配置。 | ||
| 118 | +- `summary_llm_*`:可选的案例摘要模型配置;`summary_llm_model=null` 表示不启用模型摘要。 | ||
| 119 | +- `default_library`、`libraries`:MCP 暴露的逻辑库名到内部 dataset ID 和允许租户的映射。 | ||
| 120 | + | ||
| 121 | +### `graph_memory` | ||
| 122 | + | ||
| 123 | +- `llm_*`:原子记忆写入后进行图实体/关系抽取的 OpenAI-compatible Chat 模型配置。 | ||
| 124 | +- `build_concurrency`:一次摄入请求内的图构建并发数,必须大于 0。 | ||
| 125 | +- `retrieval.weight`:图通道参与融合的权重,范围 0..=1。 | ||
| 126 | +- `rerank_with_graph`、`allow_graph_only`:是否让图结果参加重排、是否允许纯图结果。 | ||
| 127 | +- `max_graph_only_results`、`seed_limit`、`max_evidence_records_per_fact`:`null` 使用代码默认 | ||
| 128 | + 规则;显式值必须大于 0。 | ||
| 129 | +- `fail_open`:图检索失败时是否退回非图结果。图摄入失败与图检索失败是七阶段之外的行为。 | ||
| 130 | + | ||
| 131 | +## `memory_ingest` 完整服务链 | ||
| 132 | + | ||
| 133 | +七阶段只负责“从内部消息形成经过验证的原子记忆”。一次完整的 MCP 摄入还包含协议、认证、 | ||
| 134 | +请求校验、幂等、Embedding 和持久化: | ||
| 135 | + | ||
| 136 | +```text | ||
| 137 | +HTTP/MCP 解析 | ||
| 138 | + -> Bearer Token 认证和 memory:write 鉴权 | ||
| 139 | + -> request_validate | ||
| 140 | + -> scope_id / ingest_lock / idempotency_reserve | ||
| 141 | + -> 构造 prepared JSON | ||
| 142 | + -> Normalize -> Episode -> Window -> Extract -> Validate -> Ground -> Aggregate | ||
| 143 | + -> 构造 AddMemoryRequest | ||
| 144 | + -> Embedding -> vector_persist | ||
| 145 | + -> 可选 graph_build | ||
| 146 | + -> idempotency_complete | ||
| 147 | + -> MCP 响应 | ||
| 148 | +``` | ||
| 149 | + | ||
| 150 | +### 外部输入 | ||
| 151 | + | ||
| 152 | +`memory_ingest` 的 Tool arguments 为: | ||
| 153 | + | ||
| 154 | +```json | ||
| 155 | +{ | ||
| 156 | + "conversation_id": "conv-1", | ||
| 157 | + "messages": [ | ||
| 158 | + { | ||
| 159 | + "id": "m1", | ||
| 160 | + "role": "user", | ||
| 161 | + "speaker": "Alice", | ||
| 162 | + "text": "我喜欢喝绿茶。", | ||
| 163 | + "timestamp": "2026-08-17T10:00:00Z", | ||
| 164 | + "candidate": true | ||
| 165 | + } | ||
| 166 | + ] | ||
| 167 | +} | ||
| 168 | +``` | ||
| 169 | + | ||
| 170 | +### 七阶段外围的输入输出 | ||
| 171 | + | ||
| 172 | +| 处理步骤 | 输入 | 输出或副作用 | | ||
| 173 | +| --- | --- | --- | | ||
| 174 | +| MCP 协议解析 | JSON-RPC `tools/call` 请求 | 解析出 `IngestRequest`;未知字段或错误 JSON 在此拒绝 | | ||
| 175 | +| 认证鉴权 | Bearer Token、可选 `x-agent-id` | `Principal(tenant_id,user_id,agent_id,permissions)` | | ||
| 176 | +| `request_validate` | `IngestRequest` | 校验字段长度、消息数量、role、RFC3339 和请求内 ID 唯一性 | | ||
| 177 | +| scope 派生 | Principal | 不透明 `scope_id`,调用者不能自行指定 | | ||
| 178 | +| `ingest_lock` | scope、conversation、candidate message IDs | 同一摄入键的并发请求在进程内串行执行 | | ||
| 179 | +| `idempotency_reserve` | scope、conversation、message ID、content hash | 全部成功过则返回缓存;新记录或 pending 记录进入 `Proceed` | | ||
| 180 | +| prepared 转换 | 通过校验的请求、Principal、待处理 candidate IDs | 内部 `benchmark-prepared-v1` JSON | | ||
| 181 | +| 七阶段 Pipeline | prepared JSON、PipelineConfig、Extractor、Verifier | accepted memories、rejected、quarantined、stats 和运行元数据 | | ||
| 182 | +| Add 请求转换 | Aggregate 输出、Principal、pipeline run ID | `Vec<AddMemoryRequest>` | | ||
| 183 | +| Embedding | accepted memory text | 与配置维度一致的向量;`hash` 为本地计算,其他 Provider 可调用外部服务 | | ||
| 184 | +| `vector_persist` | AddMemoryRequest + embedding | 正式记忆写入 SQLite,并返回 memory IDs | | ||
| 185 | +| `graph_build` | 已接受的原子记忆 | 仅 Graph 开启时构建图;该步骤位于七阶段和原子记忆写入之后 | | ||
| 186 | +| `idempotency_complete` | 本次成功响应 | pending 记录更新为 success 并保存缓存响应 | | ||
| 187 | + | ||
| 188 | +`request_validate` 不是七阶段中的 Validate。前者校验外部 MCP 参数,失败返回 | ||
| 189 | +`INVALID_REQUEST`,不会进入幂等预占或调用模型;后者校验 Extract 模型产生的原子记忆。 | ||
| 190 | + | ||
| 191 | +当所有消息都是 `candidate=false` 时,不创建幂等记录,也不调用 Extract/Ground 模型;当前 | ||
| 192 | +实现仍可执行 Normalize、Episode、Window 和空 Aggregate,最终返回计数均为 0 的成功响应。 | ||
| 193 | + | ||
| 194 | +### 外部输出 | ||
| 195 | + | ||
| 196 | +成功响应的 Tool structured content 为: | ||
| 197 | + | ||
| 198 | +```json | ||
| 199 | +{ | ||
| 200 | + "pipeline_run_id": "run-<uuid>", | ||
| 201 | + "accepted_count": 1, | ||
| 202 | + "rejected_count": 0, | ||
| 203 | + "quarantined_count": 0, | ||
| 204 | + "memory_ids": ["mem-<hash>"], | ||
| 205 | + "idempotency_hit": false, | ||
| 206 | + "retriable": false | ||
| 207 | +} | ||
| 208 | +``` | ||
| 209 | + | ||
| 210 | +响应只公开计数和正式 memory IDs,不公开 rejected/quarantined issue 明细。七阶段发生致命错误 | ||
| 211 | +时,本次 accepted 中间结果不会写入正式记忆;在失败前创建的幂等预占记录保持 pending,允许 | ||
| 212 | +相同内容重试。已成功写入的历史记忆不受影响。 | ||
| 213 | + | ||
| 214 | +七阶段成功后的持久化和 Graph 不构成一个跨模块数据库事务。`vector_persist` 成功后如果 | ||
| 215 | +`graph_build` 或 `idempotency_complete` 失败,原子记忆可能已经存在,而幂等记录仍是 pending; | ||
| 216 | +相同内容重试依靠稳定 memory ID 做 upsert,并继续未完成工作。因此不能把“Tool 返回失败”一律 | ||
| 217 | +理解为“本次请求绝对没有写入任何数据”,只有七阶段内部的致命失败具备前述不写入保证。 | ||
| 218 | + | ||
| 219 | +## 七阶段与模型调用 | ||
| 220 | + | ||
| 221 | +| 阶段 | 是否调用模型 | 当前模型来源 | 本阶段输入 | 输出给下一阶段 | | ||
| 222 | +| --- | --- | --- | --- | --- | | ||
| 223 | +| Normalize | 否 | 本地 Rust | prepared JSON | `Vec<NormalizedMessage>` | | ||
| 224 | +| Episode | 否 | 本地 Rust | normalized messages + EpisodeConfig | `Vec<ConversationEpisode>` | | ||
| 225 | +| Window | 否 | 本地 Rust | episodes + message lookup + WindowConfig | `Vec<ExtractionWindow>` | | ||
| 226 | +| Extract | 是 | `providers.extractor_model` | 一个 window + 相关 normalized messages | `ExtractionBatch.raw_memories` | | ||
| 227 | +| Validate | 否 | 本地 Rust | raw memories + window + message lookup + ValidationConfig | valid/rejected/quarantined | | ||
| 228 | +| Ground | 是 | `providers.verifier_model` | valid atomic memories + Evidence 对应消息 | `SUPPORTED` 等验证结果 | | ||
| 229 | +| Aggregate | 否 | 本地 Rust | 所有 `SUPPORTED` atomic memories + source lookup | 去重后的正式 memory records | | ||
| 230 | + | ||
| 231 | +Embedding 模型不属于七阶段。它在七阶段成功后写入正式记忆,以及执行 dense 检索时调用。 | ||
| 232 | +Rerank、案例摘要和图抽取模型也不属于这七阶段。 | ||
| 233 | + | ||
| 234 | +## 阶段数据示例 | ||
| 235 | + | ||
| 236 | +下面使用一条上下文消息和一条候选消息说明数据如何传递。示例中的哈希 ID 用占位符表示, | ||
| 237 | +真实值由 `scope_id + conversation_id + message_id` 等字段稳定计算。 | ||
| 238 | + | ||
| 239 | +### 0. MCP 请求转换为 prepared JSON | ||
| 240 | + | ||
| 241 | +MCP 请求中的内容: | ||
| 242 | + | ||
| 243 | +```json | ||
| 244 | +{ | ||
| 245 | + "conversation_id": "conv-1", | ||
| 246 | + "messages": [ | ||
| 247 | + {"id": "m1", "role": "assistant", "text": "你平时喝什么?", "candidate": false}, | ||
| 248 | + {"id": "m2", "role": "user", "speaker": "Alice", "text": "我喜欢喝绿茶。", "timestamp": "2026-08-17T10:00:00Z", "candidate": true} | ||
| 249 | + ] | ||
| 250 | +} | ||
| 251 | +``` | ||
| 252 | + | ||
| 253 | +服务在进入七阶段前生成内部 prepared JSON: | ||
| 254 | + | ||
| 255 | +```json | ||
| 256 | +{ | ||
| 257 | + "schema_version": "benchmark-prepared-v1", | ||
| 258 | + "dataset": {"name": "memory-mcp", "split": "online"}, | ||
| 259 | + "memories": [ | ||
| 260 | + { | ||
| 261 | + "id": "source-<hash-m1>", | ||
| 262 | + "text": "你平时喝什么?", | ||
| 263 | + "metadata": { | ||
| 264 | + "scope_id": "<principal-scope-hash>", | ||
| 265 | + "session_id": "session-<hash-conv-1>", | ||
| 266 | + "role": "assistant", | ||
| 267 | + "speaker": "assistant", | ||
| 268 | + "timestamp": "", | ||
| 269 | + "turn_index": 0, | ||
| 270 | + "source_agent_id": "xiaoo", | ||
| 271 | + "memory_candidate": false | ||
| 272 | + } | ||
| 273 | + }, | ||
| 274 | + { | ||
| 275 | + "id": "source-<hash-m2>", | ||
| 276 | + "text": "我喜欢喝绿茶。", | ||
| 277 | + "metadata": { | ||
| 278 | + "scope_id": "<principal-scope-hash>", | ||
| 279 | + "session_id": "session-<hash-conv-1>", | ||
| 280 | + "role": "user", | ||
| 281 | + "speaker": "Alice", | ||
| 282 | + "timestamp": "2026-08-17T10:00:00Z", | ||
| 283 | + "turn_index": 1, | ||
| 284 | + "source_agent_id": "xiaoo", | ||
| 285 | + "memory_candidate": true | ||
| 286 | + } | ||
| 287 | + } | ||
| 288 | + ], | ||
| 289 | + "queries": [] | ||
| 290 | +} | ||
| 291 | +``` | ||
| 292 | + | ||
| 293 | +### 1. Normalize | ||
| 294 | + | ||
| 295 | +**输入:** 上面的完整 prepared JSON。 | ||
| 296 | + | ||
| 297 | +**实际检查:** `schema_version` 必须是 `benchmark-prepared-v1`;`memories` 必须是数组; | ||
| 298 | +每条记录应为对象并具有唯一非空 `id`、非空 `text`、对象类型 `metadata` 和非空 | ||
| 299 | +`metadata.scope_id`。它还读取 `session_id`、`role`、`speaker`、`timestamp`、 | ||
| 300 | +`turn_index` 和 `memory_candidate`。缺少 ID/scope/text 等单条问题形成 Normalize issue 并跳过; | ||
| 301 | +错误 schema、非数组 memories 或重复 ID 是致命错误。 | ||
| 302 | + | ||
| 303 | +局部问题和致命问题的差别是“是否还能无歧义地处理批次中的其他消息”: | ||
| 304 | + | ||
| 305 | +| 输入问题 | 处理方式 | 原因 | | ||
| 306 | +| --- | --- | --- | | ||
| 307 | +| 某条记录不是对象 | 该条形成 `invalid_source_record` issue 并跳过 | 不影响其他记录 | | ||
| 308 | +| 某条缺少 ID、scope 或有效 text | 该条形成 issue 并跳过 | 该消息不能成为可靠来源,但其他消息仍可处理 | | ||
| 309 | +| 某条 metadata 不是对象 | 该条形成 `invalid_source_metadata` issue 并跳过 | 无法提取可信 scope 和上下文信息 | | ||
| 310 | +| schema version 错误 | 整个 Pipeline 失败 | 不能确认输入协议及字段语义 | | ||
| 311 | +| `memories` 不是数组 | 整个 Pipeline 失败 | 不能把输入解释为消息序列 | | ||
| 312 | +| source ID 重复 | 整个 Pipeline 失败 | 后续 Window 和 Evidence 无法确定 ID 指向哪条消息 | | ||
| 313 | + | ||
| 314 | +Normalize issue 使用统一的 `PipelineIssue` 结构: | ||
| 315 | + | ||
| 316 | +```json | ||
| 317 | +{ | ||
| 318 | + "stage": "normalize", | ||
| 319 | + "code": "missing_scope_id", | ||
| 320 | + "message": "source memory is missing metadata.scope_id", | ||
| 321 | + "source_id": "source-<hash>", | ||
| 322 | + "scope_id": "", | ||
| 323 | + "episode_id": "", | ||
| 324 | + "window_id": "", | ||
| 325 | + "details": {} | ||
| 326 | +} | ||
| 327 | +``` | ||
| 328 | + | ||
| 329 | +这些 issue 会进入本次 `PipelineRun.rejected` 并增加 MCP 响应的 `rejected_count`。在线 MCP | ||
| 330 | +服务不持久化 issue 明细,响应和普通日志也不包含明细;请求结束后明细随 PipelineRun 释放。 | ||
| 331 | +只有离线调用 `write_pipeline_artifacts` 时,才会写入 `rejected_extractions.jsonl`。 | ||
| 332 | + | ||
| 333 | +正常 MCP 请求中的空 ID、空 text 等通常已被前面的 `request_validate` 拒绝,因此线上频繁出现 | ||
| 334 | +Normalize issue 更可能表示内部 prepared 转换或其他 Pipeline 调用方存在问题。 | ||
| 335 | + | ||
| 336 | +**输出:** | ||
| 337 | + | ||
| 338 | +```json | ||
| 339 | +[ | ||
| 340 | + {"id":"source-<hash-m1>","scope_id":"<scope>","text":"你平时喝什么?","candidate_eligible":false,"role":"assistant","speaker":"assistant","timestamp":"","session_id":"session-<hash>","turn_index":0,"source_index":0,"metadata":{"scope_id":"<scope>","session_id":"session-<hash>","role":"assistant","speaker":"assistant","timestamp":"","turn_index":0,"source_agent_id":"xiaoo","memory_candidate":false}}, | ||
| 341 | + {"id":"source-<hash-m2>","scope_id":"<scope>","text":"我喜欢喝绿茶。","candidate_eligible":true,"role":"user","speaker":"Alice","timestamp":"2026-08-17T10:00:00Z","session_id":"session-<hash>","turn_index":1,"source_index":1,"metadata":{"scope_id":"<scope>","session_id":"session-<hash>","role":"user","speaker":"Alice","timestamp":"2026-08-17T10:00:00Z","turn_index":1,"source_agent_id":"xiaoo","memory_candidate":true}} | ||
| 342 | +] | ||
| 343 | +``` | ||
| 344 | + | ||
| 345 | +该列表是 Episode 的直接输入;实现中同时建立 `id -> NormalizedMessage` lookup,供 Window、 | ||
| 346 | +Extract、Validate、Ground 和 Aggregate 使用。 | ||
| 347 | + | ||
| 348 | +### 2. Episode | ||
| 349 | + | ||
| 350 | +**输入:** ordered normalized messages + EpisodeConfig。它按相邻消息检查 `scope_id`、 | ||
| 351 | +`session_id`、配置的 metadata 边界和可选时间间隔。当前 MCP 默认情况下,同一请求的消息具有 | ||
| 352 | +相同 scope/session,因此通常形成一个 Episode。 | ||
| 353 | + | ||
| 354 | +`ordered` 表示保持 MCP 请求中的消息顺序,不按照 timestamp 重新排序。EpisodeConfig 是本地 | ||
| 355 | +分组规则,当前 MCP 服务使用以下代码默认值,尚未对外暴露这些字段: | ||
| 356 | + | ||
| 357 | +```json | ||
| 358 | +{ | ||
| 359 | + "max_time_gap_minutes": null, | ||
| 360 | + "metadata_boundary_fields": [], | ||
| 361 | + "version": "episode_v1" | ||
| 362 | +} | ||
| 363 | +``` | ||
| 364 | + | ||
| 365 | +相邻消息的 scope 或 session 变化时始终切分;配置 `metadata_boundary_fields` 后,对应 metadata | ||
| 366 | +值发生变化也会切分;配置非负 `max_time_gap_minutes` 后,可解析时间的相邻消息超过该间隔也会 | ||
| 367 | +切分。Episode 阶段的结果是对有序消息做连续分区,不会把相隔较远但 scope/session 相同的消息 | ||
| 368 | +重新拼接到旧 Episode。 | ||
| 369 | + | ||
| 370 | +**输出:** | ||
| 371 | + | ||
| 372 | +```json | ||
| 373 | +{ | ||
| 374 | + "id": "episode-<hash>", | ||
| 375 | + "scope_id": "<scope>", | ||
| 376 | + "session_id": "session-<hash>", | ||
| 377 | + "message_ids": ["source-<hash-m1>", "source-<hash-m2>"], | ||
| 378 | + "start_time": "", | ||
| 379 | + "end_time": "2026-08-17T10:00:00Z", | ||
| 380 | + "boundary_reason": "start", | ||
| 381 | + "episode_version": "episode_v1" | ||
| 382 | +} | ||
| 383 | +``` | ||
| 384 | + | ||
| 385 | +Episode 不复制消息正文,只保存消息 ID 和边界信息。Episode 列表与 message lookup 一起成为 | ||
| 386 | +Window 的输入。 | ||
| 387 | + | ||
| 388 | +### 3. Window | ||
| 389 | + | ||
| 390 | +**输入:** episodes、message lookup 和 WindowConfig。它只把 `candidate_eligible=true` 的 | ||
| 391 | +span 放入 `candidate_refs`;`false` 消息最多只能进入 context。长消息先按句子和字符 span | ||
| 392 | +切分,再按候选 Token 预算打包,并在总预算内补充前后文。 | ||
| 393 | + | ||
| 394 | +`candidate_eligible` 不是模型判断结果。在线 MCP 中内部值来自: | ||
| 395 | + | ||
| 396 | +```text | ||
| 397 | +candidate_eligible = 请求 message.candidate | ||
| 398 | + && 本次幂等预占要求处理该 message ID | ||
| 399 | +``` | ||
| 400 | + | ||
| 401 | +首次摄入时,`candidate=true` 的消息通常为 eligible;`candidate=false` 的消息只能提供上下文; | ||
| 402 | +字段未提供时默认 true。混合重试中,已经 success 的消息不会再次抽取,但仍可以作为上下文。 | ||
| 403 | +离线 prepared 数据缺少 `memory_candidate` 时,为保持旧数据兼容,Normalize 默认按 true 处理。 | ||
| 404 | + | ||
| 405 | +span 是一条消息内的连续 Unicode 字符区间,使用左闭右开 `[start_char, end_char)` 表示。例如 | ||
| 406 | +`start_char=1,end_char=6,text="喜欢喝绿茶"` 表示该片段覆盖第 1 到第 5 个字符。消息超过候选 | ||
| 407 | +Token 预算时,会优先按句子边界切成多个 span;无法按句子容纳时继续按字符拆分。 | ||
| 408 | + | ||
| 409 | +**输出:** | ||
| 410 | + | ||
| 411 | +```json | ||
| 412 | +{ | ||
| 413 | + "id": "window-<hash>", | ||
| 414 | + "scope_id": "<scope>", | ||
| 415 | + "session_id": "session-<hash>", | ||
| 416 | + "episode_id": "episode-<hash>", | ||
| 417 | + "candidate_refs": [{"message_id":"source-<hash-m2>","start_char":0,"end_char":7,"text":"我喜欢喝绿茶。"}], | ||
| 418 | + "context_before_refs": [{"message_id":"source-<hash-m1>","start_char":0,"end_char":7,"text":"你平时喝什么?"}], | ||
| 419 | + "context_after_refs": [], | ||
| 420 | + "candidate_message_ids": ["source-<hash-m2>"], | ||
| 421 | + "candidate_token_count": 7, | ||
| 422 | + "total_token_count": 14, | ||
| 423 | + "window_version": "window_v1" | ||
| 424 | +} | ||
| 425 | +``` | ||
| 426 | + | ||
| 427 | +当前启发式估算器把每个 CJK 字符和标点分别计为一个 Token。每个 Window 分别进入 Extract。 | ||
| 428 | +没有候选 span 时输出空 Window 列表,后续 Extract、Validate、Ground 不执行,Aggregate | ||
| 429 | +产生空结果。 | ||
| 430 | + | ||
| 431 | +Episode 和 Window 的基数关系为: | ||
| 432 | + | ||
| 433 | +```text | ||
| 434 | +Episode 1 -> Window 1, Window 2 | ||
| 435 | +Episode 2 -> Window 3 | ||
| 436 | +``` | ||
| 437 | + | ||
| 438 | +一个 Window 只属于一个 Episode,不能由多个 Episode 组成,也不会跨越 Episode 边界。一个 | ||
| 439 | +Episode 可以生成零个、一个或多个 Window;每个 Window 包含一组 candidate spans,并可附带 | ||
| 440 | +同一 Episode 内的前后 context spans。 | ||
| 441 | + | ||
| 442 | +### 4. Extract | ||
| 443 | + | ||
| 444 | +**输入:** 单个 ExtractionWindow,以及 Window 引用的 NormalizedMessage。模型 Prompt 使用 | ||
| 445 | +`<context>` 和 `<candidate>` 分区;上下文只能帮助消歧,不能单独产生记忆。 | ||
| 446 | + | ||
| 447 | +**模型:** OpenAI-compatible Chat Completions,模型名为 `providers.extractor_model`,当前 | ||
| 448 | +实现标识为 `LLMMemoryExtractor`,Prompt 版本 `extract_v2`,最大输出 1600 Token。 | ||
| 449 | + | ||
| 450 | +**模型输出经协议解析后的 ExtractionBatch:** | ||
| 451 | + | ||
| 452 | +```json | ||
| 453 | +{ | ||
| 454 | + "window_id": "window-<hash>", | ||
| 455 | + "schema_version": "atomic_memory_v1", | ||
| 456 | + "raw_memories": [{ | ||
| 457 | + "text": "Alice 喜欢喝绿茶。", | ||
| 458 | + "memory_type": "preference", | ||
| 459 | + "subject": {"name": "Alice", "source_speaker": "Alice"}, | ||
| 460 | + "predicate": "prefers", | ||
| 461 | + "object": {"name": "绿茶", "type": "drink"}, | ||
| 462 | + "modality": "asserted", | ||
| 463 | + "event_time": null, | ||
| 464 | + "attributes": {}, | ||
| 465 | + "evidence": [{"message_id":"source-<hash-m2>","quote":"喜欢喝绿茶","evidence_role":"primary"}], | ||
| 466 | + "model_confidence": 0.95 | ||
| 467 | + }], | ||
| 468 | + "usage": {"prompt_tokens":0,"completion_tokens":0,"total_tokens":0,"latency_ms":0}, | ||
| 469 | + "raw_response": "<provider response>" | ||
| 470 | +} | ||
| 471 | +``` | ||
| 472 | + | ||
| 473 | +`raw_memories` 是 Validate 的输入。此处只验证模型响应是 JSON 对象、schema version 正确且 | ||
| 474 | +`memories` 是对象数组;字段语义在下一阶段检查。 | ||
| 475 | + | ||
| 476 | +### 5. Validate | ||
| 477 | + | ||
| 478 | +**输入:** `raw_memories + 当前 window + message lookup + max_memory_chars`。 | ||
| 479 | + | ||
| 480 | +**实际检查:** 必需字段及 JSON 类型;memory type/modality 枚举;confidence 为 0..=1; | ||
| 481 | +Evidence message 必须在当前 Window;quote 必须在对应 span 中唯一且逐字符匹配;至少一条 | ||
| 482 | +primary Evidence 来自 candidate span;记忆字符数不超限;`asserted` 与计划、可能、否定 | ||
| 483 | +Evidence 不冲突。 | ||
| 484 | + | ||
| 485 | +原子记忆字段的当前结构规则为: | ||
| 486 | + | ||
| 487 | +| 字段 | 规则 | | ||
| 488 | +| --- | --- | | ||
| 489 | +| `text`、`predicate` | 必需的非空字符串 | | ||
| 490 | +| `subject` | 必需的 JSON 对象 | | ||
| 491 | +| `object` | 可选;只能是 object、string 或 null | | ||
| 492 | +| `memory_type`、`modality` | 必需字符串,并且属于允许枚举 | | ||
| 493 | +| `evidence` | 必需的非空对象数组 | | ||
| 494 | +| `attributes` | 可选对象,缺少时按空对象处理 | | ||
| 495 | +| `event_time` | 可选;只能是对象或 null | | ||
| 496 | +| `model_confidence` | 可选;提供时必须为 0..=1 的数值 | | ||
| 497 | + | ||
| 498 | +`model_confidence` 当前只做范围检查,没有“低于某个阈值自动隔离”的规则,也不替代 Ground。 | ||
| 499 | +memory type 和 modality 只能验证“值是否属于枚举”,不能通过确定性测试证明模型的语义选择 | ||
| 500 | +符合人的主观预期;这部分质量需要标注数据集和评测指标支撑。 | ||
| 501 | + | ||
| 502 | +每条 Evidence 必须包含 `message_id`、非空 `quote` 和 `primary/supporting` evidence role。 | ||
| 503 | +`message_id` 必须存在于当前 Window;quote 必须在该消息被 Window 覆盖的 spans 中逐 Unicode | ||
| 504 | +字符匹配。没有匹配表示模型改写或杜撰了引文;匹配多次表示无法唯一确定字符位置。至少一条 | ||
| 505 | +primary Evidence 必须来自 candidate span,context span 只能用于消歧或 supporting Evidence。 | ||
| 506 | + | ||
| 507 | +**输出:** | ||
| 508 | + | ||
| 509 | +```text | ||
| 510 | +ValidationBatch { | ||
| 511 | + valid: Vec<AtomicMemory>, | ||
| 512 | + rejected: Vec<PipelineIssue>, | ||
| 513 | + quarantined: Vec<PipelineIssue> | ||
| 514 | +} | ||
| 515 | +``` | ||
| 516 | + | ||
| 517 | +未知枚举、字段类型错误、缺少 candidate Evidence 等进入 rejected;Evidence 不精确、超长或 | ||
| 518 | +语气可疑进入 quarantine。只有 `valid` 是 Ground 的输入。该阶段不判断模型选择的记忆类型 | ||
| 519 | +是否“符合人的主观预期”,只验证值属于枚举和结构规则。 | ||
| 520 | + | ||
| 521 | +在线 MCP 响应只返回 `rejected_count` 和 `quarantined_count`。当前没有持久化 quarantine 队列、 | ||
| 522 | +人工审核或重新放行接口,因此这里的 quarantine 表示“本次请求中隔离且不写入正式记忆”, | ||
| 523 | +不是可供后续审核的长期存储。 | ||
| 524 | + | ||
| 525 | +### 6. Ground | ||
| 526 | + | ||
| 527 | +**输入:** 当前 Window、Validate 通过的 AtomicMemory,以及各 Evidence 对应的 | ||
| 528 | +NormalizedMessage。Prompt 只发送候选 claim 和已定位 Evidence,不重新发送任意历史数据。 | ||
| 529 | + | ||
| 530 | +**模型:** OpenAI-compatible Chat Completions,模型名为 `providers.verifier_model`,当前 | ||
| 531 | +实现标识为 `LLMGroundingVerifier`,Prompt 版本 `ground_v1`,最大输出 1000 Token。 | ||
| 532 | + | ||
| 533 | +**输出:** | ||
| 534 | + | ||
| 535 | +```json | ||
| 536 | +{ | ||
| 537 | + "window_id": "window-<hash>", | ||
| 538 | + "results": [{"memory_id":"candidate-<hash>","status":"SUPPORTED","reason":"证据完整支持"}], | ||
| 539 | + "usage": {"prompt_tokens":0,"completion_tokens":0,"total_tokens":0,"latency_ms":0}, | ||
| 540 | + "raw_response": "<provider response>" | ||
| 541 | +} | ||
| 542 | +``` | ||
| 543 | + | ||
| 544 | +只有 `SUPPORTED` 进入 Aggregate;`PARTIALLY_SUPPORTED`、`UNSUPPORTED`、`UNCERTAIN` 形成 | ||
| 545 | +quarantine issue。模型漏掉某个 memory ID 时,该项按 `UNCERTAIN` 处理。 | ||
| 546 | + | ||
| 547 | +这些值是 Ground 模型给出的证据支持状态,不是服务错误码: | ||
| 548 | + | ||
| 549 | +| 状态 | 判定语义 | 示例 | | ||
| 550 | +| --- | --- | --- | | ||
| 551 | +| `SUPPORTED` | Evidence 支持记忆中的所有关键内容 | Evidence 为“我喜欢绿茶”,记忆为“Alice 喜欢绿茶” | | ||
| 552 | +| `PARTIALLY_SUPPORTED` | 只支持部分关键内容 | Evidence 只支持喜欢绿茶,记忆还声称“每天喝三杯” | | ||
| 553 | +| `UNSUPPORTED` | Evidence 不支持或与记忆矛盾 | Evidence 为“我不喜欢绿茶”,记忆为“Alice 喜欢绿茶” | | ||
| 554 | +| `UNCERTAIN` | Evidence 不足、含义模糊或模型漏答,不能可靠判断 | Evidence 为“这个还行”,记忆声称“Alice 最喜欢绿茶” | | ||
| 555 | + | ||
| 556 | +Verifier 的 Prompt 要求逐条分类;本地代码验证状态枚举,并把模型漏掉的 ID 补为 | ||
| 557 | +`UNCERTAIN`。自动化测试可以验证四种状态对应的程序分流,但实际语义判断质量仍依赖 | ||
| 558 | +`providers.verifier_model` 和评测数据。 | ||
| 559 | + | ||
| 560 | +### 7. Aggregate | ||
| 561 | + | ||
| 562 | +**输入:** 所有 Window 中 Ground 为 `SUPPORTED` 的 AtomicMemory,以及 source message | ||
| 563 | +lookup。它补充 observation,按 `scope_id + canonical_content` 精确去重,合并 Evidence 和 | ||
| 564 | +observation,并生成稳定的 `mem-<hash>` ID。 | ||
| 565 | + | ||
| 566 | +`canonical_content` 是参与精确去重的核心字段集合: | ||
| 567 | + | ||
| 568 | +```json | ||
| 569 | +{ | ||
| 570 | + "memory_type": "preference", | ||
| 571 | + "text": "Alice 喜欢绿茶。", | ||
| 572 | + "subject": {"name": "Alice"}, | ||
| 573 | + "predicate": "prefers", | ||
| 574 | + "object": {"name": "绿茶", "type": "drink"}, | ||
| 575 | + "modality": "asserted", | ||
| 576 | + "event_time": null, | ||
| 577 | + "attributes": {} | ||
| 578 | +} | ||
| 579 | +``` | ||
| 580 | + | ||
| 581 | +去重键还包含 `scope_id`,但不包含 memory ID、Evidence、observed time、来源 Episode/Window、 | ||
| 582 | +model confidence 和 observation refs。因此同一 scope 多次得到完全相同核心内容时会合并来源; | ||
| 583 | +不同用户不会合并。这是结构和字符串级精确去重,不是语义去重,“喜欢绿茶”和“爱喝绿茶” | ||
| 584 | +仍可能形成两条记忆。 | ||
| 585 | + | ||
| 586 | +**输出:** 去重后的 accepted memories,以及新的 `benchmark-prepared-v1` 输出。服务随后把 | ||
| 587 | +accepted memories 转成 `AddMemoryRequest`,调用 Embedding Provider 并写入 SQLite;这一步已在 | ||
| 588 | +七阶段之外。 | ||
| 589 | + | ||
| 590 | +Aggregate 当前没有模型调用,也没有普通 MCP 输入可稳定触发的独立业务失败条件。它的测试 | ||
| 591 | +重点是输出构造、稳定 ID、Evidence 合并和精确去重,而不是人为制造模型故障。 | ||
| 592 | + | ||
| 593 | +## 七阶段日志契约 | ||
| 594 | + | ||
| 595 | +管线阶段统一使用以下事件和规范阶段名: | ||
| 596 | + | ||
| 597 | +```text | ||
| 598 | +event=ram_a.memory.ingest.stage.started stage=<stage> | ||
| 599 | +event=ram_a.memory.ingest.stage.completed stage=<stage> | ||
| 600 | +event=ram_a.memory.ingest.stage.failed stage=<stage> | ||
| 601 | +``` | ||
| 602 | + | ||
| 603 | +阶段名为 `normalize`、`episode`、`window`、`extract`、`validate`、`ground`、`aggregate`。 | ||
| 604 | +日志记录计数、耗时、cache hit、错误码及是否可重试,不记录消息、Evidence、模型原始响应或 | ||
| 605 | +记忆正文。MCP 参数校验使用独立的 `request_validate` 阶段,避免与七阶段中的 Validate 混淆。 | ||
| 606 | + | ||
| 607 | +自动化测试覆盖:成功路径七阶段 started/completed 的完整顺序;Normalize、Episode、Window、 | ||
| 608 | +Extract、Validate 配置和 Ground 的可触发失败事件;日志不泄露测试消息正文。Aggregate 当前 | ||
| 609 | +没有可达的独立失败输入,因此覆盖 started/completed,不伪造不存在的外部故障。 | ||
| 610 | + | ||
| 611 | +## `memory_search` 完整服务链 | ||
| 612 | + | ||
| 613 | +检索没有与摄入相同的固定七阶段。它根据 `retrieval.mode`、Graph 和 Rerank 配置选择执行路径, | ||
| 614 | +默认 Hybrid 路径为: | ||
| 615 | + | ||
| 616 | +```text | ||
| 617 | +HTTP/MCP 解析 | ||
| 618 | + -> Bearer Token 认证和 memory:read 鉴权 | ||
| 619 | + -> SearchRequest validate | ||
| 620 | + -> scope_id 过滤和候选池计算 | ||
| 621 | + -> Query Embedding | ||
| 622 | + -> Dense retrieve + BM25 retrieve | ||
| 623 | + -> Hybrid fuse | ||
| 624 | + -> 可选 Graph augment | ||
| 625 | + -> 可选 Rerank | ||
| 626 | + -> memory type / event time 后过滤 | ||
| 627 | + -> truncate(top_k) | ||
| 628 | + -> MCP 响应 | ||
| 629 | +``` | ||
| 630 | + | ||
| 631 | +检索不调用 Extract 或 Ground,不创建幂等记录,不获取 ingest lock,也不修改正式记忆。 | ||
| 632 | + | ||
| 633 | +### 检索输入 | ||
| 634 | + | ||
| 635 | +`memory_search` 的 Tool arguments 为: | ||
| 636 | + | ||
| 637 | +```json | ||
| 638 | +{ | ||
| 639 | + "query": "Alice 喜欢喝什么?", | ||
| 640 | + "top_k": 10, | ||
| 641 | + "memory_types": ["preference"], | ||
| 642 | + "event_time_from": "2026-01-01T00:00:00Z", | ||
| 643 | + "event_time_to": "2026-12-31T23:59:59Z" | ||
| 644 | +} | ||
| 645 | +``` | ||
| 646 | + | ||
| 647 | +参数规则:query 去除空白后非空且最多 32000 个 Unicode 字符;top_k 默认 10、范围 | ||
| 648 | +1..=100;memory_types 为空表示不限制,否则只能使用七种记忆类型;时间边界可选,提供时必须 | ||
| 649 | +为 RFC3339;请求不接受未定义字段。失败返回 `INVALID_REQUEST`,不进入核心检索。 | ||
| 650 | + | ||
| 651 | +### 检索阶段输入输出 | ||
| 652 | + | ||
| 653 | +| 阶段 | 输入 | 输出 | 模型或实现 | | ||
| 654 | +| --- | --- | --- | --- | | ||
| 655 | +| MCP 解析 | JSON-RPC `tools/call` | `SearchRequest` | 本地协议层 | | ||
| 656 | +| 认证鉴权 | Bearer Token、可选 `x-agent-id` | Principal | 本地认证 | | ||
| 657 | +| validate | SearchRequest | 规范化后的过滤条件;非法请求直接失败 | 本地 Rust | | ||
| 658 | +| scope filter | Principal | `{"scope_id":"<hash>"}` | 本地派生,调用者不能覆盖 | | ||
| 659 | +| candidate limit | final top_k | 服务层后过滤候选上限 | 本地 Rust | | ||
| 660 | +| query embedding | query text | 固定维度查询向量 | Hash 或 `providers.embedding_model` | | ||
| 661 | +| dense retrieve | query vector、scope filter、channel limit | `Vec<ScoredMemory>` | SQLite vector cosine | | ||
| 662 | +| BM25 retrieve | 清理后的 query、scope filter、channel limit | `Vec<ScoredMemory>` | SQLite FTS5 BM25 | | ||
| 663 | +| hybrid fuse | Dense/BM25 候选和权重 | 合并、归一化、排序后的候选 | 本地 Rust | | ||
| 664 | +| graph augment | query、scope、Hybrid 候选 | 补充图事实或可选图候选 | SQLite graph;检索时不调用 LLM | | ||
| 665 | +| rerank | query、候选正文 | Rerank 新顺序和分数 | 可选 `retrieval.rerank.model` | | ||
| 666 | +| predicate filter | 候选、memory types、event time 范围 | 过滤后的候选 | 本地 Rust | | ||
| 667 | +| response mapping | 最多 top_k 条候选 | `SearchResponse.memories` | 本地 Rust | | ||
| 668 | + | ||
| 669 | +### 检索模式 | ||
| 670 | + | ||
| 671 | +| `retrieval.mode` | 执行路径 | | ||
| 672 | +| --- | --- | | ||
| 673 | +| `dense` | Query Embedding -> Dense -> 可选 Graph -> 后过滤 | | ||
| 674 | +| `bm25` | BM25 -> 可选 Graph -> 后过滤,不生成 Query Embedding | | ||
| 675 | +| `hybrid` | Query Embedding -> Dense + BM25 -> Hybrid -> 可选 Graph -> 可选 Rerank -> 后过滤 | | ||
| 676 | + | ||
| 677 | +MCP 服务不允许把 mode 设置为独立的 `graph`;Graph 是 Dense、BM25 或 Hybrid 的可选增强通道。 | ||
| 678 | + | ||
| 679 | +### 候选数量 | ||
| 680 | + | ||
| 681 | +服务为后过滤预取: | ||
| 682 | + | ||
| 683 | +```text | ||
| 684 | +service_candidate_limit = min(final_top_k * 5, 500) | ||
| 685 | +``` | ||
| 686 | + | ||
| 687 | +核心 Hybrid 中每个 Dense/BM25 通道还使用 `retrieval.candidate_k`。当前完整示例显式配置为 | ||
| 688 | +100,因此 `top_k=10` 时: | ||
| 689 | + | ||
| 690 | +```text | ||
| 691 | +Dense 最多取 100 | ||
| 692 | +BM25 最多取 100 | ||
| 693 | +Hybrid/Rerank 向服务返回最多 50 | ||
| 694 | +后过滤后最终返回最多 10 | ||
| 695 | +``` | ||
| 696 | + | ||
| 697 | +当 `candidate_k=null` 时,核心使用 `max(core_request_top_k * 5, 100)` 的动态规则。这里的 | ||
| 698 | +`core_request_top_k` 已是服务层 candidate limit,因此底层单通道查询可能超过 500;最终返回 | ||
| 699 | +服务后过滤的候选仍被 `service_candidate_limit` 限制。这是当前实现的两层候选策略,不能把 | ||
| 700 | +`top_k * 5` 理解为数据库每个通道的固定扫描上限。 | ||
| 701 | + | ||
| 702 | +### Dense 输入输出 | ||
| 703 | + | ||
| 704 | +Query Embedding 使用 `providers.embedding_provider/model/dimensions`。`hash` 在本地计算; | ||
| 705 | +`openai_compatible` 调用配置的 Embedding API。Dense 输入是 query vector 和当前 scope,输出为: | ||
| 706 | + | ||
| 707 | +```json | ||
| 708 | +{ | ||
| 709 | + "record": {"id":"mem-1","text":"Alice 喜欢绿茶。","metadata":{}}, | ||
| 710 | + "score": 0.82 | ||
| 711 | +} | ||
| 712 | +``` | ||
| 713 | + | ||
| 714 | +SQLite 使用 cosine distance,并转换为 `dense_score = 1 - distance`,分数越高越相关。查询向量 | ||
| 715 | +维度或 embedding profile 与该 scope 已存记忆不一致时失败,不混用不同向量空间。 | ||
| 716 | + | ||
| 717 | +### BM25 输入输出 | ||
| 718 | + | ||
| 719 | +BM25 先把 query 按非字母数字字符拆成 FTS token,空 token 查询返回空候选。SQLite BM25 原始 | ||
| 720 | +值越低越相关,代码直接转换为 0..=1 的高分优先值: | ||
| 721 | + | ||
| 722 | +```text | ||
| 723 | +bm25_score = (max_raw - current_raw) / (max_raw - min_raw) | ||
| 724 | +``` | ||
| 725 | + | ||
| 726 | +这与“先取反再做 min-max”数学等价;所有原始值相同时统一得到 1.0。BM25 不调用模型。 | ||
| 727 | + | ||
| 728 | +### Hybrid 输入输出 | ||
| 729 | + | ||
| 730 | +Hybrid 按 memory ID 合并两个通道,并分别对当前候选集中的已有分数做 min-max: | ||
| 731 | + | ||
| 732 | +```text | ||
| 733 | +dense_norm = min_max(dense_score) | ||
| 734 | +bm25_norm = min_max(bm25_score) | ||
| 735 | +hybrid_score = embedding_weight * dense_norm + bm25_weight * bm25_norm | ||
| 736 | +``` | ||
| 737 | + | ||
| 738 | +默认权重为 0.7/0.3。某条记忆没有出现在某个通道时,该通道分数按 0;同分时按 memory ID | ||
| 739 | +稳定排序。当前代码不会在整个通道为空时重新归一化权重,例如 BM25 全空时 Dense 排序不变, | ||
| 740 | +但最终分数仍乘以 0.7。因此“空通道权重自动变为 0”不是当前实现的精确描述。 | ||
| 741 | + | ||
| 742 | +### Graph 输入输出 | ||
| 743 | + | ||
| 744 | +Graph 开启时使用 query 和当前 scope 从已经持久化的图事实、实体和 Evidence 记录中检索。搜索 | ||
| 745 | +阶段不调用图 LLM;图结构已在摄入的 `graph_build` 中生成。默认 | ||
| 746 | +`rerank_with_graph=false` 时,Graph 主要给基础候选附加 `graph_facts/graph_matches`,不改变基础 | ||
| 747 | +分数。允许图参与排序且 `allow_graph_only=true` 时,才可能加入不在基础候选中的纯图结果。 | ||
| 748 | + | ||
| 749 | +Graph augment 位于 Rerank 之前。Graph 检索失败时,`graph_memory.retrieval.fail_open=true` 返回 | ||
| 750 | +基础候选;false 时检索失败。 | ||
| 751 | + | ||
| 752 | +### Rerank 输入输出 | ||
| 753 | + | ||
| 754 | +Rerank 只允许在 Hybrid mode 开启。输入为 query 和候选的 ID/text,Provider 输出相关性分数, | ||
| 755 | +代码据此重排并截断: | ||
| 756 | + | ||
| 757 | +```json | ||
| 758 | +{ | ||
| 759 | + "results": [ | ||
| 760 | + {"index": 1, "relevance_score": 0.95}, | ||
| 761 | + {"index": 0, "relevance_score": 0.61} | ||
| 762 | + ] | ||
| 763 | +} | ||
| 764 | +``` | ||
| 765 | + | ||
| 766 | +当前 `input_k` 不是固定上限,核心使用 `max(input_k, core_request_top_k)`。例如最终 top_k=10、 | ||
| 767 | +服务候选池=50、input_k=40 时,最多会把 50 条而不是固定 40 条送入 Rerank。 | ||
| 768 | + | ||
| 769 | +`fail_open=false` 时 Provider 超时、可重试 HTTP 错误耗尽重试或协议错误返回 | ||
| 770 | +`RERANK_FAILED`;`fail_open=true` 时记录 degraded 日志并返回 Rerank 前的 Hybrid/Graph 顺序。 | ||
| 771 | + | ||
| 772 | +### 后过滤与最终输出 | ||
| 773 | + | ||
| 774 | +`memory_types` 和 event time 在 Dense/BM25/Graph/Rerank 之后过滤。时间过滤依次尝试读取 | ||
| 775 | +event_time 字符串、`event_time.normalized`、`event_time.raw`;配置时间范围后,没有可解析 | ||
| 776 | +event time 的记忆会被排除。因为过滤发生在候选召回之后,最终结果可以少于 top_k,服务不会 | ||
| 777 | +继续扩大候选池补足数量。 | ||
| 778 | + | ||
| 779 | +成功响应为: | ||
| 780 | + | ||
| 781 | +```json | ||
| 782 | +{ | ||
| 783 | + "memories": [ | ||
| 784 | + { | ||
| 785 | + "id": "mem-<hash>", | ||
| 786 | + "text": "Alice 喜欢绿茶。", | ||
| 787 | + "memory_type": "preference", | ||
| 788 | + "modality": "asserted", | ||
| 789 | + "event_time": null, | ||
| 790 | + "observed_at": "2026-08-17T10:00:00Z", | ||
| 791 | + "evidence_refs": [ | ||
| 792 | + { | ||
| 793 | + "message_id": "source-<hash>", | ||
| 794 | + "quote": "喜欢喝绿茶", | ||
| 795 | + "start_char": 1, | ||
| 796 | + "end_char": 6, | ||
| 797 | + "evidence_role": "primary" | ||
| 798 | + } | ||
| 799 | + ], | ||
| 800 | + "source_agent_id": "xiaoo", | ||
| 801 | + "graph_facts": [], | ||
| 802 | + "graph_facts_truncated": false, | ||
| 803 | + "score": 0.95 | ||
| 804 | + } | ||
| 805 | + ] | ||
| 806 | +} | ||
| 807 | +``` | ||
| 808 | + | ||
| 809 | +无匹配时成功返回空数组。score 不是事实置信度:它可能是 Dense similarity、BM25 归一化值、 | ||
| 810 | +Hybrid 融合值或 Rerank relevance score,含义取决于运行配置,不能跨模式直接比较。 | ||
| 811 | + | ||
| 812 | +### 检索日志与错误 | ||
| 813 | + | ||
| 814 | +服务层记录 `validate`、`retrieve` 和 `predicate_filter`;默认 Hybrid 核心路径记录 | ||
| 815 | +`query_embedding`、`dense_retrieve`、`bm25_retrieve`、`hybrid_fuse`,并按配置增加 | ||
| 816 | +`graph_augment` 和 `rerank`。日志记录阶段、候选计数、耗时和错误分类,不记录 query 或记忆 | ||
| 817 | +正文。 | ||
| 818 | + | ||
| 819 | +| 错误 | 当前对外结果 | | ||
| 820 | +| --- | --- | | ||
| 821 | +| 请求参数非法 | `INVALID_REQUEST`,不可重试 | | ||
| 822 | +| Rerank 失败且 fail-open 关闭 | `RERANK_FAILED`,可重试 | | ||
| 823 | +| Embedding、SQLite 或 fail-closed Graph 检索失败 | 当前统一映射为 `STORAGE_FAILED`,可重试 | | ||
| 824 | + | ||
| 825 | +最后一项是当前错误映射的实际行为,不表示所有这类故障在语义上都是 SQLite 存储错误。 | ||
| @@ -0,0 +1,288 @@ | |||
| 1 | +# RAM-A-MEM 配置字段参考 | ||
| 2 | + | ||
| 3 | +本文以 `memory_mcp::config::ServerConfig` 的当前实现为准,覆盖 `ram-a-mem` 服务 JSON | ||
| 4 | +配置的全部字段,并说明代码默认值、部署推荐值、有效范围、生效条件和测试要求。完整配置示例见 | ||
| 5 | +[`plugins/mcp/ram-a-mem.json`](../../plugins/mcp/ram-a-mem.json)。 | ||
| 6 | + | ||
| 7 | +本文字段表中的“测试要求”默认指安装 RPM 后执行的黑盒验收。源码自动化用于提前发现实现 | ||
| 8 | +回归,但不能替代 RPM 中二进制、默认目录、运行用户、文件权限和依赖环境的交付验证。 | ||
| 9 | + | ||
| 10 | +## 1. 字段分类 | ||
| 11 | + | ||
| 12 | +| 类别 | 含义 | 测试要求 | | ||
| 13 | +| --- | --- | --- | | ||
| 14 | +| 固定值 | 协议、枚举或当前实现只接受列出的值,用户不能任意扩展 | 覆盖全部合法值,并验证未知值启动失败 | | ||
| 15 | +| 推荐可配置 | 允许用户覆盖;表中推荐值是已确认的交付配置 | 验证默认值、推荐值、上下边界和越界值 | | ||
| 16 | +| 部署必配 | 必须由用户按环境填写,不存在跨环境通用值 | 验证必填、非空、格式、引用关系;不编造长度或容量上限 | | ||
| 17 | +| 条件配置 | 仅在对应功能开启或相关字段非空时生效 | 同时验证未启用、正确启用和依赖缺失三种情况 | | ||
| 18 | + | ||
| 19 | +除非字段表另有说明,本文的“字符”均不是 MCP Tool 入参长度约束。模型名、文件路径、环境 | ||
| 20 | +变量名等没有可靠业务上限,服务只校验当前实现需要的非空、格式或引用关系。 | ||
| 21 | + | ||
| 22 | +## 2. 通用规则 | ||
| 23 | + | ||
| 24 | +- 配置格式固定为 JSON,所有配置对象都拒绝未知字段。 | ||
| 25 | +- 生产启动要求顶层 `auth`、`storage` 和 `providers` 可用;`auth.tokens` 至少一项。 | ||
| 26 | +- `*_env` 的值是环境变量名称,不是 Token 或 API Key 本身。密钥必须通过进程环境注入。 | ||
| 27 | +- 顶层可选对象一旦存在,服务仍会校验其字段;功能开关只决定是否构建和对外暴露该能力。 | ||
| 28 | + 因此不要在关闭功能时保留一份无效的 `case_library` 或 `graph_memory` 配置。 | ||
| 29 | +- 带 API Key 的公网 Provider URL 固定要求 HTTPS;loopback、私网和 link-local 地址允许 HTTP。 | ||
| 30 | +- URL 不允许携带用户名、密码、query 或 fragment。 | ||
| 31 | +- 配置文件查找优先级固定为:`--config`、`RAM_A_MEM_CONFIG`、 | ||
| 32 | + `config/ram-a-mem.json`、`$HOME/.config/ram-a/ram-a-mem.json`、 | ||
| 33 | + `/etc/ram-a/ram-a-mem.json`。 | ||
| 34 | +- 代码默认监听端口是 `8080`;RPM/本项目交付示例推荐使用 `18081`。两者不能混写为同一种 | ||
| 35 | + 默认值。 | ||
| 36 | + | ||
| 37 | +顶层字段如下: | ||
| 38 | + | ||
| 39 | +| 字段 | 类别 | 省略时 | 说明 | | ||
| 40 | +| --- | --- | --- | --- | | ||
| 41 | +| `auth` | 部署必配 | 不允许省略 | Bearer Token 与主体映射 | | ||
| 42 | +| `features` | 推荐可配置 | 使用各功能默认值 | 对外能力开关 | | ||
| 43 | +| `http` | 推荐可配置 | 使用本地监听默认值 | HTTP 监听和来源限制 | | ||
| 44 | +| `limits` | 推荐可配置 | 使用服务保护默认值 | 请求、并发和 Session 限制 | | ||
| 45 | +| `pipeline` | 推荐可配置 | `fail_fast=true`、`max_memory_chars=500` | 记忆管线策略 | | ||
| 46 | +| `storage` | 部署必配 | 生产校验失败 | 个人记忆 SQLite 文件 | | ||
| 47 | +| `providers` | 部署必配 | 生产校验失败 | Extract、Ground 和 Embedding Provider | | ||
| 48 | +| `retrieval` | 推荐可配置 | Hybrid,不启用 Rerank | 个人记忆检索策略 | | ||
| 49 | +| `case_library` | 条件配置 | `null` | 案例库配置 | | ||
| 50 | +| `graph_memory` | 条件配置 | `null` | 图记忆配置 | | ||
| 51 | + | ||
| 52 | +## 3. `auth` | ||
| 53 | + | ||
| 54 | +`auth.tokens[]` 每项创建一个 `Principal=(tenant_id,user_id,agent_id,permissions)`。 | ||
| 55 | +`scope_id` 固定由 `tenant_id + user_id` 派生;修改任一字段会切换数据隔离范围。 | ||
| 56 | + | ||
| 57 | +| 字段 | 类别 | 默认/推荐 | 约束与生效条件 | 测试要求 | | ||
| 58 | +| --- | --- | --- | --- | --- | | ||
| 59 | +| `tokens` | 部署必配 | 至少配置一个 | 非空数组 | 空数组启动失败;多 Token 可启动 | | ||
| 60 | +| `tokens[].token_env` | 部署必配 | 使用能说明客户端的环境变量名,例如 `RAM_A_XIAOO_TOKEN` | 非空、无前后空格;数组内唯一;环境变量必须存在且值非空 | 缺失、空值、重复环境名及重复实际 Token 均失败;配置和日志不得出现密钥值 | | ||
| 61 | +| `tokens[].tenant_id` | 部署必配 | 使用稳定租户标识 | 非空、无前后空格;参与 `scope_id` | 相同/不同租户的 scope 隔离 | | ||
| 62 | +| `tokens[].user_id` | 部署必配 | 使用稳定用户标识 | 非空、无前后空格;参与 `scope_id` | 相同/不同用户的 scope 隔离 | | ||
| 63 | +| `tokens[].agent_id` | 部署必配 | 使用调用 Agent 的稳定标识 | 非空、无前后空格;`x-agent-id` 提供时必须一致 | Header 缺省、匹配和不匹配 | | ||
| 64 | +| `tokens[].permissions` | 固定值集合 | 按最小权限配置 | 元素只能是 `memory:read`、`memory:write`、`cases:read`、`cases:write`,同一 Token 内不得重复;空数组表示只能认证、不能调用受保护工具 | 四种合法值、未知值、重复值和权限不足 | | ||
| 65 | + | ||
| 66 | +Token 本身没有由 RAM-A 管理的过期时间。轮换方式是更新环境变量或增加新 Token 配置并重启 | ||
| 67 | +服务;生命周期由部署系统负责。 | ||
| 68 | + | ||
| 69 | +## 4. `features` | ||
| 70 | + | ||
| 71 | +| 字段 | 类别 | 代码默认 | 推荐值 | 约束与生效条件 | | ||
| 72 | +| --- | --- | ---: | ---: | --- | | ||
| 73 | +| `memory.enabled` | 推荐可配置 | `true` | `true` | 控制 `memory_ingest`、`memory_search`;图记忆开启时必须为 `true` | | ||
| 74 | +| `case_library.enabled` | 条件配置 | `null` | 明确写 `true` 或 `false` | `null` 时由顶层 `case_library` 是否存在决定;`true` 时必须提供 `case_library` | | ||
| 75 | +| `graph_memory.enabled` | 条件配置 | `false` | `false` | `true` 时必须开启 memory 并提供 `graph_memory` | | ||
| 76 | + | ||
| 77 | +测试必须验证工具列表和调用行为随开关变化,而不只验证 JSON 能解析。 | ||
| 78 | + | ||
| 79 | +## 5. `http` | ||
| 80 | + | ||
| 81 | +| 字段 | 类别 | 代码默认 | 推荐值 | 约束与生效条件 | | ||
| 82 | +| --- | --- | --- | --- | --- | | ||
| 83 | +| `bind_address` | 推荐可配置 | `127.0.0.1` | 单机/反向代理部署使用 `127.0.0.1` | IP 地址;非 loopback 时触发 TLS 和 Host 额外校验 | | ||
| 84 | +| `port` | 推荐可配置 | `8080` | RPM/交付环境使用 `18081` | `u16`;应避免 `0` 和已占用端口,端口可绑定性由启动测试验证 | | ||
| 85 | +| `allowed_origins` | 部署可配置 | `[]` | 无浏览器跨域需求时保持 `[]` | 精确匹配允许的 Origin;不影响非浏览器客户端 | | ||
| 86 | +| `allowed_hosts` | 部署必配 | `localhost`、`127.0.0.1`、`::1` | 写实际访问 Host,包含端口时也应与请求一致 | 非空数组,元素非空;外部监听至少包含一个非 loopback Host | | ||
| 87 | +| `tls_termination_acknowledged` | 条件配置 | `false` | loopback 为 `false`;外部监听且已有 TLS termination 时为 `true` | 只表示部署者确认,不会让 RAM-A 自己启用 TLS | | ||
| 88 | + | ||
| 89 | +测试应覆盖本地默认配置、外部监听未确认 TLS、仅 loopback Host、外部 Host 正确配置四种情况。 | ||
| 90 | + | ||
| 91 | +## 6. `limits` | ||
| 92 | + | ||
| 93 | +以下代码默认值同时作为推荐值,配置时不能超过固定支持范围。 | ||
| 94 | + | ||
| 95 | +| 字段 | 推荐值 | 固定有效范围 | 作用 | | ||
| 96 | +| --- | ---: | ---: | --- | | ||
| 97 | +| `max_body_bytes` | 16777216 | 1..=67108864 | 单个 MCP HTTP 请求体字节上限 | | ||
| 98 | +| `requests_per_second` | 20 | 1..=10000 | 每 Principal、每 Tool 持续速率 | | ||
| 99 | +| `rate_burst` | 40 | 1..=100000 | 每 Principal、每 Tool 突发容量 | | ||
| 100 | +| `max_in_flight_per_principal_tool` | 4 | 1..=1024 | 每 Principal、每 Tool 并发上限;超限不排队 | | ||
| 101 | +| `initialize_requests_per_second` | 4 | 1..=1000 | MCP initialize 持续速率 | | ||
| 102 | +| `initialize_rate_burst` | 8 | 1..=10000 | MCP initialize 突发容量 | | ||
| 103 | +| `max_active_sessions_per_principal` | 8 | 1..=1024 | 单 Principal 活动 MCP Session 上限 | | ||
| 104 | +| `max_active_sessions_global` | 256 | 1..=100000 | 单进程活动 MCP Session 总上限 | | ||
| 105 | +| `session_idle_timeout_seconds` | 1800 | 1..=86400 | MCP Session 空闲回收时间;同一值同时配置 RAM-A Admission 和底层 `rmcp` Session Worker | | ||
| 106 | + | ||
| 107 | +`max_active_sessions_global` 固定不得小于 `max_active_sessions_per_principal`。每个字段都必须有 | ||
| 108 | +默认值、下边界、上边界、0、上边界加一的配置测试;并发、速率、请求体和 Session 回收还应有 | ||
| 109 | +HTTP 行为测试。Session 超时测试必须覆盖配置值大于 `rmcp` 历史默认 300 秒的场景,证明底层 | ||
| 110 | +Session Worker 不会先于 `session_idle_timeout_seconds` 终止;活动请求必须同时刷新两层的空闲期限。 | ||
| 111 | + | ||
| 112 | +## 7. `pipeline` | ||
| 113 | + | ||
| 114 | +| 字段 | 类别 | 代码默认/推荐 | 固定范围 | 生效条件与结果 | | ||
| 115 | +| --- | --- | --- | --- | --- | | ||
| 116 | +| `fail_fast` | 推荐可配置 | `true` | `true`/`false` | 只控制 Extract/Ground 窗口错误;`true` 终止本次摄入,`false` 跳过失败窗口并继续 | | ||
| 117 | +| `max_memory_chars` | 推荐可配置 | `500` | 1..=32000 Unicode 字符 | 抽取记忆超限进入 quarantine,不截断 | | ||
| 118 | + | ||
| 119 | +Episode 和 Window 的内部参数当前不是服务配置字段,不能写入 JSON。测试必须覆盖两种 | ||
| 120 | +`fail_fast` 行为,以及 `max_memory_chars` 的 1、500、32000、0、32001。 | ||
| 121 | + | ||
| 122 | +## 8. `storage` | ||
| 123 | + | ||
| 124 | +| 字段 | 类别 | 默认/推荐 | 约束 | 测试要求 | | ||
| 125 | +| --- | --- | --- | --- | --- | | ||
| 126 | +| `database_path` | 部署必配 | RPM 推荐 `/var/lib/ram-a/ram-a-memory.sqlite` | 非空持久化 SQLite 文件路径;生产不接受 `:memory:`;不得与 `case_library.index_store` 相同 | 使用推荐绝对路径时服务启动成功、`/ready` 返回 2xx,并生成 SQLite/WAL 文件;空路径和 `:memory:` 必须启动失败;父目录只读或运行用户无写权限时必须启动失败并输出明确日志 | | ||
| 127 | + | ||
| 128 | +`data/ram-a-memory.sqlite` 适合源码目录运行,不作为 RPM 推荐值。相对路径按进程工作目录解析; | ||
| 129 | +RPM 验收应使用绝对路径,避免启动方式改变实际落盘位置。磁盘满、只读文件系统和 SQLite 锁冲突 | ||
| 130 | +应使用已安装的 RPM 二进制做存储可靠性故障注入,并检查进程状态、MCP 错误和脱敏日志。 | ||
| 131 | + | ||
| 132 | +## 9. `providers` | ||
| 133 | + | ||
| 134 | +Extract 和 Ground 固定使用 OpenAI-compatible Chat API。Embedding 可以使用 | ||
| 135 | +OpenAI-compatible API 或本地确定性 hash。 | ||
| 136 | + | ||
| 137 | +| 字段 | 类别 | 代码默认 | 推荐值 | 约束与回退 | | ||
| 138 | +| --- | --- | --- | --- | --- | | ||
| 139 | +| `api_key_env` | 部署必配 | 无 | Chat Provider 的密钥环境变量名 | 非空且环境变量必须存在;即使 Embedding 使用 hash 也必需 | | ||
| 140 | +| `base_url` | 部署可配置 | `https://openrouter.ai/api/v1` | 使用实际 Chat Provider 的 OpenAI-compatible `/v1` 基址 | 绝对 HTTP(S) URL;带密钥的公网地址必须 HTTPS | | ||
| 141 | +| `embedding_provider` | 固定值枚举 | `openai_compatible` | 生产推荐 `openai_compatible`;离线自测可用 `hash` | 只接受 `openai_compatible`、兼容别名 `open_router`、`hash` | | ||
| 142 | +| `embedding_api_key_env` | 条件配置 | `null` | 独立 Embedding 服务才填写 | `null` 回退到 `api_key_env`;配置时非空 | | ||
| 143 | +| `embedding_base_url` | 条件配置 | `null` | 独立 Embedding 服务才填写 | `null` 回退到 `base_url`;URL 规则同上 | | ||
| 144 | +| `embedding_model` | 部署必配 | 无 | `hash` Provider 写 `hash`;外部 Provider 写实际模型名 | 非空 | | ||
| 145 | +| `embedding_dimensions` | 部署必配 | 无 | hash 自测推荐 1024;外部 Provider 必须使用模型实际输出维度 | 大于 0;已存向量与查询向量维度必须一致 | | ||
| 146 | +| `extractor_model` | 部署必配 | 无 | 使用已验证能稳定输出 JSON 的 Chat 模型 | 非空;模型是否存在由联通测试验证 | | ||
| 147 | +| `verifier_model` | 部署必配 | 无 | 推荐与 `extractor_model` 使用相同模型 | 非空;用于 Ground,也支持独立配置 | | ||
| 148 | +| `timeout_seconds` | 推荐可配置 | 120 | 120 | 大于 0;Chat 请求超时 | | ||
| 149 | +| `max_retries` | 推荐可配置 | 3 | 3 | 大于 0;仅作用于 Extract/Ground 共用的 Chat 客户端 | | ||
| 150 | + | ||
| 151 | +配置测试验证枚举、默认值、非空、URL 安全规则和正数约束。模型存在性、API Key 权限、余额、 | ||
| 152 | +限流、响应 JSON 质量必须由带真实 Provider 的集成测试验证。 | ||
| 153 | + | ||
| 154 | +## 10. `retrieval` | ||
| 155 | + | ||
| 156 | +| 字段 | 类别 | 代码默认 | 推荐值 | 约束与生效条件 | | ||
| 157 | +| --- | --- | --- | --- | --- | | ||
| 158 | +| `mode` | 固定值枚举 | `hybrid` | `hybrid` | MCP 服务只接受 `dense`、`bm25`、`hybrid`;独立 `graph` mode 启动失败 | | ||
| 159 | +| `embedding_weight` | 推荐可配置 | 0.7 | 0.7 | Hybrid 时为有限数且 0..=1 | | ||
| 160 | +| `bm25_weight` | 推荐可配置 | 0.3 | 0.3 | Hybrid 时为有限数且 0..=1;两权重之和固定为 1 | | ||
| 161 | +| `candidate_k` | 推荐可配置 | `null` | 100 | `null` 使用 `max(top_k*5,100)`;显式值为 1..=500 | | ||
| 162 | +| `rerank` | 条件配置 | 见下表 | 关闭 | 仅 Hybrid 可启用 | | ||
| 163 | + | ||
| 164 | +Hybrid 推荐使用 0.7/0.3 权重组合。Dense 或 BM25 单通道模式不使用 Hybrid 权重。 | ||
| 165 | + | ||
| 166 | +### 10.1 `retrieval.rerank` | ||
| 167 | + | ||
| 168 | +| 字段 | 类别 | 代码默认/推荐 | 约束与生效条件 | | ||
| 169 | +| --- | --- | --- | --- | | ||
| 170 | +| `enabled` | 条件配置 | `false` | `true` 时要求 `retrieval.mode=hybrid` 并构建 Rerank 客户端 | | ||
| 171 | +| `provider` | 固定值 | `openrouter` | 当前只接受 `openrouter`;可指向兼容该请求协议的自托管端点 | | ||
| 172 | +| `model` | 部署可配置 | `cohere/rerank-v3.5` | 启用时非空;需与端点实际模型一致 | | ||
| 173 | +| `api_key_env` | 条件配置 | `OPENROUTER_API_KEY` | 可设 `null` 以访问无需认证的可信本地端点;非空时环境变量必须存在 | | ||
| 174 | +| `base_url` | 部署可配置 | `https://openrouter.ai/api/v1` | 启用时必须是合法 URL;公网带密钥固定要求 HTTPS | | ||
| 175 | +| `input_k` | 推荐可配置 | 40 | 1..=500;运行时实际送入数至少为请求 `top_k` | | ||
| 176 | +| `timeout_ms` | 推荐可配置 | 30000 | 启用时必须为 1..=120000;禁用时不生效 | | ||
| 177 | +| `fail_open` | 推荐可配置 | `false` | `false`:Rerank 异常使本次 search 返回 `RERANK_FAILED`;`true`:返回 Rerank 前 Hybrid 顺序 | | ||
| 178 | + | ||
| 179 | +测试必须覆盖启用/禁用、非 Hybrid 启用失败、input/timeout 边界、无认证本地端点、公网 HTTP | ||
| 180 | +拒绝,以及 `fail_open` 两种故障结果。排序效果和稳定性不能只靠配置测试证明。 | ||
| 181 | + | ||
| 182 | +## 11. `case_library` | ||
| 183 | + | ||
| 184 | +只有案例库功能生效时,下列 Provider、导入 Worker 和 REST 管理接口才会构建。 | ||
| 185 | + | ||
| 186 | +| 字段 | 类别 | 代码默认 | 推荐值 | 约束与生效条件 | | ||
| 187 | +| --- | --- | --- | --- | --- | | ||
| 188 | +| `rag_store` | 部署可配置 | `data/memory-cases.sqlite` | 使用独立持久化文件 | 非空、非 `:memory:`,且不同于 `index_store` | | ||
| 189 | +| `index_store` | 部署可配置 | `data/memory-cases-index.sqlite` | 使用独立持久化文件 | 非空、非 `:memory:`,且不同于 `rag_store` 和个人记忆库 | | ||
| 190 | +| `source_dir` | 条件配置 | `null` | 仅需启动时自动扫描本地案例时填写 | 配置时路径非空;目录可读性由集成测试验证 | | ||
| 191 | +| `api_token_env` | 条件配置 | `null` | 不使用案例管理 REST API 时保持 `null` | 配置后启用管理 API;环境变量名及值必须非空、无前后空格 | | ||
| 192 | +| `ingestion_poll_ms` | 推荐可配置 | 1000 | 1000 | 大于 0;案例摄入 Worker 轮询间隔 | | ||
| 193 | +| `embedding_provider` | 固定值枚举 | `openai_compatible` | 生产明确配置 `openai_compatible`;离线自测明确配置 `hash` | 枚举同 `providers.embedding_provider`;不要依赖默认 Provider 与默认模型的组合 | | ||
| 194 | +| `embedding_api_key_env` | 条件配置 | `null` | 独立案例 Embedding 服务才填写 | `null` 回退到 `providers.api_key_env` | | ||
| 195 | +| `embedding_base_url` | 条件配置 | `null` | 独立案例 Embedding 服务才填写 | `null` 回退到 `providers.base_url` | | ||
| 196 | +| `embedding_model` | 推荐可配置 | `hash` | hash 自测用 `hash`;生产写实际模型 | 非空 | | ||
| 197 | +| `embedding_dimensions` | 推荐可配置 | 1024 | hash 自测 1024;生产与实际模型一致 | 大于 0 | | ||
| 198 | +| `chunk_size` | 推荐可配置 | 160 | 160 | 大于 0;案例切块大小 | | ||
| 199 | +| `summary_llm_model` | 条件配置 | `null` | 不需要模型摘要时保持 `null` | 非 `null` 时启用摘要模型 | | ||
| 200 | +| `summary_llm_api_key_env` | 条件配置 | `null` | 摘要服务使用独立密钥时填写 | `null` 回退到 `providers.api_key_env` | | ||
| 201 | +| `summary_llm_base_url` | 条件配置 | `null` | 摘要服务使用独立端点时填写 | `null` 回退到 `providers.base_url` | | ||
| 202 | +| `summary_llm_timeout_ms` | 推荐可配置 | 30000 | 30000 | 大于 0;只在摘要模型启用时实际调用 | | ||
| 203 | +| `default_library` | 部署必配 | 无 | 选择最常用逻辑库 | 非空,必须引用 `libraries[].name` | | ||
| 204 | +| `libraries` | 部署必配 | 无 | 至少一项 | 非空;`name` 唯一 | | ||
| 205 | +| `libraries[].name` | 部署必配 | 无 | 使用稳定、面向调用方的逻辑库名 | 非空、无前后空格、数组内唯一 | | ||
| 206 | +| `libraries[].dataset_id` | 部署必配 | 无 | 使用内部稳定 dataset ID | 非空、无前后空格 | | ||
| 207 | +| `libraries[].tenant_ids` | 部署必配 | 无 | 明确列出允许访问的租户 | 非空数组,元素非空且无前后空格 | | ||
| 208 | + | ||
| 209 | +配置测试覆盖默认值、持久化路径隔离、正数约束和映射关系。目录内容、文档导入、Embedding | ||
| 210 | +维度及摘要模型输出属于案例库集成测试。 | ||
| 211 | + | ||
| 212 | +## 12. `graph_memory` | ||
| 213 | + | ||
| 214 | +| 字段 | 类别 | 代码默认 | 推荐值 | 约束与生效条件 | | ||
| 215 | +| --- | --- | --- | --- | --- | | ||
| 216 | +| `llm_api_key_env` | 部署必配 | 无 | 图模型密钥环境变量名 | 非空;功能开启时环境变量必须存在 | | ||
| 217 | +| `llm_base_url` | 部署可配置 | `https://openrouter.ai/api/v1` | 使用实际 OpenAI-compatible 图模型端点 | URL 安全规则同主 Provider | | ||
| 218 | +| `llm_model` | 部署必配 | 无 | 使用支持图 schema 输出的模型 | 非空 | | ||
| 219 | +| `llm_timeout_ms` | 推荐可配置 | 60000 | 60000 | 大于 0 | | ||
| 220 | +| `build_concurrency` | 推荐可配置 | 1 | 1 | 大于 0;一次摄入内图构建并发数 | | ||
| 221 | +| `retrieval.weight` | 推荐可配置 | 0.2 | 0.2 | 有限数,0..=1 | | ||
| 222 | +| `retrieval.rerank_with_graph` | 推荐可配置 | `false` | `false` | 决定图结果是否进入 Rerank 输入 | | ||
| 223 | +| `retrieval.allow_graph_only` | 推荐可配置 | `false` | `false` | 是否允许无核心记忆支撑的纯图结果 | | ||
| 224 | +| `retrieval.max_graph_only_results` | 条件配置 | `null` | `null` | 显式配置时大于 0;限制纯图结果数 | | ||
| 225 | +| `retrieval.seed_limit` | 条件配置 | `null` | `null` | 显式配置时 1..=5000;`null` 使用核心动态规则 | | ||
| 226 | +| `retrieval.max_evidence_records_per_fact` | 条件配置 | `null` | `null` | 显式配置时 1..=100 | | ||
| 227 | +| `retrieval.fail_open` | 推荐可配置 | `false` | 图是增强通道且要求核心检索可用时推荐 `true` | `true` 时图检索失败退回非图结果;不控制图摄入失败 | | ||
| 228 | + | ||
| 229 | +测试覆盖功能依赖、默认值、weight、三个可选容量字段及 `fail_open`。图模型输出质量、图构建 | ||
| 230 | +效果和融合排序需要真实模型或确定性桩测试。 | ||
| 231 | + | ||
| 232 | +## 13. 非 JSON 启动配置 | ||
| 233 | + | ||
| 234 | +这些值在日志或配置文件加载之前生效,因此不放入 `ram-a-mem.json`。 | ||
| 235 | + | ||
| 236 | +| 环境变量 | 类别 | 默认/推荐 | 固定值或约束 | | ||
| 237 | +| --- | --- | --- | --- | | ||
| 238 | +| `RAM_A_MEM_CONFIG` | 部署可配置 | 未使用 `--config` 时指向交付配置 | 文件路径;优先级低于 CLI `--config` | | ||
| 239 | +| `RAM_A_LOG_FORMAT` | 固定值枚举 | 默认 `json`;终端调试推荐 `compact` | 只接受 `json`、`compact`,非法值启动失败 | | ||
| 240 | +| `RAM_A_LOG_SOURCE` | 固定值枚举 | 默认 `false`;现场定位推荐临时设 `true` | 只接受 `true`、`false`,非法值启动失败 | | ||
| 241 | +| `RUST_LOG` | 部署可配置 | 未设置时使用服务默认过滤级别 | tracing filter 表达式;不改变业务结果 | | ||
| 242 | +| 各 `*_env` 指向的变量 | 部署必配/条件配置 | 无 | 保存实际密钥;不得写入 JSON、日志或测试快照 | | ||
| 243 | + | ||
| 244 | +## 14. RPM 验收与源码自动化边界 | ||
| 245 | + | ||
| 246 | +配置验收以 RPM 安装后的真实二进制为对象。每个字段用例按以下方式执行: | ||
| 247 | + | ||
| 248 | +1. 从推荐配置生成一个字段变体,注入有效 Token 和 Provider 环境变量。 | ||
| 249 | +2. 启动 RPM 提供的 `ram-a-mem` 二进制;合法配置要求进程存活且 `/healthy`、`/ready` 返回 2xx。 | ||
| 250 | +3. 对功能开关、限流、Pipeline、检索和权限字段发送 MCP 请求,验证外部行为而不只检查启动结果。 | ||
| 251 | +4. 对非法值要求进程启动失败或请求返回规定错误,并检查日志包含字段或错误分类且不泄露密钥。 | ||
| 252 | +5. 保存实际配置、RPM 版本、进程退出码、HTTP/MCP 响应和日志作为验收证据。 | ||
| 253 | + | ||
| 254 | +源码自动化应至少覆盖: | ||
| 255 | + | ||
| 256 | +1. 完整示例可以反序列化、通过运行时校验,并与 `ServerConfig` 全字段序列化结果一致。 | ||
| 257 | +2. 所有有代码默认值的字段在省略后得到文档值。 | ||
| 258 | +3. 所有固定枚举接受全部合法值并拒绝未知值。 | ||
| 259 | +4. 所有具有固定数值范围的字段接受上下边界并拒绝 0、越界值和不一致组合。 | ||
| 260 | +5. 所有条件配置覆盖关闭、正确开启、依赖缺失和回退字段。 | ||
| 261 | +6. 所有部署字符串和路径覆盖非空、规范化、URL 安全、引用关系或文件隔离,不增加无依据长度上限。 | ||
| 262 | + | ||
| 263 | +当前源码自动化已覆盖上述六项: | ||
| 264 | + | ||
| 265 | +| 条目 | 主要自动化用例 | | ||
| 266 | +| --- | --- | | ||
| 267 | +| 完整示例与全字段 Schema | `packaged_rpm_example_matches_server_schema` | | ||
| 268 | +| 全部代码默认值 | `feature_http_and_provider_defaults_are_stable`、`graph_and_case_library_defaults_are_stable`、`http_limit_defaults_are_stable_and_supported`、`pipeline_defaults_and_boundaries_are_stable`、`retrieval_defaults_preserve_current_hybrid_behavior` | | ||
| 269 | +| 固定枚举 | `configurable_enums_reject_unsupported_values`、`authentication_configuration_enforces_fixed_permissions_and_canonical_ids` | | ||
| 270 | +| 数值范围和组合约束 | `http_limits_accept_documented_lower_boundaries`、`http_limits_accept_documented_upper_boundaries`、`http_limits_reject_zero_out_of_range_and_inconsistent_sessions`、`graph_configuration_accepts_all_documented_boundaries`、`graph_configuration_rejects_invalid_configurable_values`、`retrieval_accepts_hybrid_weight_boundaries`、`retrieval_candidate_and_rerank_limits_accept_boundaries`、`retrieval_rejects_candidate_and_rerank_values_outside_limits` | | ||
| 271 | +| 条件配置与回退 | `provider_and_case_library_fallbacks_are_explicit_and_overridable`、`disabled_rerank_ignores_inactive_provider_fields`、`enabled_rerank_rejects_every_invalid_provider_field`,以及 memory-core 的 Rerank/Graph `fail_open` 用例 | | ||
| 272 | +| 字符串、URL 和路径 | `provider_configuration_rejects_incomplete_configurable_values`、`provider_base_url_rejects_credentials_query_and_fragment`、`case_library_paths_and_mappings_reject_every_invalid_shape`、`storage_configuration_rejects_nonpersistent_paths_and_accepts_file_paths`、`http_configuration_covers_host_and_port_boundaries` | | ||
| 273 | + | ||
| 274 | +配置单元测试不能证明以下内容,必须由 RPM 容器或目标环境验收:端口和目录权限、Provider 网络可达、 | ||
| 275 | +API Key 权限/余额、模型名存在、模型 JSON 输出质量、Embedding 实际维度、Rerank 排序效果、 | ||
| 276 | +SQLite 磁盘满/只读/锁冲突、吞吐容量和 TLS termination 是否真实部署。 | ||
| 277 | + | ||
| 278 | +源码中的 `CaseServiceConfig` 是独立案例服务客户端的内部配置类型,不是 `ServerConfig` 顶层 | ||
| 279 | +字段;`base_url`、`bearer_token_env` 等字段不能写入 `ram-a-mem.json`。同样,xiaoO 的 | ||
| 280 | +`memory_automation` 和 MCP Server 注册项属于 Agent 配置,不属于 RAM-A 服务配置。 | ||
| 281 | + | ||
| 282 | +配置相关源码测试命令: | ||
| 283 | + | ||
| 284 | +```bash | ||
| 285 | +cargo test -p memory-mcp --lib config::tests | ||
| 286 | +cargo test -p memory-mcp --test config_auth | ||
| 287 | +cargo test -p memory-mcp --test http_mcp | ||
| 288 | +``` | ||
| @@ -0,0 +1,662 @@ | |||
| 1 | +# RAM-A RPM + xiaoO 容器端到端自测 | ||
| 2 | + | ||
| 3 | +本文用于在干净的 openEuler 容器中安装 RAM-A RPM,先通过真实 HTTP MCP 验证 | ||
| 4 | +`ram-a-mem`,再接入 xiaoO 验证自动记忆摄入和召回。命令默认 RAM-A、xiaoO 在同一容器中 | ||
| 5 | +运行;容器内不依赖 systemd,服务直接作为后台进程启动。 | ||
| 6 | + | ||
| 7 | +本文不假定未确认的 RPM 发布 URL。执行前由测试人员提供 RAM-A RPM URL 或本地 RPM;模型 | ||
| 8 | +服务必须提供 OpenAI-compatible `/chat/completions`。Embedding 使用本地 hash,把外部依赖 | ||
| 9 | +限制为 Chat 模型。基础验收不强制测试 TLS、Graph、案例库和 Rerank。 | ||
| 10 | + | ||
| 11 | +## 测试对象与版本门禁 | ||
| 12 | + | ||
| 13 | +RPM 验收和源码自动化测试验证的是不同对象: | ||
| 14 | + | ||
| 15 | +- `cargo test` 会编译当前源码并在测试进程中使用 mock 或进程内 HTTP Router,不能证明系统中 | ||
| 16 | + 已安装的 RPM 包含相同代码。 | ||
| 17 | +- 本文通过 `/usr/bin/ram-a-mem`(或 RPM 安装出的实际路径)发起真实 HTTP 请求,验证的是 RPM | ||
| 18 | + 二进制、交付配置、运行时依赖和持久化行为,但不能替代内部边界和故障注入单元测试。 | ||
| 19 | +- 发布仓中已有的旧 RPM 只能用于旧版本回归。Rerank `fail_open`、HTTP 并发限制和 Pipeline | ||
| 20 | + 阶段错误契约必须使用包含对应源码提交的候选 RPM 才能作为交付验收结论。 | ||
| 21 | + | ||
| 22 | +测试前必须记录 RPM 的 NEVRA 和 SHA-256,并从打包流水线记录确认其源码提交。RAM-A 当前二进制 | ||
| 23 | +没有提供可用于核对提交号的 `--version` 输出,仅凭文件时间或包名不能证明包含某次修改。如果 | ||
| 24 | +候选 RPM 尚未构建,应先运行下列源码测试;这些结果标记为“源码验证”,不能标记为“RPM 验收”: | ||
| 25 | + | ||
| 26 | +```bash | ||
| 27 | +cd /path/to/RAM-A | ||
| 28 | + | ||
| 29 | +# Rerank fail-closed/fail-open、重试分类。 | ||
| 30 | +cargo test -p memory-core hybrid_search_fail_ | ||
| 31 | +cargo test -p memory-core retry_classification_is_limited_to_transient_failures | ||
| 32 | + | ||
| 33 | +# HTTP 限流、并发上限和 MCP Session 生命周期。 | ||
| 34 | +cargo test -p memory-mcp --test http_mcp \ | ||
| 35 | + concurrent_tool_limit_rejects_excess_work_without_queueing -- --exact | ||
| 36 | +cargo test -p memory-mcp --test http_mcp \ | ||
| 37 | + tool_rate_limit_is_scoped_to_the_authenticated_principal_and_tool -- --exact | ||
| 38 | +cargo test -p memory-mcp --test http_mcp \ | ||
| 39 | + active_session_cap_is_enforced_per_principal -- --exact | ||
| 40 | +cargo test -p memory-mcp --test http_mcp \ | ||
| 41 | + configured_idle_timeout_is_applied_to_the_rmcp_session_worker -- --exact | ||
| 42 | + | ||
| 43 | +# Pipeline fail_fast、阶段日志以及对外错误结构。 | ||
| 44 | +cargo test -p memory-pipeline --test offline_pipeline \ | ||
| 45 | + fail_fast_controls_extraction_and_grounding_failures -- --exact | ||
| 46 | +cargo test -p memory-pipeline --test pipeline_logging | ||
| 47 | +cargo test -p memory-mcp --lib \ | ||
| 48 | + service::tests::provider_failures_keep_pipeline_stage_and_rerank_classification -- --exact | ||
| 49 | +cargo test -p memory-mcp --lib \ | ||
| 50 | + mcp_server::tests::structured_service_errors_expose_stable_pipeline_and_rerank_contracts -- --exact | ||
| 51 | +``` | ||
| 52 | + | ||
| 53 | +若要形成 RPM 验收结论,先由打包流水线从同一源码提交产生候选 RPM,再执行本文后续命令。当前 | ||
| 54 | +RAM-A 源码仓中没有 RPM spec 文件,因此不能在本仓库内用一条通用的 `rpmbuild` 命令可靠地产出 | ||
| 55 | +正式 RPM;RPM 的源码提交关系应由实际发行版打包仓或构建流水线提供。 | ||
| 56 | + | ||
| 57 | +## 1. 启动容器 | ||
| 58 | + | ||
| 59 | +以下命令在宿主机执行。Podman 可替换为 Docker;本地模型在宿主机时保留 `--network host`: | ||
| 60 | + | ||
| 61 | +```bash | ||
| 62 | +export CONTAINER_IMAGE="${CONTAINER_IMAGE:-openeuler/openeuler:24.03-lts}" | ||
| 63 | +podman run --rm -it --name ram-a-e2e --network host "$CONTAINER_IMAGE" bash | ||
| 64 | +``` | ||
| 65 | + | ||
| 66 | +后续命令均在容器内执行: | ||
| 67 | + | ||
| 68 | +```bash | ||
| 69 | +set -euo pipefail | ||
| 70 | +dnf install -y \ | ||
| 71 | + ca-certificates curl git jq openssl sqlite procps-ng \ | ||
| 72 | + findutils sed gawk coreutils gcc gcc-c++ make \ | ||
| 73 | + pkgconf-pkg-config openssl-devel | ||
| 74 | +mkdir -p /root/ram-a-selftest/results /var/lib/ram-a /var/log/ram-a | ||
| 75 | +``` | ||
| 76 | + | ||
| 77 | +构建 xiaoO 需要 Rust;如果镜像没有 Rust: | ||
| 78 | + | ||
| 79 | +```bash | ||
| 80 | +if ! command -v cargo >/dev/null 2>&1; then | ||
| 81 | + dnf install -y rust cargo | ||
| 82 | +fi | ||
| 83 | +cargo --version | ||
| 84 | +rustc --version | ||
| 85 | +``` | ||
| 86 | + | ||
| 87 | +若发行版仓库的 Rust 版本不满足 xiaoO,应改用项目认可的 toolchain,不要在验收记录中隐藏 | ||
| 88 | +编译器版本变化。 | ||
| 89 | + | ||
| 90 | +## 2. 下载并安装 RAM-A RPM | ||
| 91 | + | ||
| 92 | +从发布地址下载: | ||
| 93 | + | ||
| 94 | +```bash | ||
| 95 | +: "${RAM_A_RPM_URL:?请设置 RAM_A_RPM_URL}" | ||
| 96 | +curl --fail --location --retry 3 "$RAM_A_RPM_URL" -o /tmp/ram-a.rpm | ||
| 97 | +``` | ||
| 98 | + | ||
| 99 | +若使用本地 RPM,在启动容器时把它挂载为 `/tmp/ram-a.rpm`,跳过下载。随后执行: | ||
| 100 | + | ||
| 101 | +```bash | ||
| 102 | +sha256sum /tmp/ram-a.rpm | tee /root/ram-a-selftest/results/ram-a-rpm.sha256 | ||
| 103 | +rpm -qip /tmp/ram-a.rpm | tee /root/ram-a-selftest/results/ram-a-rpm-info.txt | ||
| 104 | +dnf install -y /tmp/ram-a.rpm | ||
| 105 | + | ||
| 106 | +RAM_A_PACKAGE=$(rpm -qp --queryformat '%{NAME}' /tmp/ram-a.rpm) | ||
| 107 | +rpm -ql "$RAM_A_PACKAGE" | tee /root/ram-a-selftest/results/ram-a-rpm-files.txt | ||
| 108 | +rpm -V "$RAM_A_PACKAGE" | ||
| 109 | + | ||
| 110 | +RAM_A_BIN=$(command -v ram-a-mem) | ||
| 111 | +test -n "$RAM_A_BIN" | ||
| 112 | +"$RAM_A_BIN" --help | tee /root/ram-a-selftest/results/ram-a-help.txt | ||
| 113 | +rpm -ql "$RAM_A_PACKAGE" | grep -E '/systemd/|\.service$' || true | ||
| 114 | +``` | ||
| 115 | + | ||
| 116 | +通过判据:RPM 安装无依赖错误,`rpm -V` 没有非预期输出,`ram-a-mem` 可执行。 | ||
| 117 | + | ||
| 118 | +## 3. 验证模型服务 | ||
| 119 | + | ||
| 120 | +在同一个容器 shell 中设置: | ||
| 121 | + | ||
| 122 | +```bash | ||
| 123 | +: "${MODEL_BASE_URL:?例如 http://127.0.0.1:8000/v1}" | ||
| 124 | +: "${CHAT_MODEL:?请设置实际模型名}" | ||
| 125 | +: "${LLM_API_KEY:?远程服务填真实 key;免认证本地端点也要填非空占位值}" | ||
| 126 | +export MODEL_BASE_URL CHAT_MODEL LLM_API_KEY | ||
| 127 | +export RAM_A_XIAOO_TOKEN="$(openssl rand -hex 32)" | ||
| 128 | +``` | ||
| 129 | + | ||
| 130 | +先把模型故障与 RAM-A 故障分离: | ||
| 131 | + | ||
| 132 | +```bash | ||
| 133 | +curl --fail --silent --show-error \ | ||
| 134 | + -H "Authorization: Bearer $LLM_API_KEY" \ | ||
| 135 | + -H 'Content-Type: application/json' \ | ||
| 136 | + "$MODEL_BASE_URL/chat/completions" \ | ||
| 137 | + -d "$(jq -nc --arg model "$CHAT_MODEL" \ | ||
| 138 | + '{model:$model,messages:[{role:"user",content:"只回复 OK"}],temperature:0,max_tokens:16}')" \ | ||
| 139 | + | tee /root/ram-a-selftest/results/model-smoke.json \ | ||
| 140 | + | jq -e '.choices[0].message.content | length > 0' | ||
| 141 | +``` | ||
| 142 | + | ||
| 143 | +## 4. 创建 RAM-A 配置 | ||
| 144 | + | ||
| 145 | +只开启个人记忆,密钥值不写入配置: | ||
| 146 | + | ||
| 147 | +```bash | ||
| 148 | +install -d -m 0750 /etc/ram-a /var/lib/ram-a | ||
| 149 | +jq -n --arg base_url "$MODEL_BASE_URL" --arg model "$CHAT_MODEL" '{ | ||
| 150 | + auth:{tokens:[{ | ||
| 151 | + token_env:"RAM_A_XIAOO_TOKEN",tenant_id:"tenant-e2e", | ||
| 152 | + user_id:"user-e2e",agent_id:"xiaoo", | ||
| 153 | + permissions:["memory:read","memory:write"] | ||
| 154 | + }]}, | ||
| 155 | + features:{ | ||
| 156 | + memory:{enabled:true},case_library:{enabled:false},graph_memory:{enabled:false} | ||
| 157 | + }, | ||
| 158 | + http:{ | ||
| 159 | + bind_address:"127.0.0.1",port:18081,allowed_origins:[], | ||
| 160 | + allowed_hosts:["127.0.0.1:18081"],tls_termination_acknowledged:false | ||
| 161 | + }, | ||
| 162 | + limits:{ | ||
| 163 | + max_body_bytes:16777216,requests_per_second:20,rate_burst:40, | ||
| 164 | + max_in_flight_per_principal_tool:4,initialize_requests_per_second:4, | ||
| 165 | + initialize_rate_burst:8,max_active_sessions_per_principal:8, | ||
| 166 | + max_active_sessions_global:256,session_idle_timeout_seconds:1800 | ||
| 167 | + }, | ||
| 168 | + pipeline:{fail_fast:true,max_memory_chars:500}, | ||
| 169 | + storage:{database_path:"/var/lib/ram-a/ram-a-memory.sqlite"}, | ||
| 170 | + providers:{ | ||
| 171 | + api_key_env:"LLM_API_KEY",base_url:$base_url, | ||
| 172 | + embedding_provider:"hash",embedding_api_key_env:null,embedding_base_url:null, | ||
| 173 | + embedding_model:"hash",embedding_dimensions:1024, | ||
| 174 | + extractor_model:$model,verifier_model:$model,timeout_seconds:120,max_retries:3 | ||
| 175 | + }, | ||
| 176 | + retrieval:{ | ||
| 177 | + mode:"hybrid",embedding_weight:0.7,bm25_weight:0.3,candidate_k:100, | ||
| 178 | + rerank:{ | ||
| 179 | + enabled:false,provider:"openrouter",model:"cohere/rerank-v3.5", | ||
| 180 | + api_key_env:null,base_url:"http://127.0.0.1:19090/v1", | ||
| 181 | + input_k:40,timeout_ms:30000,fail_open:false | ||
| 182 | + } | ||
| 183 | + }, | ||
| 184 | + case_library:null,graph_memory:null | ||
| 185 | +}' >/etc/ram-a/ram-a-mem.json | ||
| 186 | + | ||
| 187 | +chmod 0640 /etc/ram-a/ram-a-mem.json | ||
| 188 | +jq empty /etc/ram-a/ram-a-mem.json | ||
| 189 | +``` | ||
| 190 | + | ||
| 191 | +## 5. 启动 RAM-A | ||
| 192 | + | ||
| 193 | +```bash | ||
| 194 | +export RUST_LOG=info | ||
| 195 | +"$RAM_A_BIN" --config /etc/ram-a/ram-a-mem.json \ | ||
| 196 | + >/var/log/ram-a/ram-a-mem.jsonl 2>&1 & | ||
| 197 | +RAM_A_PID=$! | ||
| 198 | +echo "$RAM_A_PID" >/run/ram-a-mem.pid | ||
| 199 | + | ||
| 200 | +for attempt in $(seq 1 30); do | ||
| 201 | + curl --fail --silent http://127.0.0.1:18081/ready >/dev/null && break | ||
| 202 | + if ! kill -0 "$RAM_A_PID" 2>/dev/null; then | ||
| 203 | + cat /var/log/ram-a/ram-a-mem.jsonl | ||
| 204 | + exit 1 | ||
| 205 | + fi | ||
| 206 | + sleep 1 | ||
| 207 | +done | ||
| 208 | + | ||
| 209 | +curl --fail --silent http://127.0.0.1:18081/healthy \ | ||
| 210 | + | tee /root/ram-a-selftest/results/healthy.txt | ||
| 211 | +curl --fail --silent http://127.0.0.1:18081/ready \ | ||
| 212 | + | tee /root/ram-a-selftest/results/ready.txt | ||
| 213 | +``` | ||
| 214 | + | ||
| 215 | +通过判据:进程存活,`/healthy` 和 `/ready` 均返回 2xx。 | ||
| 216 | + | ||
| 217 | +## 6. 创建 MCP 调用脚本 | ||
| 218 | + | ||
| 219 | +脚本完成 initialize、initialized notification 和 `tools/call`,兼容 JSON 与 SSE 响应: | ||
| 220 | + | ||
| 221 | +```bash | ||
| 222 | +cat >/root/ram-a-selftest/mcp.sh <<'EOF' | ||
| 223 | +#!/usr/bin/env bash | ||
| 224 | +set -euo pipefail | ||
| 225 | +: "${RAM_A_XIAOO_TOKEN:?RAM_A_XIAOO_TOKEN is required}" | ||
| 226 | +BASE_URL="${RAM_A_MCP_URL:-http://127.0.0.1:18081/mcp}" | ||
| 227 | +STATE_DIR="${RAM_A_MCP_STATE_DIR:-/root/ram-a-selftest}" | ||
| 228 | +SESSION_FILE="$STATE_DIR/mcp-session-id" | ||
| 229 | + | ||
| 230 | +normalize_response() { | ||
| 231 | + local raw="$1" | ||
| 232 | + if jq -e . "$raw" >/dev/null 2>&1; then | ||
| 233 | + cat "$raw" | ||
| 234 | + else | ||
| 235 | + sed -n 's/^data: //p' "$raw" | tail -n 1 | jq . | ||
| 236 | + fi | ||
| 237 | +} | ||
| 238 | + | ||
| 239 | +common_headers=( | ||
| 240 | + -H 'Content-Type: application/json' | ||
| 241 | + -H 'Accept: application/json, text/event-stream' | ||
| 242 | + -H "Authorization: Bearer $RAM_A_XIAOO_TOKEN" | ||
| 243 | + -H 'X-Agent-ID: xiaoo' | ||
| 244 | + -H 'MCP-Protocol-Version: 2025-11-25' | ||
| 245 | +) | ||
| 246 | + | ||
| 247 | +case "${1:-}" in | ||
| 248 | + init) | ||
| 249 | + curl --fail --silent --show-error -D "$STATE_DIR/init.headers" \ | ||
| 250 | + -o "$STATE_DIR/init.raw" "${common_headers[@]}" -X POST "$BASE_URL" \ | ||
| 251 | + -d '{"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":"2025-11-25","capabilities":{},"clientInfo":{"name":"ram-a-selftest","version":"1.0"}}}' | ||
| 252 | + awk -F': *' 'tolower($1)=="mcp-session-id" {gsub("\r","",$2); print $2}' \ | ||
| 253 | + "$STATE_DIR/init.headers" | tail -n 1 >"$SESSION_FILE" | ||
| 254 | + test -s "$SESSION_FILE" | ||
| 255 | + normalize_response "$STATE_DIR/init.raw" | ||
| 256 | + curl --fail --silent --show-error -o /dev/null "${common_headers[@]}" \ | ||
| 257 | + -H "Mcp-Session-Id: $(cat "$SESSION_FILE")" -X POST "$BASE_URL" \ | ||
| 258 | + -d '{"jsonrpc":"2.0","method":"notifications/initialized","params":{}}' | ||
| 259 | + ;; | ||
| 260 | + list) | ||
| 261 | + curl --fail --silent --show-error -o "$STATE_DIR/call.raw" \ | ||
| 262 | + "${common_headers[@]}" -H "Mcp-Session-Id: $(cat "$SESSION_FILE")" \ | ||
| 263 | + -X POST "$BASE_URL" \ | ||
| 264 | + -d '{"jsonrpc":"2.0","id":2,"method":"tools/list","params":{}}' | ||
| 265 | + normalize_response "$STATE_DIR/call.raw" | ||
| 266 | + ;; | ||
| 267 | + call) | ||
| 268 | + tool="${2:?tool name required}" | ||
| 269 | + request_id="${3:?request id required}" | ||
| 270 | + arguments="${4:?arguments JSON required}" | ||
| 271 | + jq -e . <<<"$arguments" >/dev/null | ||
| 272 | + payload=$(jq -nc --argjson id "$request_id" --arg name "$tool" \ | ||
| 273 | + --argjson arguments "$arguments" \ | ||
| 274 | + '{jsonrpc:"2.0",id:$id,method:"tools/call",params:{name:$name,arguments:$arguments}}') | ||
| 275 | + curl --fail --silent --show-error -o "$STATE_DIR/call.raw" \ | ||
| 276 | + "${common_headers[@]}" -H "Mcp-Session-Id: $(cat "$SESSION_FILE")" \ | ||
| 277 | + -X POST "$BASE_URL" -d "$payload" | ||
| 278 | + normalize_response "$STATE_DIR/call.raw" | ||
| 279 | + ;; | ||
| 280 | + *) | ||
| 281 | + echo "usage: $0 init | list | call TOOL REQUEST_ID ARGUMENTS_JSON" >&2 | ||
| 282 | + exit 2 | ||
| 283 | + ;; | ||
| 284 | +esac | ||
| 285 | +EOF | ||
| 286 | +chmod 0700 /root/ram-a-selftest/mcp.sh | ||
| 287 | +``` | ||
| 288 | + | ||
| 289 | +初始化并确认工具: | ||
| 290 | + | ||
| 291 | +```bash | ||
| 292 | +/root/ram-a-selftest/mcp.sh init \ | ||
| 293 | + | tee /root/ram-a-selftest/results/mcp-initialize.json \ | ||
| 294 | + | jq -e '.result.protocolVersion == "2025-11-25"' | ||
| 295 | +/root/ram-a-selftest/mcp.sh list \ | ||
| 296 | + | tee /root/ram-a-selftest/results/mcp-tools.json \ | ||
| 297 | + | jq -e '[.result.tools[].name] | index("memory_ingest") != null and index("memory_search") != null' | ||
| 298 | +``` | ||
| 299 | + | ||
| 300 | +## 7. 验证无 candidate 路径 | ||
| 301 | + | ||
| 302 | +```bash | ||
| 303 | +NO_CANDIDATE_ARGS='{ | ||
| 304 | + "conversation_id":"e2e-context-only", | ||
| 305 | + "messages":[{ | ||
| 306 | + "id":"context-1","role":"user", | ||
| 307 | + "text":"这条消息只作为上下文,不应产生记忆。","candidate":false | ||
| 308 | + }] | ||
| 309 | +}' | ||
| 310 | +/root/ram-a-selftest/mcp.sh call memory_ingest 10 "$NO_CANDIDATE_ARGS" \ | ||
| 311 | + | tee /root/ram-a-selftest/results/ingest-no-candidate.json \ | ||
| 312 | + | jq -e ' | ||
| 313 | + .result.isError != true and | ||
| 314 | + .result.structuredContent.accepted_count == 0 and | ||
| 315 | + .result.structuredContent.rejected_count == 0 and | ||
| 316 | + .result.structuredContent.quarantined_count == 0 and | ||
| 317 | + (.result.structuredContent.memory_ids | length) == 0 and | ||
| 318 | + .result.structuredContent.idempotency_hit == false' | ||
| 319 | +``` | ||
| 320 | + | ||
| 321 | +## 8. 验证七阶段摄入 | ||
| 322 | + | ||
| 323 | +```bash | ||
| 324 | +MARKER="RAMA-E2E-$(date +%s)" | ||
| 325 | +export MARKER | ||
| 326 | +INGEST_ARGS=$(jq -nc --arg marker "$MARKER" '{ | ||
| 327 | + conversation_id:"e2e-conversation-1", | ||
| 328 | + messages:[ | ||
| 329 | + {id:"context-1",role:"assistant",text:"你平时喜欢什么饮品?",candidate:false}, | ||
| 330 | + {id:"candidate-1",role:"user",speaker:"Alice", | ||
| 331 | + text:("我的长期测试代号是 " + $marker + ",我明确喜欢喝绿茶。"), | ||
| 332 | + timestamp:"2026-08-17T10:00:00Z",candidate:true} | ||
| 333 | + ] | ||
| 334 | +}') | ||
| 335 | +/root/ram-a-selftest/mcp.sh call memory_ingest 11 "$INGEST_ARGS" \ | ||
| 336 | + | tee /root/ram-a-selftest/results/ingest-first.json | ||
| 337 | +jq -e ' | ||
| 338 | + .result.isError != true and | ||
| 339 | + .result.structuredContent.accepted_count >= 1 and | ||
| 340 | + (.result.structuredContent.memory_ids | length) >= 1 and | ||
| 341 | + .result.structuredContent.idempotency_hit == false | ||
| 342 | +' /root/ram-a-selftest/results/ingest-first.json | ||
| 343 | +``` | ||
| 344 | + | ||
| 345 | +若请求成功但 accepted 为 0,先检查 rejected/quarantined 计数和阶段日志。这表示模型结果未通过 | ||
| 346 | +Evidence、枚举或 Grounding 校验,不应直接判定 HTTP、认证或 SQLite 失败。 | ||
| 347 | + | ||
| 348 | +## 9. 验证幂等缓存和冲突 | ||
| 349 | + | ||
| 350 | +```bash | ||
| 351 | +/root/ram-a-selftest/mcp.sh call memory_ingest 12 "$INGEST_ARGS" \ | ||
| 352 | + | tee /root/ram-a-selftest/results/ingest-cached.json | ||
| 353 | +jq -e '.result.isError != true and .result.structuredContent.idempotency_hit == true' \ | ||
| 354 | + /root/ram-a-selftest/results/ingest-cached.json | ||
| 355 | +diff \ | ||
| 356 | + <(jq -S '.result.structuredContent.memory_ids' /root/ram-a-selftest/results/ingest-first.json) \ | ||
| 357 | + <(jq -S '.result.structuredContent.memory_ids' /root/ram-a-selftest/results/ingest-cached.json) | ||
| 358 | + | ||
| 359 | +CONFLICT_ARGS=$(jq -nc --arg marker "$MARKER" '{ | ||
| 360 | + conversation_id:"e2e-conversation-1", | ||
| 361 | + messages:[{id:"candidate-1",role:"user", | ||
| 362 | + text:("修改后的冲突内容 " + $marker),candidate:true}] | ||
| 363 | +}') | ||
| 364 | +/root/ram-a-selftest/mcp.sh call memory_ingest 13 "$CONFLICT_ARGS" \ | ||
| 365 | + | tee /root/ram-a-selftest/results/ingest-conflict.json | ||
| 366 | +jq -e ' | ||
| 367 | + .result.isError == true and | ||
| 368 | + .result.structuredContent.code == "IDEMPOTENCY_CONFLICT" | ||
| 369 | +' /root/ram-a-selftest/results/ingest-conflict.json | ||
| 370 | +``` | ||
| 371 | + | ||
| 372 | +## 10. 验证记忆检索 | ||
| 373 | + | ||
| 374 | +```bash | ||
| 375 | +SEARCH_ARGS=$(jq -nc --arg marker "$MARKER" '{query:($marker + " 绿茶"),top_k:10}') | ||
| 376 | +/root/ram-a-selftest/mcp.sh call memory_search 20 "$SEARCH_ARGS" \ | ||
| 377 | + | tee /root/ram-a-selftest/results/search.json | ||
| 378 | +jq -e --arg marker "$MARKER" ' | ||
| 379 | + .result.isError != true and | ||
| 380 | + (.result.structuredContent.memories | length) >= 1 and | ||
| 381 | + any(.result.structuredContent.memories[]; .text | contains($marker)) | ||
| 382 | +' /root/ram-a-selftest/results/search.json | ||
| 383 | + | ||
| 384 | +FILTER_ARGS=$(jq -nc --arg marker "$MARKER" \ | ||
| 385 | + '{query:($marker + " 绿茶"),top_k:10,memory_types:["preference"]}') | ||
| 386 | +/root/ram-a-selftest/mcp.sh call memory_search 21 "$FILTER_ARGS" \ | ||
| 387 | + | tee /root/ram-a-selftest/results/search-preference.json \ | ||
| 388 | + | jq '.result.structuredContent.memories' | ||
| 389 | +``` | ||
| 390 | + | ||
| 391 | +类型过滤结果用于观察;模型可能把 marker 和偏好拆成不同记忆,不要求所有模型都在此返回一条 | ||
| 392 | +`preference`。 | ||
| 393 | + | ||
| 394 | +## 11. 验证 SQLite 和重启持久化 | ||
| 395 | + | ||
| 396 | +```bash | ||
| 397 | +sqlite3 /var/lib/ram-a/ram-a-memory.sqlite '.tables' \ | ||
| 398 | + | tee /root/ram-a-selftest/results/sqlite-tables.txt | ||
| 399 | +sqlite3 /var/lib/ram-a/ram-a-memory.sqlite 'SELECT COUNT(*) FROM memories;' \ | ||
| 400 | + | tee /root/ram-a-selftest/results/sqlite-memory-count.txt | ||
| 401 | +sqlite3 /var/lib/ram-a/ram-a-memory.sqlite \ | ||
| 402 | + 'SELECT status, COUNT(*) FROM mcp_ingest_idempotency GROUP BY status ORDER BY status;' \ | ||
| 403 | + | tee /root/ram-a-selftest/results/sqlite-idempotency-count.txt | ||
| 404 | + | ||
| 405 | +kill "$RAM_A_PID" | ||
| 406 | +wait "$RAM_A_PID" || true | ||
| 407 | +"$RAM_A_BIN" --config /etc/ram-a/ram-a-mem.json \ | ||
| 408 | + >>/var/log/ram-a/ram-a-mem.jsonl 2>&1 & | ||
| 409 | +RAM_A_PID=$! | ||
| 410 | +echo "$RAM_A_PID" >/run/ram-a-mem.pid | ||
| 411 | +for attempt in $(seq 1 30); do | ||
| 412 | + curl --fail --silent http://127.0.0.1:18081/ready >/dev/null && break | ||
| 413 | + sleep 1 | ||
| 414 | +done | ||
| 415 | + | ||
| 416 | +/root/ram-a-selftest/mcp.sh init >/root/ram-a-selftest/results/mcp-reinitialize.json | ||
| 417 | +/root/ram-a-selftest/mcp.sh call memory_search 30 "$SEARCH_ARGS" \ | ||
| 418 | + | tee /root/ram-a-selftest/results/search-after-restart.json | ||
| 419 | +jq -e --arg marker "$MARKER" ' | ||
| 420 | + any(.result.structuredContent.memories[]; .text | contains($marker)) | ||
| 421 | +' /root/ram-a-selftest/results/search-after-restart.json | ||
| 422 | +``` | ||
| 423 | + | ||
| 424 | +## 12. 验证阶段日志 | ||
| 425 | + | ||
| 426 | +```bash | ||
| 427 | +jq -r ' | ||
| 428 | + select(.fields.event == "ram_a.memory.ingest.stage.started" or | ||
| 429 | + .fields.event == "ram_a.memory.ingest.stage.completed" or | ||
| 430 | + .fields.event == "ram_a.memory.ingest.stage.failed") | ||
| 431 | + | [.fields.event,.fields.stage,(.fields.error_code // "-"),(.fields.elapsed_ms // "-")] | ||
| 432 | + | @tsv | ||
| 433 | +' /var/log/ram-a/ram-a-mem.jsonl \ | ||
| 434 | + | tee /root/ram-a-selftest/results/ingest-stage-logs.tsv | ||
| 435 | + | ||
| 436 | +for stage in normalize episode window extract validate ground aggregate; do | ||
| 437 | + grep -q $'completed\t'"$stage"$'\t' \ | ||
| 438 | + /root/ram-a-selftest/results/ingest-stage-logs.tsv | ||
| 439 | +done | ||
| 440 | +if grep -q "$MARKER" /var/log/ram-a/ram-a-mem.jsonl; then | ||
| 441 | + echo "FAIL: marker leaked into service log" >&2 | ||
| 442 | + exit 1 | ||
| 443 | +fi | ||
| 444 | +``` | ||
| 445 | + | ||
| 446 | +通过判据:成功摄入存在七个规范阶段的 completed,日志不包含测试消息 marker。 | ||
| 447 | + | ||
| 448 | +## 13. 可选:验证 Extract 故障和 pending 重试 | ||
| 449 | + | ||
| 450 | +把 Provider 临时指向不可连接端口,验证 `fail_fast=true` 的稳定错误: | ||
| 451 | + | ||
| 452 | +```bash | ||
| 453 | +cp /etc/ram-a/ram-a-mem.json /etc/ram-a/ram-a-mem.good.json | ||
| 454 | +jq ' | ||
| 455 | + .providers.base_url="http://127.0.0.1:9/v1" | | ||
| 456 | + .providers.timeout_seconds=2 | | ||
| 457 | + .providers.max_retries=1 | ||
| 458 | +' /etc/ram-a/ram-a-mem.good.json >/etc/ram-a/ram-a-mem.json | ||
| 459 | + | ||
| 460 | +kill "$RAM_A_PID" | ||
| 461 | +wait "$RAM_A_PID" || true | ||
| 462 | +"$RAM_A_BIN" --config /etc/ram-a/ram-a-mem.json \ | ||
| 463 | + >>/var/log/ram-a/ram-a-mem.jsonl 2>&1 & | ||
| 464 | +RAM_A_PID=$! | ||
| 465 | +for attempt in $(seq 1 30); do | ||
| 466 | + curl --fail --silent http://127.0.0.1:18081/ready >/dev/null && break | ||
| 467 | + sleep 1 | ||
| 468 | +done | ||
| 469 | + | ||
| 470 | +FAIL_ARGS=$(jq -nc --arg marker "$MARKER" '{ | ||
| 471 | + conversation_id:"e2e-provider-failure", | ||
| 472 | + messages:[{id:"provider-failure-1",role:"user", | ||
| 473 | + text:("请记住故障恢复标记 " + $marker),candidate:true}] | ||
| 474 | +}') | ||
| 475 | +/root/ram-a-selftest/mcp.sh init >/dev/null | ||
| 476 | +/root/ram-a-selftest/mcp.sh call memory_ingest 40 "$FAIL_ARGS" \ | ||
| 477 | + | tee /root/ram-a-selftest/results/ingest-extract-failure.json | ||
| 478 | +jq -e ' | ||
| 479 | + .result.isError == true and | ||
| 480 | + .result.structuredContent.code == "PIPELINE_FAILED" and | ||
| 481 | + .result.structuredContent.stage == "extract" and | ||
| 482 | + .result.structuredContent.retriable == true | ||
| 483 | +' /root/ram-a-selftest/results/ingest-extract-failure.json | ||
| 484 | +``` | ||
| 485 | + | ||
| 486 | +恢复配置并使用完全相同的请求重试: | ||
| 487 | + | ||
| 488 | +```bash | ||
| 489 | +mv /etc/ram-a/ram-a-mem.good.json /etc/ram-a/ram-a-mem.json | ||
| 490 | +kill "$RAM_A_PID" | ||
| 491 | +wait "$RAM_A_PID" || true | ||
| 492 | +"$RAM_A_BIN" --config /etc/ram-a/ram-a-mem.json \ | ||
| 493 | + >>/var/log/ram-a/ram-a-mem.jsonl 2>&1 & | ||
| 494 | +RAM_A_PID=$! | ||
| 495 | +for attempt in $(seq 1 30); do | ||
| 496 | + curl --fail --silent http://127.0.0.1:18081/ready >/dev/null && break | ||
| 497 | + sleep 1 | ||
| 498 | +done | ||
| 499 | +/root/ram-a-selftest/mcp.sh init >/dev/null | ||
| 500 | +/root/ram-a-selftest/mcp.sh call memory_ingest 41 "$FAIL_ARGS" \ | ||
| 501 | + | tee /root/ram-a-selftest/results/ingest-after-provider-recovery.json | ||
| 502 | +jq -e '.result.isError != true' \ | ||
| 503 | + /root/ram-a-selftest/results/ingest-after-provider-recovery.json | ||
| 504 | +``` | ||
| 505 | + | ||
| 506 | +该用例证明 Pipeline 失败后的 pending 幂等记录允许相同内容重试。模型是否接受该记忆仍由 | ||
| 507 | +Extract、Validate 和 Ground 的结果决定。 | ||
| 508 | + | ||
| 509 | +## 14. 安装或构建 xiaoO | ||
| 510 | + | ||
| 511 | +有 xiaoO RPM 时: | ||
| 512 | + | ||
| 513 | +```bash | ||
| 514 | +if [[ -n "${XIAOO_RPM_URL:-}" ]]; then | ||
| 515 | + curl --fail --location --retry 3 "$XIAOO_RPM_URL" -o /tmp/xiaoo.rpm | ||
| 516 | + sha256sum /tmp/xiaoo.rpm | tee /root/ram-a-selftest/results/xiaoo-rpm.sha256 | ||
| 517 | + rpm -qip /tmp/xiaoo.rpm | tee /root/ram-a-selftest/results/xiaoo-rpm-info.txt | ||
| 518 | + dnf install -y /tmp/xiaoo.rpm | ||
| 519 | +fi | ||
| 520 | +``` | ||
| 521 | + | ||
| 522 | +没有 Agent RPM 时,从目标仓库构建: | ||
| 523 | + | ||
| 524 | +```bash | ||
| 525 | +if ! command -v xiaoo >/dev/null 2>&1; then | ||
| 526 | + git clone --depth 1 https://gitcode.com/openeuler/xiaoO.git /opt/xiaoO | ||
| 527 | + git -C /opt/xiaoO rev-parse HEAD \ | ||
| 528 | + | tee /root/ram-a-selftest/results/xiaoo-commit.txt | ||
| 529 | + cargo build --manifest-path /opt/xiaoO/Cargo.toml \ | ||
| 530 | + -p xiaoo-endside --bin xiaoo --release | ||
| 531 | + install -m 0755 /opt/xiaoO/target/release/xiaoo /usr/local/bin/xiaoo | ||
| 532 | +fi | ||
| 533 | + | ||
| 534 | +XIAOO_BIN=$(command -v xiaoo) | ||
| 535 | +"$XIAOO_BIN" --help | tee /root/ram-a-selftest/results/xiaoo-help.txt | ||
| 536 | +``` | ||
| 537 | + | ||
| 538 | +如果目标版本的 package、binary 或 CLI 参数变化,以该版本 `Cargo.toml` 和 `xiaoo --help` | ||
| 539 | +为准;不要把猜测的构建参数记录为已验证接口。 | ||
| 540 | + | ||
| 541 | +## 15. 配置 xiaoO | ||
| 542 | + | ||
| 543 | +```bash | ||
| 544 | +install -d -m 0700 /etc/xiaoo /var/lib/xiaoo | ||
| 545 | +cat >/etc/xiaoo/mcp.json <<'EOF' | ||
| 546 | +{ | ||
| 547 | + "mcpServers": { | ||
| 548 | + "ram-a": { | ||
| 549 | + "transport": "streamable_http", | ||
| 550 | + "url": "http://127.0.0.1:18081/mcp", | ||
| 551 | + "bearer_token_env": "RAM_A_XIAOO_TOKEN", | ||
| 552 | + "agent_id": "xiaoo", | ||
| 553 | + "timeout_ms": 30000 | ||
| 554 | + } | ||
| 555 | + } | ||
| 556 | +} | ||
| 557 | +EOF | ||
| 558 | + | ||
| 559 | +cat >/etc/xiaoo/config.toml <<EOF | ||
| 560 | +[llm] | ||
| 561 | +provider = "anthropic" | ||
| 562 | +api_base = "$MODEL_BASE_URL" | ||
| 563 | +model = "$CHAT_MODEL" | ||
| 564 | +api_key_env = "LLM_API_KEY" | ||
| 565 | +max_tokens = 8192 | ||
| 566 | +reasoning_effort = "off" | ||
| 567 | + | ||
| 568 | +[memory_automation] | ||
| 569 | +enabled = true | ||
| 570 | +server = "ram-a" | ||
| 571 | +recall_top_k = 5 | ||
| 572 | +recall_token_budget = 512 | ||
| 573 | +context_messages = 4 | ||
| 574 | +queue_path = "/var/lib/xiaoo/memory-automation-queue.jsonl" | ||
| 575 | +queue_capacity = 256 | ||
| 576 | +max_retries = 5 | ||
| 577 | +retry_backoff_ms = 250 | ||
| 578 | +allowed_agent_roles = ["main", "defaultagent"] | ||
| 579 | +EOF | ||
| 580 | +chmod 0600 /etc/xiaoo/mcp.json /etc/xiaoo/config.toml | ||
| 581 | +``` | ||
| 582 | + | ||
| 583 | +`memory_automation` 是 xiaoO 配置,不是 MCP 协议或 RAM-A 配置。未启用时,xiaoO 仍可以看到 | ||
| 584 | +`memory_ingest/memory_search` 工具,但不会自动在每轮前后执行召回和摄入。 | ||
| 585 | + | ||
| 586 | +## 16. 验证 xiaoO 自动摄入和召回 | ||
| 587 | + | ||
| 588 | +使用一个没有被前面裸 MCP 摄入过的新 marker,避免把已有记忆误判为 xiaoO 自动摄入成功。 | ||
| 589 | +第一轮要求回复复述 marker,确保最终 assistant 消息也带有可观察信息: | ||
| 590 | + | ||
| 591 | +```bash | ||
| 592 | +AGENT_MARKER="XIAOO-E2E-$(date +%s)" | ||
| 593 | +export AGENT_MARKER | ||
| 594 | +"$XIAOO_BIN" --cli \ | ||
| 595 | + --mcp-config /etc/xiaoo/mcp.json \ | ||
| 596 | + run --config /etc/xiaoo/config.toml --debug \ | ||
| 597 | + -p "请记住:我的长期测试代号是 $AGENT_MARKER,我喜欢喝绿茶。请在回复中准确复述这两项。" \ | ||
| 598 | + 2>&1 | tee /root/ram-a-selftest/results/xiaoo-ingest-turn.log | ||
| 599 | +``` | ||
| 600 | + | ||
| 601 | +等待自动摄入队列处理,并通过裸 MCP 排除“Agent 回答了但没有摄入”: | ||
| 602 | + | ||
| 603 | +```bash | ||
| 604 | +AGENT_SEARCH_ARGS=$(jq -nc --arg marker "$AGENT_MARKER" \ | ||
| 605 | + '{query:($marker + " 绿茶"),top_k:10}') | ||
| 606 | +for attempt in $(seq 1 30); do | ||
| 607 | + /root/ram-a-selftest/mcp.sh init >/dev/null | ||
| 608 | + /root/ram-a-selftest/mcp.sh call memory_search $((100 + attempt)) "$AGENT_SEARCH_ARGS" \ | ||
| 609 | + >/root/ram-a-selftest/results/search-after-xiaoo.json | ||
| 610 | + if jq -e --arg marker "$AGENT_MARKER" ' | ||
| 611 | + any(.result.structuredContent.memories[]?; .text | contains($marker)) | ||
| 612 | + ' /root/ram-a-selftest/results/search-after-xiaoo.json >/dev/null; then | ||
| 613 | + break | ||
| 614 | + fi | ||
| 615 | + sleep 1 | ||
| 616 | +done | ||
| 617 | +jq -e --arg marker "$AGENT_MARKER" ' | ||
| 618 | + any(.result.structuredContent.memories[]?; .text | contains($marker)) | ||
| 619 | +' /root/ram-a-selftest/results/search-after-xiaoo.json | ||
| 620 | +``` | ||
| 621 | + | ||
| 622 | +第二轮不重复 marker,验证自动 recall: | ||
| 623 | + | ||
| 624 | +```bash | ||
| 625 | +"$XIAOO_BIN" --cli \ | ||
| 626 | + --mcp-config /etc/xiaoo/mcp.json \ | ||
| 627 | + run --config /etc/xiaoo/config.toml --debug \ | ||
| 628 | + -p '我之前告诉你的长期测试代号和饮品偏好分别是什么?请依据长期记忆回答。' \ | ||
| 629 | + 2>&1 | tee /root/ram-a-selftest/results/xiaoo-recall-turn.log | ||
| 630 | + | ||
| 631 | +grep -q "$AGENT_MARKER" /root/ram-a-selftest/results/xiaoo-recall-turn.log | ||
| 632 | +grep -q '绿茶' /root/ram-a-selftest/results/xiaoo-recall-turn.log | ||
| 633 | +``` | ||
| 634 | + | ||
| 635 | +通过判据: | ||
| 636 | + | ||
| 637 | +1. xiaoO debug 日志显示 `ram-a` MCP server 连接成功; | ||
| 638 | +2. 第一轮结束后,裸 MCP 能检索到包含 marker 的记忆; | ||
| 639 | +3. 第二轮问题不含 marker,但最终回答正确给出 marker 和绿茶; | ||
| 640 | +4. RAM-A 日志能观察到相应 search/ingest 请求且不包含消息正文; | ||
| 641 | +5. `/var/lib/xiaoo/memory-automation-queue.jsonl` 不持续累积未处理任务。 | ||
| 642 | + | ||
| 643 | +若第一轮裸 MCP 已检索成功而第二轮回答失败,问题位于 xiaoO recall 注入、角色过滤、Token | ||
| 644 | +预算或 Agent Prompt,而不是 RAM-A 摄入和存储。若裸 MCP 也检索不到,应检查 | ||
| 645 | +`memory_automation`、当前 agent role 和摄入队列重试日志。 | ||
| 646 | + | ||
| 647 | +## 17. 结果归档和清理 | ||
| 648 | + | ||
| 649 | +不要归档 Token、模型 API Key 或生产对话。当前测试使用专用 marker,可以保留测试响应和脱敏 | ||
| 650 | +日志: | ||
| 651 | + | ||
| 652 | +```bash | ||
| 653 | +tar -C /root -czf /tmp/ram-a-selftest-results.tar.gz ram-a-selftest/results | ||
| 654 | +sha256sum /tmp/ram-a-selftest-results.tar.gz | ||
| 655 | +kill "$RAM_A_PID" 2>/dev/null || true | ||
| 656 | +wait "$RAM_A_PID" 2>/dev/null || true | ||
| 657 | +unset RAM_A_XIAOO_TOKEN LLM_API_KEY | ||
| 658 | +``` | ||
| 659 | + | ||
| 660 | +验收记录至少包含:容器镜像、RAM-A RPM NEVRA/SHA-256、xiaoO commit 或 RPM NEVRA、模型名、 | ||
| 661 | +配置 SHA-256、每步通过/失败和结果归档 SHA-256。模型质量问题应与协议、认证、存储和 Agent | ||
| 662 | +调度问题分别记录。 | ||
| @@ -77,8 +77,8 @@ RAM-A 服务端不会主动判断用户意图或自动触发工具。xiaoO 建 | |||
| 77 | ``` | 77 | ``` |
| 78 | 78 | ||
| 79 | - `query` 必填,最多 32000 个 Unicode 字符。 | 79 | - `query` 必填,最多 32000 个 Unicode 字符。 |
| 80 | -- `library` 可选;省略时使用服务端 `default_library`。它是公开别名,不是 | 80 | +- `library` 可选,最多 255 个 Unicode 字符;省略时使用服务端 `default_library`。 |
| 81 | - `dataset_id`。 | 81 | + 它是公开别名,不是 `dataset_id`。 |
| 82 | - `top_k` 可选,默认 `5`,范围 `1..20`。 | 82 | - `top_k` 可选,默认 `5`,范围 `1..20`。 |
| 83 | 83 | ||
| 84 | 返回示例: | 84 | 返回示例: |
| @@ -41,10 +41,11 @@ | |||
| 41 | "bind_address": "127.0.0.1", | 41 | "bind_address": "127.0.0.1", |
| 42 | "port": 18081, | 42 | "port": 18081, |
| 43 | "allowed_origins": ["http://127.0.0.1:18080"], | 43 | "allowed_origins": ["http://127.0.0.1:18080"], |
| 44 | - "allowed_hosts": ["127.0.0.1:18081"] | 44 | + "allowed_hosts": ["127.0.0.1:18081"], |
| 45 | + "tls_termination_acknowledged": false | ||
| 45 | }, | 46 | }, |
| 46 | "limits": { | 47 | "limits": { |
| 47 | - "max_body_bytes": 1048576, | 48 | + "max_body_bytes": 16777216, |
| 48 | "requests_per_second": 20, | 49 | "requests_per_second": 20, |
| 49 | "rate_burst": 40, | 50 | "rate_burst": 40, |
| 50 | "max_in_flight_per_principal_tool": 4, | 51 | "max_in_flight_per_principal_tool": 4, |
| @@ -54,6 +55,10 @@ | |||
| 54 | "max_active_sessions_global": 256, | 55 | "max_active_sessions_global": 256, |
| 55 | "session_idle_timeout_seconds": 1800 | 56 | "session_idle_timeout_seconds": 1800 |
| 56 | }, | 57 | }, |
| 58 | + "pipeline": { | ||
| 59 | + "fail_fast": true, | ||
| 60 | + "max_memory_chars": 500 | ||
| 61 | + }, | ||
| 57 | "storage": { | 62 | "storage": { |
| 58 | "database_path": "data/ram-a-memory.sqlite" | 63 | "database_path": "data/ram-a-memory.sqlite" |
| 59 | }, | 64 | }, |
| @@ -61,6 +66,8 @@ | |||
| 61 | "api_key_env": "LLM_API_KEY", | 66 | "api_key_env": "LLM_API_KEY", |
| 62 | "base_url": "http://127.0.0.1:8000/v1", | 67 | "base_url": "http://127.0.0.1:8000/v1", |
| 63 | "embedding_provider": "hash", | 68 | "embedding_provider": "hash", |
| 69 | + "embedding_api_key_env": null, | ||
| 70 | + "embedding_base_url": null, | ||
| 64 | "embedding_model": "hash", | 71 | "embedding_model": "hash", |
| 65 | "embedding_dimensions": 1024, | 72 | "embedding_dimensions": 1024, |
| 66 | "extractor_model": "GLM-5.2", | 73 | "extractor_model": "GLM-5.2", |
| @@ -91,9 +98,15 @@ | |||
| 91 | "api_token_env": "RAM_A_CASES_ADMIN_TOKEN", | 98 | "api_token_env": "RAM_A_CASES_ADMIN_TOKEN", |
| 92 | "ingestion_poll_ms": 1000, | 99 | "ingestion_poll_ms": 1000, |
| 93 | "embedding_provider": "hash", | 100 | "embedding_provider": "hash", |
| 101 | + "embedding_api_key_env": null, | ||
| 102 | + "embedding_base_url": null, | ||
| 94 | "embedding_model": "hash", | 103 | "embedding_model": "hash", |
| 95 | "embedding_dimensions": 1024, | 104 | "embedding_dimensions": 1024, |
| 96 | "chunk_size": 160, | 105 | "chunk_size": 160, |
| 106 | + "summary_llm_model": null, | ||
| 107 | + "summary_llm_api_key_env": null, | ||
| 108 | + "summary_llm_base_url": null, | ||
| 109 | + "summary_llm_timeout_ms": 30000, | ||
| 97 | "default_library": "ops", | 110 | "default_library": "ops", |
| 98 | "libraries": [ | 111 | "libraries": [ |
| 99 | { | 112 | { |