已合并
feat(rsi): add evidence-driven recursive harness evolution framework #2497
zhitinggao创建于 10 天前
feat(rsi): add evidence-driven recursive harness evolution framework #2497
已合并
共 48 个文件变更+30646-598
| @@ -0,0 +1,1183 @@ | |||
| 1 | +# coding: utf-8 | ||
| 2 | +# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. | ||
| 3 | +"""Deterministic, decision-centered compression for analyzer evidence.""" | ||
| 4 | + | ||
| 5 | +from __future__ import annotations | ||
| 6 | + | ||
| 7 | +import hashlib | ||
| 8 | +import json | ||
| 9 | +import re | ||
| 10 | +from collections import Counter, defaultdict | ||
| 11 | +from collections.abc import Mapping, Sequence | ||
| 12 | +from pathlib import Path | ||
| 13 | +from typing import Any | ||
| 14 | + | ||
| 15 | +_MIN_SELECTED_CALLS_PER_TRIAL = 8 | ||
| 16 | +_MAX_SELECTED_CALLS_PER_TRIAL = 20 | ||
| 17 | +_MAX_TOOL_SEQUENCE = 80 | ||
| 18 | +_MUTATING_TOOL_PATTERN = re.compile( | ||
| 19 | + r"(?:^|_)(?:assign|cancel|create|delete|disable|enable|finish|modify|order|patch|save|schedule|send|submit|" | ||
| 20 | + r"update|write)(?:_|$)", | ||
| 21 | + re.IGNORECASE, | ||
| 22 | +) | ||
| 23 | +_ARTIFACT_PATTERN = re.compile( | ||
| 24 | + r"(?:^|[\s`'\"])([^\s`'\"]+\.(?:csv|docx|html|json|md|pdf|pptx|txt|xlsx))(?:$|[\s`'\"])", re.IGNORECASE | ||
| 25 | +) | ||
| 26 | +_FAILED_RESPONSE_PATTERN = re.compile( | ||
| 27 | + r"(?:success\s*=\s*false|\"success\"\s*:\s*false|exit\s+code\s*:\s*[1-9]\d*|traceback\s*\(|access\s+denied)", | ||
| 28 | + re.IGNORECASE, | ||
| 29 | +) | ||
| 30 | +_BASH_MUTATION_PATTERN = re.compile( | ||
| 31 | + r"(?:^|[;&|\n]\s*)(?:cp|mv|rm|mkdir|touch|install)\b|" | ||
| 32 | + r"(?:\.save\s*\(|save_workbook\s*\(|write_text\s*\(|write_bytes\s*\(|" | ||
| 33 | + r"shutil\.(?:copy|copy2|copytree|move)|libreoffice\b.*--convert-to|(?:^|\s)>\s*[^&])", | ||
| 34 | + re.IGNORECASE, | ||
| 35 | +) | ||
| 36 | +_BASH_CONTENT_PATTERN = re.compile( | ||
| 37 | + r"(?:pdftotext|python\w*\b.*(?:docx|openpyxl|pypdf|python-pptx)|" | ||
| 38 | + r"(?:cat|sed|head|tail|unzip)\s)", | ||
| 39 | + re.IGNORECASE | re.DOTALL, | ||
| 40 | +) | ||
| 41 | +_CRITICAL_EVIDENCE_TERM_PATTERN = re.compile(r"[A-Za-z][A-Za-z0-9_.-]{2,}|\d+(?:\.\d+)?|[\u4e00-\u9fff]{2,8}") | ||
| 42 | +_CRITICAL_EVIDENCE_STOPWORDS = { | ||
| 43 | + "and", | ||
| 44 | + "agent", | ||
| 45 | + "answer", | ||
| 46 | + "assessment", | ||
| 47 | + "because", | ||
| 48 | + "between", | ||
| 49 | + "but", | ||
| 50 | + "calculating", | ||
| 51 | + "conclusion", | ||
| 52 | + "correctly", | ||
| 53 | + "criterion", | ||
| 54 | + "did", | ||
| 55 | + "directly", | ||
| 56 | + "evidence", | ||
| 57 | + "explicitly", | ||
| 58 | + "failed", | ||
| 59 | + "failure", | ||
| 60 | + "first", | ||
| 61 | + "identified", | ||
| 62 | + "met", | ||
| 63 | + "must", | ||
| 64 | + "not", | ||
| 65 | + "requirement", | ||
| 66 | + "required", | ||
| 67 | + "response", | ||
| 68 | + "second", | ||
| 69 | + "specific", | ||
| 70 | + "state", | ||
| 71 | + "stated", | ||
| 72 | + "states", | ||
| 73 | + "task", | ||
| 74 | + "text", | ||
| 75 | + "than", | ||
| 76 | + "the", | ||
| 77 | + "that", | ||
| 78 | + "this", | ||
| 79 | + "timeline", | ||
| 80 | + "timelines", | ||
| 81 | + "with", | ||
| 82 | +} | ||
| 83 | +_COMPACTION_MARKER = ( | ||
| 84 | + "[ANALYZER_EVIDENCE_COMPACTION: omitted {omitted} source chars; this marker was not observed by the task agent]" | ||
| 85 | +) | ||
| 86 | + | ||
| 87 | + | ||
| 88 | +def public_task_contract_snapshot(task: Mapping[str, Any]) -> dict[str, Any]: | ||
| 89 | + """Keep the public task and tool contract while excluding scorer internals.""" | ||
| 90 | + public = task.get("public_task_contract") | ||
| 91 | + public = public if isinstance(public, Mapping) else {} | ||
| 92 | + metadata = public.get("metadata", task.get("metadata")) | ||
| 93 | + metadata = metadata if isinstance(metadata, Mapping) else {} | ||
| 94 | + tool_schemas = public.get("tool_schemas") | ||
| 95 | + tool_schemas = tool_schemas if isinstance(tool_schemas, Sequence) and not isinstance(tool_schemas, str) else [] | ||
| 96 | + task_metadata: dict[str, Any] = {} | ||
| 97 | + for key, value in metadata.items(): | ||
| 98 | + if key in {"canary", "category", "difficulty", "language"} and _is_scalar(value): | ||
| 99 | + task_metadata[str(key)] = value | ||
| 100 | + normalized_tool_schemas: list[dict[str, Any]] = [] | ||
| 101 | + for schema in tool_schemas: | ||
| 102 | + if not isinstance(schema, Mapping): | ||
| 103 | + continue | ||
| 104 | + normalized = _normalize_tool_schema(schema) | ||
| 105 | + if normalized: | ||
| 106 | + normalized_tool_schemas.append(normalized) | ||
| 107 | + return { | ||
| 108 | + "schema_version": 1, | ||
| 109 | + "provenance": "official_suite.public_task_contract", | ||
| 110 | + "task_id": str(task.get("id") or public.get("task_id") or ""), | ||
| 111 | + "domain": str(public.get("domain") or task.get("domain") or ""), | ||
| 112 | + "prompt": str(public.get("prompt") or task.get("prompt") or ""), | ||
| 113 | + "task_metadata": task_metadata, | ||
| 114 | + "tool_schemas": normalized_tool_schemas, | ||
| 115 | + } | ||
| 116 | + | ||
| 117 | + | ||
| 118 | +def load_public_task_contract( | ||
| 119 | + *, | ||
| 120 | + case_id: str, | ||
| 121 | + result_path: str, | ||
| 122 | + evaluation_metadata: Mapping[str, Any], | ||
| 123 | + task_input: str, | ||
| 124 | +) -> dict[str, Any]: | ||
| 125 | + """Load a materialized public contract, with a safe fallback for older runs.""" | ||
| 126 | + direct = evaluation_metadata.get("analysis_task_contract") | ||
| 127 | + if isinstance(direct, Mapping): | ||
| 128 | + return dict(direct) | ||
| 129 | + | ||
| 130 | + result = Path(result_path) | ||
| 131 | + evaluation_dir = result.parent.parent.parent if len(result.parents) >= 3 else Path() | ||
| 132 | + suite_path = evaluation_dir / "official" / "suite.json" | ||
| 133 | + suite = _read_mapping(suite_path) | ||
| 134 | + for split_name in ("validation", "evaluation", "tasks"): | ||
| 135 | + tasks = suite.get(split_name) | ||
| 136 | + if not isinstance(tasks, list): | ||
| 137 | + continue | ||
| 138 | + for task in tasks: | ||
| 139 | + if isinstance(task, Mapping) and str(task.get("id") or "") == case_id: | ||
| 140 | + return public_task_contract_snapshot(task) | ||
| 141 | + | ||
| 142 | + return { | ||
| 143 | + "schema_version": 1, | ||
| 144 | + "provenance": "case.input", | ||
| 145 | + "task_id": case_id, | ||
| 146 | + "prompt": task_input, | ||
| 147 | + "tool_schemas": [], | ||
| 148 | + } | ||
| 149 | + | ||
| 150 | + | ||
| 151 | +def build_causal_evidence_digest( # pylint: disable=huawei-too-many-arguments | ||
| 152 | + *, | ||
| 153 | + case_id: str, | ||
| 154 | + task_input: str, | ||
| 155 | + response: str, | ||
| 156 | + evaluation_passed: bool, | ||
| 157 | + evaluation_score: float, | ||
| 158 | + evaluation_reason: str, | ||
| 159 | + evaluation_metadata: Mapping[str, Any], | ||
| 160 | + trace_data: Mapping[str, Any], | ||
| 161 | + task_contract: Mapping[str, Any], | ||
| 162 | +) -> dict[str, Any]: | ||
| 163 | + """Build a compact evidence packet that preserves trial differences and exact actions.""" | ||
| 164 | + traces = trace_data.get("traces") | ||
| 165 | + traces = traces if isinstance(traces, list) else [] | ||
| 166 | + trial_scores = _aligned_list(evaluation_metadata.get("trial_scores"), len(traces)) | ||
| 167 | + trial_passed = _aligned_list(evaluation_metadata.get("trial_passed"), len(traces)) | ||
| 168 | + trial_exit_reasons = _aligned_list(evaluation_metadata.get("trial_exit_reasons"), len(traces)) | ||
| 169 | + trial_details = _aligned_list(evaluation_metadata.get("trial_details"), len(traces)) | ||
| 170 | + has_trial_outcomes = bool(traces) and bool(trial_scores) and bool(trial_passed) | ||
| 171 | + critical_evidence_terms = _failed_requirement_terms(evaluation_metadata) | ||
| 172 | + | ||
| 173 | + trial_records: list[dict[str, Any]] = [] | ||
| 174 | + all_calls: list[dict[str, Any]] = [] | ||
| 175 | + raw_message_count = 0 | ||
| 176 | + for index, raw_trace in enumerate(traces): | ||
| 177 | + if not isinstance(raw_trace, Mapping): | ||
| 178 | + continue | ||
| 179 | + score = trial_scores[index] if has_trial_outcomes else (evaluation_score if len(traces) == 1 else None) | ||
| 180 | + passed = trial_passed[index] if has_trial_outcomes else (evaluation_passed if len(traces) == 1 else None) | ||
| 181 | + exit_reason = trial_exit_reasons[index] if trial_exit_reasons else "" | ||
| 182 | + trial = _build_trial_record( | ||
| 183 | + raw_trace, | ||
| 184 | + index=index, | ||
| 185 | + score=score, | ||
| 186 | + passed=passed, | ||
| 187 | + exit_reason=exit_reason, | ||
| 188 | + critical_evidence_terms=critical_evidence_terms, | ||
| 189 | + ) | ||
| 190 | + trial["trial_evaluation"] = _compact_trial_evaluation(trial_details[index] if trial_details else None) | ||
| 191 | + raw_message_count += int(trial.pop("_raw_message_count")) | ||
| 192 | + all_calls.extend(trial.pop("_all_calls")) | ||
| 193 | + trial_records.append(trial) | ||
| 194 | + | ||
| 195 | + _deduplicate_selected_payloads(trial_records) | ||
| 196 | + tool_contracts = _tool_contract_observations(task_contract, all_calls) | ||
| 197 | + compact_task_contract = { | ||
| 198 | + key: _compact_value(value) for key, value in task_contract.items() if key not in {"prompt", "tool_schemas"} | ||
| 199 | + } | ||
| 200 | + compact_task_contract["prompt_ref"] = "authoritative_task_contract.input_excerpt" | ||
| 201 | + compact_task_contract["tool_schema_ref"] = "tool_contract_observations" | ||
| 202 | + digest = { | ||
| 203 | + "schema_version": 1, | ||
| 204 | + "compression_policy": { | ||
| 205 | + "method": "deterministic_decision_centered", | ||
| 206 | + "no_llm_summarization": True, | ||
| 207 | + "trial_boundaries_preserved": has_trial_outcomes, | ||
| 208 | + "exact_numbers_and_structured_tool_arguments_preserved": True, | ||
| 209 | + "action_selection": ( | ||
| 210 | + "dynamic decision-linked budget; prioritize observed failures, state mutations, " | ||
| 211 | + "content-bearing reads, tool boundaries, and the terminal window" | ||
| 212 | + ), | ||
| 213 | + "response_only_field_policy": ( | ||
| 214 | + "A response-only field is not a missing request field unless the public tool schema declares it." | ||
| 215 | + ), | ||
| 216 | + "lossless_evidence_policy": ( | ||
| 217 | + "Compacted response excerpts are display views, not task-agent observations. " | ||
| 218 | + "ANALYZER_EVIDENCE_COMPACTION markers are generated after execution. " | ||
| 219 | + "Exact failed-requirement-linked spans are retained separately with raw evidence pointers." | ||
| 220 | + ), | ||
| 221 | + }, | ||
| 222 | + "task_contract": compact_task_contract, | ||
| 223 | + "outcome": { | ||
| 224 | + "case_id": case_id, | ||
| 225 | + "passed": evaluation_passed, | ||
| 226 | + "score": evaluation_score, | ||
| 227 | + "reason": _compact_text(evaluation_reason, 1_000), | ||
| 228 | + "trial_count": len(trial_records), | ||
| 229 | + "judge_dimensions": _judge_dimensions(evaluation_metadata), | ||
| 230 | + "judge_evidence": _compact_judge_evidence(evaluation_metadata), | ||
| 231 | + }, | ||
| 232 | + "tool_contract_observations": tool_contracts, | ||
| 233 | + "critical_evidence_terms": critical_evidence_terms, | ||
| 234 | + "trials": trial_records, | ||
| 235 | + "cross_trial_contrast": _cross_trial_contrast(trial_records), | ||
| 236 | + "fallback_final_response": ( | ||
| 237 | + {"available": False, "reason": "per_trial_final_outputs_available"} | ||
| 238 | + if trial_records | ||
| 239 | + else {"available": True, **_output_summary(response)} | ||
| 240 | + ), | ||
| 241 | + "compression_stats": { | ||
| 242 | + "raw_trace_count": len(traces), | ||
| 243 | + "raw_message_count": raw_message_count, | ||
| 244 | + "raw_tool_call_count": len(all_calls), | ||
| 245 | + "selected_tool_call_count": sum(len(trial["selected_actions"]) for trial in trial_records), | ||
| 246 | + }, | ||
| 247 | + } | ||
| 248 | + return digest | ||
| 249 | + | ||
| 250 | + | ||
| 251 | +def _compact_trial_evaluation(value: Any) -> dict[str, Any]: | ||
| 252 | + """Preserve per-trial grader evidence without treating absence as zero.""" | ||
| 253 | + if not isinstance(value, Mapping): | ||
| 254 | + return { | ||
| 255 | + "schema_version": 1, | ||
| 256 | + "availability": { | ||
| 257 | + "score_file": "not_instrumented", | ||
| 258 | + "score_reason": "not_instrumented", | ||
| 259 | + "judge_detail": "not_instrumented", | ||
| 260 | + "dimension_scores": "not_instrumented", | ||
| 261 | + }, | ||
| 262 | + "score_reason": "", | ||
| 263 | + "judge_detail": None, | ||
| 264 | + "dimension_scores": {}, | ||
| 265 | + "source": {}, | ||
| 266 | + } | ||
| 267 | + | ||
| 268 | + raw_availability = value.get("availability") | ||
| 269 | + raw_availability = raw_availability if isinstance(raw_availability, Mapping) else {} | ||
| 270 | + availability = { | ||
| 271 | + str(key): str(item) for key, item in raw_availability.items() if isinstance(key, str) and isinstance(item, str) | ||
| 272 | + } | ||
| 273 | + raw_dimensions = value.get("dimension_scores") | ||
| 274 | + raw_dimensions = raw_dimensions if isinstance(raw_dimensions, Mapping) else {} | ||
| 275 | + dimensions: dict[str, dict[str, Any]] = {} | ||
| 276 | + for name, raw_dimension in raw_dimensions.items(): | ||
| 277 | + if not isinstance(raw_dimension, Mapping): | ||
| 278 | + continue | ||
| 279 | + state = str(raw_dimension.get("availability") or "not_available") | ||
| 280 | + raw_score = raw_dimension.get("value") | ||
| 281 | + dimensions[str(name)] = { | ||
| 282 | + "availability": state, | ||
| 283 | + "value": raw_score if _is_number(raw_score) else None, | ||
| 284 | + "source": str(raw_dimension.get("source")) if raw_dimension.get("source") else None, | ||
| 285 | + } | ||
| 286 | + raw_judge_detail = value.get("judge_detail") | ||
| 287 | + judge_detail = _compact_value(raw_judge_detail) if isinstance(raw_judge_detail, Mapping) else None | ||
| 288 | + source = value.get("source") | ||
| 289 | + source = _compact_value(source) if isinstance(source, Mapping) else {} | ||
| 290 | + return { | ||
| 291 | + "schema_version": 1, | ||
| 292 | + "availability": availability, | ||
| 293 | + "score": value.get("score") if _is_number(value.get("score")) else None, | ||
| 294 | + "passed": value.get("passed") if isinstance(value.get("passed"), bool) else None, | ||
| 295 | + "score_reason": _compact_text(value.get("score_reason"), 1_000), | ||
| 296 | + "judge_detail": judge_detail, | ||
| 297 | + "dimension_scores": dimensions, | ||
| 298 | + "source": source, | ||
| 299 | + } | ||
| 300 | + | ||
| 301 | + | ||
| 302 | +def _compact_judge_evidence(metadata: Mapping[str, Any]) -> dict[str, Any]: | ||
| 303 | + """Keep aggregate criterion feedback that is not present in trial score files.""" | ||
| 304 | + normalized = metadata.get("judge_evidence") | ||
| 305 | + if isinstance(normalized, Mapping): | ||
| 306 | + criteria = normalized.get("criteria") | ||
| 307 | + return { | ||
| 308 | + "schema_version": 1, | ||
| 309 | + "availability": str(normalized.get("availability") or "not_available"), | ||
| 310 | + "grading_run_status": str(normalized.get("grading_run_status") or ""), | ||
| 311 | + "criteria": _compact_value(criteria) if isinstance(criteria, list) else [], | ||
| 312 | + } | ||
| 313 | + | ||
| 314 | + detail = metadata.get("judge_detail") | ||
| 315 | + if not isinstance(detail, Mapping): | ||
| 316 | + return { | ||
| 317 | + "schema_version": 1, | ||
| 318 | + "availability": "not_instrumented", | ||
| 319 | + "grading_run_status": "", | ||
| 320 | + "criteria": [], | ||
| 321 | + } | ||
| 322 | + raw_criteria = detail.get("criteria") | ||
| 323 | + if not isinstance(raw_criteria, list): | ||
| 324 | + return { | ||
| 325 | + "schema_version": 1, | ||
| 326 | + "availability": "not_available", | ||
| 327 | + "grading_run_status": str(detail.get("grading_run_status") or ""), | ||
| 328 | + "criteria": [], | ||
| 329 | + } | ||
| 330 | + criteria = [_compact_value(item) for item in raw_criteria if isinstance(item, Mapping)] | ||
| 331 | + return { | ||
| 332 | + "schema_version": 1, | ||
| 333 | + "availability": "available" if criteria else "invalid", | ||
| 334 | + "grading_run_status": str(detail.get("grading_run_status") or ""), | ||
| 335 | + "criteria": criteria, | ||
| 336 | + } | ||
| 337 | + | ||
| 338 | + | ||
| 339 | +def compact_candidate_feedback(feedback: Mapping[str, Any] | None) -> dict[str, Any]: | ||
| 340 | + """Normalize paired intervention evidence without losing outcome signals. | ||
| 341 | + | ||
| 342 | + Candidate evaluation and historical journal records use slightly different | ||
| 343 | + field names. Normalize both into one analyzer-facing schema so a diagnosis | ||
| 344 | + always sees the pre-evaluation prediction, activation evidence, strict | ||
| 345 | + acceptance score, continuous diagnostic score, and per-requirement delta. | ||
| 346 | + """ | ||
| 347 | + if not isinstance(feedback, Mapping): | ||
| 348 | + return {} | ||
| 349 | + records = feedback.get("experiments") | ||
| 350 | + if not isinstance(records, list): | ||
| 351 | + return {} | ||
| 352 | + experiments: list[dict[str, Any]] = [] | ||
| 353 | + for record in records[-3:]: | ||
| 354 | + if not isinstance(record, Mapping): | ||
| 355 | + continue | ||
| 356 | + diagnoses = record.get("candidate_failure_diagnoses") | ||
| 357 | + diagnoses = diagnoses if isinstance(diagnoses, list) else [] | ||
| 358 | + raw_observed = record.get("observed_outcome") | ||
| 359 | + raw_observed = raw_observed if isinstance(raw_observed, Mapping) else {} | ||
| 360 | + raw_prediction = record.get("prediction") | ||
| 361 | + raw_prediction = raw_prediction if isinstance(raw_prediction, Mapping) else {} | ||
| 362 | + strict_score = _compact_score_comparison( | ||
| 363 | + raw_observed.get("strict_score"), | ||
| 364 | + source=_first_present(raw_observed, record, keys=("source_target_score", "source_strict_score")), | ||
| 365 | + candidate=_first_present( | ||
| 366 | + raw_observed, | ||
| 367 | + record, | ||
| 368 | + keys=("candidate_target_score", "candidate_strict_score"), | ||
| 369 | + ), | ||
| 370 | + delta=_first_present(raw_observed, record, keys=("target_score_delta", "strict_score_delta")), | ||
| 371 | + ) | ||
| 372 | + continuous_score = _compact_score_comparison( | ||
| 373 | + raw_observed.get("continuous_score"), | ||
| 374 | + source=_first_present(raw_observed, record, keys=("source_native_score", "source_continuous_score")), | ||
| 375 | + candidate=_first_present( | ||
| 376 | + raw_observed, | ||
| 377 | + record, | ||
| 378 | + keys=("candidate_native_score", "candidate_continuous_score"), | ||
| 379 | + ), | ||
| 380 | + delta=_first_present(raw_observed, record, keys=("native_score_delta", "continuous_score_delta")), | ||
| 381 | + ) | ||
| 382 | + continuous_score.update( | ||
| 383 | + { | ||
| 384 | + "source_signal": str( | ||
| 385 | + _first_present(raw_observed, record, keys=("source_native_signal", "source_continuous_signal")) | ||
| 386 | + or "" | ||
| 387 | + ), | ||
| 388 | + "candidate_signal": str( | ||
| 389 | + _first_present( | ||
| 390 | + raw_observed, | ||
| 391 | + record, | ||
| 392 | + keys=("candidate_native_signal", "candidate_continuous_signal"), | ||
| 393 | + ) | ||
| 394 | + or "" | ||
| 395 | + ), | ||
| 396 | + "role": str(_first_present(raw_observed, record, keys=("native_signal_role",)) or ""), | ||
| 397 | + } | ||
| 398 | + ) | ||
| 399 | + requirement_delta = _first_mapping( | ||
| 400 | + raw_observed.get("requirement_delta"), | ||
| 401 | + raw_observed.get("verifier_delta"), | ||
| 402 | + record.get("requirement_delta"), | ||
| 403 | + record.get("verifier_delta"), | ||
| 404 | + ) | ||
| 405 | + dimension_deltas = _first_mapping( | ||
| 406 | + raw_observed.get("dimension_deltas"), | ||
| 407 | + raw_observed.get("native_dimension_deltas"), | ||
| 408 | + record.get("dimension_deltas"), | ||
| 409 | + record.get("native_dimension_deltas"), | ||
| 410 | + ) | ||
| 411 | + contracts = raw_prediction.get("causal_intervention_contracts") | ||
| 412 | + if not isinstance(contracts, list): | ||
| 413 | + contracts = record.get("causal_intervention_contracts") | ||
| 414 | + contracts = contracts if isinstance(contracts, list) else [] | ||
| 415 | + activation = record.get("activation") | ||
| 416 | + activation = activation if isinstance(activation, Mapping) else {} | ||
| 417 | + observed_outcome = { | ||
| 418 | + "status": str(_first_present(raw_observed, record, keys=("status", "outcome")) or ""), | ||
| 419 | + "reason": str(_first_present(raw_observed, record, keys=("reason",)) or ""), | ||
| 420 | + "strict_score": strict_score, | ||
| 421 | + "continuous_score": continuous_score, | ||
| 422 | + "requirement_delta": _compact_value(requirement_delta), | ||
| 423 | + "dimension_deltas": _compact_value(dimension_deltas), | ||
| 424 | + "selected_for_promotion": _first_present( | ||
| 425 | + raw_observed, | ||
| 426 | + record, | ||
| 427 | + keys=("selected_for_promotion",), | ||
| 428 | + ), | ||
| 429 | + # Preserve v1 aliases while analyzer consumers migrate to the | ||
| 430 | + # explicit strict/continuous comparison objects above. | ||
| 431 | + "source_target_score": strict_score["source"], | ||
| 432 | + "candidate_target_score": strict_score["candidate"], | ||
| 433 | + "target_score_delta": strict_score["delta"], | ||
| 434 | + "source_native_score": continuous_score["source"], | ||
| 435 | + "candidate_native_score": continuous_score["candidate"], | ||
| 436 | + "native_score_delta": continuous_score["delta"], | ||
| 437 | + "verifier_delta": _compact_value(requirement_delta), | ||
| 438 | + } | ||
| 439 | + experiments.append( | ||
| 440 | + { | ||
| 441 | + "schema_version": 2, | ||
| 442 | + "experiment_id": str(record.get("experiment_id") or ""), | ||
| 443 | + "surface": str(record.get("surface") or ""), | ||
| 444 | + "predicted_rank": record.get("predicted_rank"), | ||
| 445 | + "predicted_score": record.get("predicted_score"), | ||
| 446 | + "prediction": { | ||
| 447 | + "predicted_rank": _first_present(raw_prediction, record, keys=("predicted_rank",)), | ||
| 448 | + "predicted_score": _first_present(raw_prediction, record, keys=("predicted_score",)), | ||
| 449 | + "candidate_patch_excerpt": _compact_text( | ||
| 450 | + _first_present(raw_prediction, record, keys=("candidate_patch_excerpt",)), | ||
| 451 | + 2_000, | ||
| 452 | + ), | ||
| 453 | + "causal_intervention_contracts": _compact_value(contracts), | ||
| 454 | + }, | ||
| 455 | + "observed_outcome": observed_outcome, | ||
| 456 | + "activation": _compact_value(activation), | ||
| 457 | + "causal_intervention_contracts": _compact_value(contracts), | ||
| 458 | + "verifier_delta": _compact_value(requirement_delta), | ||
| 459 | + "candidate_failure_diagnoses": _compact_candidate_failure_diagnoses(diagnoses), | ||
| 460 | + } | ||
| 461 | + ) | ||
| 462 | + return {"case_id": str(feedback.get("case_id") or ""), "experiments": experiments} | ||
| 463 | + | ||
| 464 | + | ||
| 465 | +def _first_present(*values: Mapping[str, Any], keys: Sequence[str]) -> Any: | ||
| 466 | + """Return the first explicitly present alias, including false and zero.""" | ||
| 467 | + for value in values: | ||
| 468 | + for key in keys: | ||
| 469 | + if key in value: | ||
| 470 | + return value.get(key) | ||
| 471 | + return None | ||
| 472 | + | ||
| 473 | + | ||
| 474 | +def _compact_candidate_failure_diagnoses(diagnoses: Sequence[Any]) -> list[dict[str, Any]]: | ||
| 475 | + keys = ( | ||
| 476 | + "summary", | ||
| 477 | + "root_cause", | ||
| 478 | + "target_ref", | ||
| 479 | + "recommendation", | ||
| 480 | + "decision_contract", | ||
| 481 | + "hypothesis_assessment", | ||
| 482 | + "prior_experiment_assessment", | ||
| 483 | + ) | ||
| 484 | + compacted: list[dict[str, Any]] = [] | ||
| 485 | + for diagnosis in diagnoses[:2]: | ||
| 486 | + if not isinstance(diagnosis, Mapping): | ||
| 487 | + continue | ||
| 488 | + item: dict[str, Any] = {} | ||
| 489 | + for key in keys: | ||
| 490 | + if key in diagnosis: | ||
| 491 | + item[key] = _compact_value(diagnosis.get(key)) | ||
| 492 | + compacted.append(item) | ||
| 493 | + return compacted | ||
| 494 | + | ||
| 495 | + | ||
| 496 | +def _first_mapping(*values: Any) -> Mapping[str, Any]: | ||
| 497 | + for value in values: | ||
| 498 | + if isinstance(value, Mapping): | ||
| 499 | + return value | ||
| 500 | + return {} | ||
| 501 | + | ||
| 502 | + | ||
| 503 | +def _compact_score_comparison( | ||
| 504 | + nested: Any, | ||
| 505 | + *, | ||
| 506 | + source: Any, | ||
| 507 | + candidate: Any, | ||
| 508 | + delta: Any, | ||
| 509 | +) -> dict[str, Any]: | ||
| 510 | + nested = nested if isinstance(nested, Mapping) else {} | ||
| 511 | + source_value = nested.get("source") if "source" in nested else source | ||
| 512 | + candidate_value = nested.get("candidate") if "candidate" in nested else candidate | ||
| 513 | + delta_value = nested.get("delta") if "delta" in nested else delta | ||
| 514 | + if not _is_number(delta_value) and _is_number(source_value) and _is_number(candidate_value): | ||
| 515 | + delta_value = float(candidate_value) - float(source_value) | ||
| 516 | + return { | ||
| 517 | + "source": source_value if _is_number(source_value) else None, | ||
| 518 | + "candidate": candidate_value if _is_number(candidate_value) else None, | ||
| 519 | + "delta": delta_value if _is_number(delta_value) else None, | ||
| 520 | + } | ||
| 521 | + | ||
| 522 | + | ||
| 523 | +def _build_trial_record( | ||
| 524 | + trace: Mapping[str, Any], | ||
| 525 | + *, | ||
| 526 | + index: int, | ||
| 527 | + score: Any, | ||
| 528 | + passed: Any, | ||
| 529 | + exit_reason: Any, | ||
| 530 | + critical_evidence_terms: Sequence[str] = (), | ||
| 531 | +) -> dict[str, Any]: | ||
| 532 | + trace_id = str(trace.get("trace_id") or f"trace_{index + 1}") | ||
| 533 | + role = str(trace.get("member_role") or trace.get("role") or "") | ||
| 534 | + messages = trace.get("messages") | ||
| 535 | + messages = messages if isinstance(messages, list) else [] | ||
| 536 | + calls: list[dict[str, Any]] = [] | ||
| 537 | + assistant_outputs: list[dict[str, Any]] = [] | ||
| 538 | + for message in messages: | ||
| 539 | + if not isinstance(message, Mapping): | ||
| 540 | + continue | ||
| 541 | + message_index = message.get("message_index", "") | ||
| 542 | + step_pointer = str(message.get("step_pointer") or "") | ||
| 543 | + content = str(message.get("content") or "").strip() | ||
| 544 | + if str(message.get("role") or "") == "assistant" and content: | ||
| 545 | + assistant_outputs.append( | ||
| 546 | + { | ||
| 547 | + "message_index": message_index, | ||
| 548 | + "step_pointer": step_pointer, | ||
| 549 | + "content": content, | ||
| 550 | + } | ||
| 551 | + ) | ||
| 552 | + raw_calls = message.get("tool_calls") | ||
| 553 | + if not isinstance(raw_calls, list): | ||
| 554 | + continue | ||
| 555 | + for call_index, raw_call in enumerate(raw_calls): | ||
| 556 | + if not isinstance(raw_call, Mapping): | ||
| 557 | + continue | ||
| 558 | + raw_input = raw_call.get("input", "") | ||
| 559 | + raw_output = raw_call.get("output", "") | ||
| 560 | + raw_output_text = str(raw_output or "") | ||
| 561 | + raw_error = str(raw_call.get("error") or "") | ||
| 562 | + evidence_id = f"{trace_id}:message_{message_index}:call_{call_index}" | ||
| 563 | + calls.append( | ||
| 564 | + { | ||
| 565 | + "evidence_id": evidence_id, | ||
| 566 | + "trace_id": trace_id, | ||
| 567 | + "role": role, | ||
| 568 | + "message_index": message_index, | ||
| 569 | + "step_pointer": str(raw_call.get("step_pointer") or step_pointer), | ||
| 570 | + "tool": str(raw_call.get("name") or ""), | ||
| 571 | + "request": _parse_and_compact(raw_input), | ||
| 572 | + "response": _parse_and_compact(raw_output), | ||
| 573 | + "response_evidence": _response_evidence_view( | ||
| 574 | + raw_output_text, | ||
| 575 | + evidence_id=evidence_id, | ||
| 576 | + critical_terms=critical_evidence_terms, | ||
| 577 | + ), | ||
| 578 | + "error": _compact_text(raw_error, 1_000), | ||
| 579 | + "decision_context": _compact_text(content, 600), | ||
| 580 | + "_raw_input": str(raw_input or ""), | ||
| 581 | + "_raw_output": raw_output_text, | ||
| 582 | + } | ||
| 583 | + ) | ||
| 584 | + selected = _select_calls(calls) | ||
| 585 | + public_selected = [_public_call(call) for call in selected] | ||
| 586 | + final_output = assistant_outputs[-1] if assistant_outputs else {} | ||
| 587 | + delivered_output, delivered_reference = _finish_delivery(calls) | ||
| 588 | + if delivered_output: | ||
| 589 | + final_output = {"content": delivered_output, **delivered_reference} | ||
| 590 | + delivery_channel = "finish_tool_request" | ||
| 591 | + else: | ||
| 592 | + delivery_channel = "assistant_message" | ||
| 593 | + sequence = [str(call.get("tool") or "") for call in calls] | ||
| 594 | + return { | ||
| 595 | + "trial_id": trace_id, | ||
| 596 | + "role": role, | ||
| 597 | + "passed": passed if isinstance(passed, bool) else None, | ||
| 598 | + "score": score if _is_number(score) else None, | ||
| 599 | + "exit_reason": str(exit_reason or ""), | ||
| 600 | + "tool_call_count": len(calls), | ||
| 601 | + "tool_sequence": sequence[:_MAX_TOOL_SEQUENCE], | ||
| 602 | + "tool_sequence_truncated": len(sequence) > _MAX_TOOL_SEQUENCE, | ||
| 603 | + "selected_actions": public_selected, | ||
| 604 | + "selection_coverage": { | ||
| 605 | + "policy": "decision_linked_dynamic_budget", | ||
| 606 | + "selected_count": len(selected), | ||
| 607 | + "omitted_count": max(0, len(calls) - len(selected)), | ||
| 608 | + "failed_call_count": sum(1 for call in calls if _call_failed(call)), | ||
| 609 | + "selected_failed_call_count": sum(1 for call in selected if _call_failed(call)), | ||
| 610 | + "state_mutation_call_count": sum(1 for call in calls if _call_mutates_state(call)), | ||
| 611 | + "selected_state_mutation_call_count": sum(1 for call in selected if _call_mutates_state(call)), | ||
| 612 | + "content_evidence_call_count": sum(1 for call in calls if _call_has_content_evidence(call)), | ||
| 613 | + "selected_content_evidence_call_count": sum(1 for call in selected if _call_has_content_evidence(call)), | ||
| 614 | + }, | ||
| 615 | + "final_output": { | ||
| 616 | + **_output_summary(str(final_output.get("content") or "")), | ||
| 617 | + "delivery_channel": delivery_channel, | ||
| 618 | + "evidence_ref": { | ||
| 619 | + "trace_id": trace_id, | ||
| 620 | + "role": role, | ||
| 621 | + "message_index": final_output.get("message_index", ""), | ||
| 622 | + "step_pointer": final_output.get("step_pointer", ""), | ||
| 623 | + }, | ||
| 624 | + }, | ||
| 625 | + "_raw_message_count": len(messages), | ||
| 626 | + "_all_calls": calls, | ||
| 627 | + } | ||
| 628 | + | ||
| 629 | + | ||
| 630 | +def _finish_delivery(calls: list[dict[str, Any]]) -> tuple[str, dict[str, Any]]: | ||
| 631 | + for call in reversed(calls): | ||
| 632 | + if str(call.get("tool") or "").lower() != "finish": | ||
| 633 | + continue | ||
| 634 | + raw_input = str(call.get("_raw_input") or "") | ||
| 635 | + try: | ||
| 636 | + payload = json.loads(raw_input) | ||
| 637 | + except json.JSONDecodeError: | ||
| 638 | + continue | ||
| 639 | + answer = payload.get("answer") if isinstance(payload, Mapping) else None | ||
| 640 | + if isinstance(answer, str) and answer.strip(): | ||
| 641 | + return answer, { | ||
| 642 | + "message_index": call.get("message_index", ""), | ||
| 643 | + "step_pointer": call.get("step_pointer", ""), | ||
| 644 | + } | ||
| 645 | + return "", {} | ||
| 646 | + | ||
| 647 | + | ||
| 648 | +def _select_calls(calls: list[dict[str, Any]]) -> list[dict[str, Any]]: | ||
| 649 | + budget = min( | ||
| 650 | + _MAX_SELECTED_CALLS_PER_TRIAL, | ||
| 651 | + max(_MIN_SELECTED_CALLS_PER_TRIAL, (len(calls) * 2 + 2) // 3), | ||
| 652 | + ) | ||
| 653 | + if len(calls) <= budget: | ||
| 654 | + for call in calls: | ||
| 655 | + call["selection_reasons"] = ["complete_trace"] | ||
| 656 | + return calls | ||
| 657 | + | ||
| 658 | + first_by_tool: dict[str, int] = {} | ||
| 659 | + last_by_tool: dict[str, int] = {} | ||
| 660 | + for index, call in enumerate(calls): | ||
| 661 | + tool = str(call.get("tool") or "") | ||
| 662 | + first_by_tool.setdefault(tool, index) | ||
| 663 | + last_by_tool[tool] = index | ||
| 664 | + | ||
| 665 | + first_indexes = set(first_by_tool.values()) | ||
| 666 | + last_indexes = set(last_by_tool.values()) | ||
| 667 | + | ||
| 668 | + def _reasons(index: int) -> list[str]: | ||
| 669 | + call = calls[index] | ||
| 670 | + reasons: list[str] = [] | ||
| 671 | + if _call_failed(call): | ||
| 672 | + reasons.append("observed_failure") | ||
| 673 | + if _call_mutates_state(call): | ||
| 674 | + reasons.append("state_mutation") | ||
| 675 | + if _call_has_content_evidence(call): | ||
| 676 | + reasons.append("content_evidence") | ||
| 677 | + if index in first_indexes: | ||
| 678 | + reasons.append("first_use_of_tool") | ||
| 679 | + if index in last_indexes: | ||
| 680 | + reasons.append("last_use_of_tool") | ||
| 681 | + if index >= len(calls) - 3: | ||
| 682 | + reasons.append("terminal_window") | ||
| 683 | + return reasons | ||
| 684 | + | ||
| 685 | + ranked = sorted( | ||
| 686 | + range(len(calls)), | ||
| 687 | + key=lambda index: ( | ||
| 688 | + not _call_failed(calls[index]), | ||
| 689 | + not _call_mutates_state(calls[index]), | ||
| 690 | + not _call_has_content_evidence(calls[index]), | ||
| 691 | + index not in first_indexes and index not in last_indexes, | ||
| 692 | + index < len(calls) - 3, | ||
| 693 | + -len(str(calls[index].get("_raw_output") or "")), | ||
| 694 | + index, | ||
| 695 | + ), | ||
| 696 | + ) | ||
| 697 | + selected_indexes = set(ranked[:budget]) | ||
| 698 | + for index in selected_indexes: | ||
| 699 | + calls[index]["selection_reasons"] = _reasons(index) or ["dynamic_budget"] | ||
| 700 | + return [calls[index] for index in sorted(selected_indexes)] | ||
| 701 | + | ||
| 702 | + | ||
| 703 | +def _public_call(call: Mapping[str, Any]) -> dict[str, Any]: | ||
| 704 | + public = { | ||
| 705 | + key: value | ||
| 706 | + for key, value in call.items() | ||
| 707 | + if not key.startswith("_") and (value not in ("", {}, []) or key in {"evidence_id", "tool", "request"}) | ||
| 708 | + } | ||
| 709 | + if not _call_failed(call) and not _call_mutates_state(call): | ||
| 710 | + public.pop("decision_context", None) | ||
| 711 | + return public | ||
| 712 | + | ||
| 713 | + | ||
| 714 | +def _deduplicate_selected_payloads(trials: list[dict[str, Any]]) -> None: | ||
| 715 | + seen: dict[str, str] = {} | ||
| 716 | + for trial in trials: | ||
| 717 | + for call in trial.get("selected_actions", []): | ||
| 718 | + payload = { | ||
| 719 | + "tool": call.get("tool"), | ||
| 720 | + "request": call.get("request"), | ||
| 721 | + "response": call.get("response"), | ||
| 722 | + "error": call.get("error"), | ||
| 723 | + } | ||
| 724 | + fingerprint = hashlib.sha256( | ||
| 725 | + json.dumps(payload, ensure_ascii=False, sort_keys=True, default=str).encode("utf-8") | ||
| 726 | + ).hexdigest() | ||
| 727 | + original = seen.get(fingerprint) | ||
| 728 | + if original: | ||
| 729 | + call["duplicate_of"] = original | ||
| 730 | + call.pop("request", None) | ||
| 731 | + call.pop("response", None) | ||
| 732 | + call.pop("decision_context", None) | ||
| 733 | + else: | ||
| 734 | + seen[fingerprint] = str(call.get("evidence_id") or "") | ||
| 735 | + | ||
| 736 | + | ||
| 737 | +def _tool_contract_observations( | ||
| 738 | + task_contract: Mapping[str, Any], | ||
| 739 | + calls: list[dict[str, Any]], | ||
| 740 | +) -> list[dict[str, Any]]: | ||
| 741 | + schemas: dict[str, dict[str, Any]] = {} | ||
| 742 | + raw_schemas = task_contract.get("tool_schemas") | ||
| 743 | + if isinstance(raw_schemas, list): | ||
| 744 | + for schema in raw_schemas: | ||
| 745 | + if isinstance(schema, Mapping): | ||
| 746 | + name = str(schema.get("name") or "") | ||
| 747 | + if name: | ||
| 748 | + schemas[name] = dict(schema) | ||
| 749 | + | ||
| 750 | + observed_request_fields: dict[str, set[str]] = defaultdict(set) | ||
| 751 | + observed_response_fields: dict[str, set[str]] = defaultdict(set) | ||
| 752 | + observed_response_leaf_fields: dict[str, set[str]] = defaultdict(set) | ||
| 753 | + call_counts: Counter[str] = Counter() | ||
| 754 | + for call in calls: | ||
| 755 | + tool = str(call.get("tool") or "") | ||
| 756 | + if not tool: | ||
| 757 | + continue | ||
| 758 | + call_counts[tool] += 1 | ||
| 759 | + request = call.get("request") | ||
| 760 | + response = call.get("response") | ||
| 761 | + if isinstance(request, Mapping): | ||
| 762 | + observed_request_fields[tool].update(map(str, request)) | ||
| 763 | + if isinstance(response, Mapping): | ||
| 764 | + observed_response_fields[tool].update(map(str, response)) | ||
| 765 | + observed_response_leaf_fields[tool].update(_leaf_field_names(response)) | ||
| 766 | + | ||
| 767 | + observations: list[dict[str, Any]] = [] | ||
| 768 | + for tool in sorted(set(call_counts) | set(schemas)): | ||
| 769 | + schema = schemas.get(tool, {}) | ||
| 770 | + allowed = set(map(str, schema.get("allowed_request_fields", []))) | ||
| 771 | + required = set(map(str, schema.get("required_request_fields", []))) | ||
| 772 | + request_fields = observed_request_fields[tool] | ||
| 773 | + response_fields = observed_response_fields[tool] | ||
| 774 | + response_leaf_fields = observed_response_leaf_fields[tool] | ||
| 775 | + observation = { | ||
| 776 | + "tool": tool, | ||
| 777 | + "call_count": call_counts[tool], | ||
| 778 | + "public_schema_available": bool(schema), | ||
| 779 | + "description": schema.get("description", ""), | ||
| 780 | + "allowed_request_fields": sorted(allowed), | ||
| 781 | + "required_request_fields": sorted(required), | ||
| 782 | + "request_field_contracts": schema.get("request_field_contracts", {}), | ||
| 783 | + "observed_request_fields": sorted(request_fields), | ||
| 784 | + "observed_response_fields": sorted(response_fields), | ||
| 785 | + "observed_response_leaf_fields": sorted(response_leaf_fields), | ||
| 786 | + "response_only_fields": sorted(response_fields - request_fields), | ||
| 787 | + } | ||
| 788 | + if schema: | ||
| 789 | + observation["response_fields_not_in_public_request_schema"] = sorted(response_fields - allowed) | ||
| 790 | + observation["response_leaf_fields_not_in_public_request_schema"] = sorted(response_leaf_fields - allowed) | ||
| 791 | + observation["observed_request_fields_outside_public_schema"] = sorted(request_fields - allowed) | ||
| 792 | + observations.append(observation) | ||
| 793 | + return observations | ||
| 794 | + | ||
| 795 | + | ||
| 796 | +def _cross_trial_contrast(trials: list[dict[str, Any]]) -> dict[str, Any]: | ||
| 797 | + if not trials: | ||
| 798 | + return {"available": False, "reason": "no_normalized_traces"} | ||
| 799 | + sequences = [list(map(str, trial.get("tool_sequence", []))) for trial in trials] | ||
| 800 | + tool_sets = [set(sequence) for sequence in sequences] | ||
| 801 | + passed_ids = [trial["trial_id"] for trial in trials if trial.get("passed") is True] | ||
| 802 | + failed_ids = [trial["trial_id"] for trial in trials if trial.get("passed") is False] | ||
| 803 | + success_tools = set().union( | ||
| 804 | + *(tool_sets[index] for index, trial in enumerate(trials) if trial.get("passed") is True) | ||
| 805 | + ) | ||
| 806 | + failure_tools = set().union( | ||
| 807 | + *(tool_sets[index] for index, trial in enumerate(trials) if trial.get("passed") is False) | ||
| 808 | + ) | ||
| 809 | + stable_tools = set.intersection(*tool_sets) if tool_sets else set() | ||
| 810 | + return { | ||
| 811 | + "available": len(trials) > 1, | ||
| 812 | + "successful_trials": passed_ids, | ||
| 813 | + "failed_trials": failed_ids, | ||
| 814 | + "stable_tools": sorted(stable_tools), | ||
| 815 | + "success_only_tools": sorted(success_tools - failure_tools) if passed_ids and failed_ids else [], | ||
| 816 | + "failure_only_tools": sorted(failure_tools - success_tools) if passed_ids and failed_ids else [], | ||
| 817 | + "first_tool_sequence_divergence": _first_sequence_divergence(trials, sequences), | ||
| 818 | + "terminal_action_variants": _terminal_action_variants(trials), | ||
| 819 | + "final_output_comparison": [ | ||
| 820 | + { | ||
| 821 | + "trial_id": trial["trial_id"], | ||
| 822 | + "passed": trial.get("passed"), | ||
| 823 | + "score": trial.get("score"), | ||
| 824 | + "character_count": trial.get("final_output", {}).get("character_count", 0), | ||
| 825 | + "line_count": trial.get("final_output", {}).get("line_count", 0), | ||
| 826 | + "delivery_channel": trial.get("final_output", {}).get("delivery_channel", ""), | ||
| 827 | + "artifact_mentions": trial.get("final_output", {}).get("artifact_mentions", []), | ||
| 828 | + "evidence_ref": trial.get("final_output", {}).get("evidence_ref", {}), | ||
| 829 | + } | ||
| 830 | + for trial in trials | ||
| 831 | + ], | ||
| 832 | + } | ||
| 833 | + | ||
| 834 | + | ||
| 835 | +def _first_sequence_divergence(trials: list[dict[str, Any]], sequences: list[list[str]]) -> dict[str, Any]: | ||
| 836 | + if len(sequences) < 2: | ||
| 837 | + return {} | ||
| 838 | + max_length = max(map(len, sequences), default=0) | ||
| 839 | + for index in range(max_length): | ||
| 840 | + values = [sequence[index] if index < len(sequence) else "<end>" for sequence in sequences] | ||
| 841 | + if len(set(values)) > 1: | ||
| 842 | + return { | ||
| 843 | + "tool_index": index, | ||
| 844 | + "by_trial": {str(trial["trial_id"]): value for trial, value in zip(trials, values, strict=True)}, | ||
| 845 | + } | ||
| 846 | + return {} | ||
| 847 | + | ||
| 848 | + | ||
| 849 | +def _terminal_action_variants(trials: list[dict[str, Any]]) -> list[dict[str, Any]]: | ||
| 850 | + by_tool: dict[str, list[dict[str, Any]]] = defaultdict(list) | ||
| 851 | + for trial in trials: | ||
| 852 | + actions = trial.get("selected_actions", []) | ||
| 853 | + for action in actions: | ||
| 854 | + tool = str(action.get("tool") or "") | ||
| 855 | + if _is_mutating_tool(tool): | ||
| 856 | + by_tool[tool].append( | ||
| 857 | + { | ||
| 858 | + "trial_id": trial["trial_id"], | ||
| 859 | + "passed": trial.get("passed"), | ||
| 860 | + "score": trial.get("score"), | ||
| 861 | + "request": action.get("request"), | ||
| 862 | + "evidence_id": action.get("evidence_id"), | ||
| 863 | + } | ||
| 864 | + ) | ||
| 865 | + return [{"tool": tool, "variants": variants} for tool, variants in sorted(by_tool.items())] | ||
| 866 | + | ||
| 867 | + | ||
| 868 | +def _normalize_tool_schema(schema: Mapping[str, Any]) -> dict[str, Any]: | ||
| 869 | + function = schema.get("function") | ||
| 870 | + function = function if isinstance(function, Mapping) else schema | ||
| 871 | + name = str(function.get("name") or "") | ||
| 872 | + if not name: | ||
| 873 | + return {} | ||
| 874 | + parameters = function.get("parameters") | ||
| 875 | + parameters = parameters if isinstance(parameters, Mapping) else {} | ||
| 876 | + properties = parameters.get("properties") | ||
| 877 | + properties = properties if isinstance(properties, Mapping) else {} | ||
| 878 | + return { | ||
| 879 | + "name": name, | ||
| 880 | + "description": _compact_text(function.get("description"), 500), | ||
| 881 | + "allowed_request_fields": sorted(map(str, properties)), | ||
| 882 | + "required_request_fields": sorted(map(str, parameters.get("required", []))), | ||
| 883 | + "request_field_contracts": { | ||
| 884 | + str(field): { | ||
| 885 | + key: _compact_value(spec.get(key)) | ||
| 886 | + for key in ("type", "description", "enum", "items") | ||
| 887 | + if isinstance(spec, Mapping) and key in spec | ||
| 888 | + } | ||
| 889 | + for field, spec in properties.items() | ||
| 890 | + }, | ||
| 891 | + } | ||
| 892 | + | ||
| 893 | + | ||
| 894 | +def _judge_dimensions(metadata: Mapping[str, Any]) -> dict[str, Any]: | ||
| 895 | + detail = metadata.get("judge_detail") | ||
| 896 | + if not isinstance(detail, Mapping): | ||
| 897 | + return {} | ||
| 898 | + return {str(key): value for key, value in detail.items() if _is_scalar(value)} | ||
| 899 | + | ||
| 900 | + | ||
| 901 | +def _leaf_field_names(value: Mapping[str, Any], *, depth: int = 0) -> set[str]: | ||
| 902 | + leaves: set[str] = set() | ||
| 903 | + for key, item in value.items(): | ||
| 904 | + name = str(key) | ||
| 905 | + if isinstance(item, Mapping) and depth < 3: | ||
| 906 | + leaves.update(_leaf_field_names(item, depth=depth + 1)) | ||
| 907 | + else: | ||
| 908 | + leaves.add(name) | ||
| 909 | + return leaves | ||
| 910 | + | ||
| 911 | + | ||
| 912 | +def _output_summary(value: str) -> dict[str, Any]: | ||
| 913 | + text = str(value or "") | ||
| 914 | + artifacts = [] | ||
| 915 | + for match in _ARTIFACT_PATTERN.finditer(text): | ||
| 916 | + candidate = match.group(1) | ||
| 917 | + if candidate not in artifacts: | ||
| 918 | + artifacts.append(candidate) | ||
| 919 | + return { | ||
| 920 | + "character_count": len(text), | ||
| 921 | + "line_count": len(text.splitlines()), | ||
| 922 | + "artifact_mentions": artifacts[:12], | ||
| 923 | + "excerpt": _head_tail(text, 1_600), | ||
| 924 | + } | ||
| 925 | + | ||
| 926 | + | ||
| 927 | +def _failed_requirement_terms(metadata: Mapping[str, Any]) -> list[str]: | ||
| 928 | + """Extract bounded anchors from failed evaluator criteria. | ||
| 929 | + | ||
| 930 | + These terms are used only to retain exact public trajectory spans. They do | ||
| 931 | + not become conclusions and do not change the evaluator outcome. | ||
| 932 | + """ | ||
| 933 | + normalized = metadata.get("judge_evidence") | ||
| 934 | + detail = normalized if isinstance(normalized, Mapping) else metadata.get("judge_detail") | ||
| 935 | + detail = detail if isinstance(detail, Mapping) else {} | ||
| 936 | + criteria = detail.get("criteria") | ||
| 937 | + criteria = criteria if isinstance(criteria, list) else [] | ||
| 938 | + values: list[str] = [] | ||
| 939 | + for criterion in criteria: | ||
| 940 | + if not isinstance(criterion, Mapping): | ||
| 941 | + continue | ||
| 942 | + score = criterion.get("score") | ||
| 943 | + if _is_number(score) and float(score) > 0: | ||
| 944 | + continue | ||
| 945 | + values.extend( | ||
| 946 | + str(criterion.get(key) or "") for key in ("criterion_id", "verifier_id", "rationale") if criterion.get(key) | ||
| 947 | + ) | ||
| 948 | + | ||
| 949 | + terms: list[str] = [] | ||
| 950 | + seen: set[str] = set() | ||
| 951 | + for value in values: | ||
| 952 | + for match in _CRITICAL_EVIDENCE_TERM_PATTERN.findall(value): | ||
| 953 | + normalized_term = match.casefold() | ||
| 954 | + if normalized_term in _CRITICAL_EVIDENCE_STOPWORDS or normalized_term in seen: | ||
| 955 | + continue | ||
| 956 | + seen.add(normalized_term) | ||
| 957 | + terms.append(match) | ||
| 958 | + if len(terms) >= 48: | ||
| 959 | + return terms | ||
| 960 | + return terms | ||
| 961 | + | ||
| 962 | + | ||
| 963 | +def extract_critical_evidence_spans( | ||
| 964 | + value: Any, | ||
| 965 | + terms: Sequence[str], | ||
| 966 | + *, | ||
| 967 | + max_spans: int = 3, | ||
| 968 | + max_total_chars: int = 6_000, | ||
| 969 | +) -> list[dict[str, Any]]: | ||
| 970 | + """Return exact line windows tied to failed-requirement terms. | ||
| 971 | + | ||
| 972 | + Unlike the display excerpt, these windows are selected from the raw tool | ||
| 973 | + response before compaction. The returned text never uses an unlabelled | ||
| 974 | + omission marker. | ||
| 975 | + """ | ||
| 976 | + text, projection = _readable_evidence_text(str(value or "")) | ||
| 977 | + normalized_terms = list(dict.fromkeys(str(term).casefold() for term in terms if str(term).strip())) | ||
| 978 | + if not text or not normalized_terms: | ||
| 979 | + return [] | ||
| 980 | + if max_spans <= 0 or max_total_chars <= 0: | ||
| 981 | + return [] | ||
| 982 | + | ||
| 983 | + lines = text.splitlines() or [text] | ||
| 984 | + ranked: list[tuple[int, int, list[str]]] = [] | ||
| 985 | + for index, line in enumerate(lines): | ||
| 986 | + lowered = line.casefold() | ||
| 987 | + matched = [term for term in normalized_terms if term in lowered] | ||
| 988 | + if matched: | ||
| 989 | + ranked.append((len(set(matched)), index, matched)) | ||
| 990 | + if not ranked: | ||
| 991 | + return [] | ||
| 992 | + | ||
| 993 | + selected_lines = sorted(ranked, key=lambda item: (-item[0], item[1]))[:max_spans] | ||
| 994 | + windows: list[tuple[int, int, set[str]]] = [] | ||
| 995 | + for _, index, matched in sorted(selected_lines, key=lambda item: item[1]): | ||
| 996 | + start = max(0, index - 3) | ||
| 997 | + end = min(len(lines), index + 4) | ||
| 998 | + if windows and start <= windows[-1][1]: | ||
| 999 | + previous_start, previous_end, previous_terms = windows[-1] | ||
| 1000 | + windows[-1] = (previous_start, max(previous_end, end), previous_terms | set(matched)) | ||
| 1001 | + else: | ||
| 1002 | + windows.append((start, end, set(matched))) | ||
| 1003 | + | ||
| 1004 | + spans: list[dict[str, Any]] = [] | ||
| 1005 | + remaining = max_total_chars | ||
| 1006 | + for start, end, matched in windows: | ||
| 1007 | + if remaining <= 0: | ||
| 1008 | + break | ||
| 1009 | + raw_span = "\n".join(lines[start:end]) | ||
| 1010 | + span_text, complete = _bounded_critical_span(raw_span, matched, remaining) | ||
| 1011 | + if not span_text: | ||
| 1012 | + continue | ||
| 1013 | + spans.append( | ||
| 1014 | + { | ||
| 1015 | + "source": "raw_tool_response", | ||
| 1016 | + "projection": projection, | ||
| 1017 | + "line_start": start + 1, | ||
| 1018 | + "line_end": end, | ||
| 1019 | + "matched_terms": sorted(matched), | ||
| 1020 | + "text": span_text, | ||
| 1021 | + "window_complete": complete, | ||
| 1022 | + } | ||
| 1023 | + ) | ||
| 1024 | + remaining -= len(span_text) | ||
| 1025 | + return spans | ||
| 1026 | + | ||
| 1027 | + | ||
| 1028 | +def _readable_evidence_text(text: str) -> tuple[str, str]: | ||
| 1029 | + """Project serialized tool-result newlines without semantic summarization.""" | ||
| 1030 | + if "\n" not in text and "\\n" in text: | ||
| 1031 | + return ( | ||
| 1032 | + text.replace("\\r\\n", "\n").replace("\\n", "\n").replace("\\t", "\t"), | ||
| 1033 | + "escaped_newlines_normalized", | ||
| 1034 | + ) | ||
| 1035 | + return text, "verbatim" | ||
| 1036 | + | ||
| 1037 | + | ||
| 1038 | +def _bounded_critical_span(text: str, matched_terms: set[str], limit: int) -> tuple[str, bool]: | ||
| 1039 | + if len(text) <= limit: | ||
| 1040 | + return text, True | ||
| 1041 | + lowered = text.casefold() | ||
| 1042 | + positions = [lowered.find(term) for term in matched_terms if lowered.find(term) >= 0] | ||
| 1043 | + center = min(positions) if positions else len(text) // 2 | ||
| 1044 | + headroom = max(0, limit // 3) | ||
| 1045 | + start = max(0, center - headroom) | ||
| 1046 | + end = min(len(text), start + limit) | ||
| 1047 | + start = max(0, end - limit) | ||
| 1048 | + return text[start:end], False | ||
| 1049 | + | ||
| 1050 | + | ||
| 1051 | +def _response_evidence_view( | ||
| 1052 | + raw_output: str, | ||
| 1053 | + *, | ||
| 1054 | + evidence_id: str, | ||
| 1055 | + critical_terms: Sequence[str], | ||
| 1056 | +) -> dict[str, Any]: | ||
| 1057 | + source_chars = len(raw_output) | ||
| 1058 | + return { | ||
| 1059 | + "raw_evidence_ref": evidence_id, | ||
| 1060 | + "source": "public_normalized_execution_trace.raw_tool_response", | ||
| 1061 | + "source_char_count": source_chars, | ||
| 1062 | + "display_excerpt_complete": source_chars <= 1_200, | ||
| 1063 | + "display_omission_origin": "none" if source_chars <= 1_200 else "analyzer_evidence_compactor", | ||
| 1064 | + "task_agent_observed_display_omission_marker": False, | ||
| 1065 | + "critical_spans": extract_critical_evidence_spans(raw_output, critical_terms), | ||
| 1066 | + } | ||
| 1067 | + | ||
| 1068 | + | ||
| 1069 | +def _parse_and_compact(value: Any) -> Any: | ||
| 1070 | + if not isinstance(value, str): | ||
| 1071 | + return _compact_value(value) | ||
| 1072 | + text = value.strip() | ||
| 1073 | + if not text: | ||
| 1074 | + return "" | ||
| 1075 | + try: | ||
| 1076 | + parsed = json.loads(text) | ||
| 1077 | + except json.JSONDecodeError: | ||
| 1078 | + return _head_tail(text, 2_000) | ||
| 1079 | + return _compact_value(parsed) | ||
| 1080 | + | ||
| 1081 | + | ||
| 1082 | +def _compact_value(value: Any, *, depth: int = 0) -> Any: | ||
| 1083 | + if depth >= 5: | ||
| 1084 | + return _compact_text(value, 500) | ||
| 1085 | + if isinstance(value, Mapping): | ||
| 1086 | + items = list(value.items()) | ||
| 1087 | + compact = {str(key): _compact_value(item, depth=depth + 1) for key, item in items[:30]} | ||
| 1088 | + if len(items) > 30: | ||
| 1089 | + compact["_omitted_field_count"] = len(items) - 30 | ||
| 1090 | + return compact | ||
| 1091 | + if isinstance(value, Sequence) and not isinstance(value, str | bytes | bytearray): | ||
| 1092 | + items = list(value) | ||
| 1093 | + compact_items = [_compact_value(item, depth=depth + 1) for item in items[:30]] | ||
| 1094 | + if len(items) > 30: | ||
| 1095 | + compact_items.append({"_omitted_item_count": len(items) - 30}) | ||
| 1096 | + return compact_items | ||
| 1097 | + if isinstance(value, str): | ||
| 1098 | + return _head_tail(value, 1_200) | ||
| 1099 | + if _is_scalar(value): | ||
| 1100 | + return value | ||
| 1101 | + return _compact_text(value, 500) | ||
| 1102 | + | ||
| 1103 | + | ||
| 1104 | +def _aligned_list(value: Any, expected_length: int) -> list[Any]: | ||
| 1105 | + return list(value) if isinstance(value, list) and len(value) == expected_length else [] | ||
| 1106 | + | ||
| 1107 | + | ||
| 1108 | +def _is_mutating_tool(tool: str) -> bool: | ||
| 1109 | + return bool(_MUTATING_TOOL_PATTERN.search(tool)) | ||
| 1110 | + | ||
| 1111 | + | ||
| 1112 | +def _call_failed(call: Mapping[str, Any]) -> bool: | ||
| 1113 | + if str(call.get("error") or "").strip(): | ||
| 1114 | + return True | ||
| 1115 | + return bool(_FAILED_RESPONSE_PATTERN.search(str(call.get("_raw_output") or call.get("response") or ""))) | ||
| 1116 | + | ||
| 1117 | + | ||
| 1118 | +def _call_mutates_state(call: Mapping[str, Any]) -> bool: | ||
| 1119 | + tool = str(call.get("tool") or "") | ||
| 1120 | + if _is_mutating_tool(tool): | ||
| 1121 | + return True | ||
| 1122 | + if tool.lower() not in {"bash", "shell", "exec", "exec_command"}: | ||
| 1123 | + return False | ||
| 1124 | + return bool(_BASH_MUTATION_PATTERN.search(_call_command(call))) | ||
| 1125 | + | ||
| 1126 | + | ||
| 1127 | +def _call_has_content_evidence(call: Mapping[str, Any]) -> bool: | ||
| 1128 | + if _call_failed(call): | ||
| 1129 | + return False | ||
| 1130 | + tool = str(call.get("tool") or "").lower() | ||
| 1131 | + raw_output = str(call.get("_raw_output") or call.get("response") or "") | ||
| 1132 | + if len(raw_output) < 300: | ||
| 1133 | + return False | ||
| 1134 | + if tool.startswith("read") or tool in {"open_file", "view_file"}: | ||
| 1135 | + return True | ||
| 1136 | + return tool in {"bash", "shell", "exec", "exec_command"} and bool(_BASH_CONTENT_PATTERN.search(_call_command(call))) | ||
| 1137 | + | ||
| 1138 | + | ||
| 1139 | +def _call_command(call: Mapping[str, Any]) -> str: | ||
| 1140 | + raw_input = call.get("_raw_input") or call.get("request") or "" | ||
| 1141 | + if isinstance(raw_input, Mapping): | ||
| 1142 | + return str(raw_input.get("command") or raw_input.get("cmd") or "") | ||
| 1143 | + if not isinstance(raw_input, str): | ||
| 1144 | + return "" | ||
| 1145 | + try: | ||
| 1146 | + parsed = json.loads(raw_input) | ||
| 1147 | + except json.JSONDecodeError: | ||
| 1148 | + return raw_input | ||
| 1149 | + if not isinstance(parsed, Mapping): | ||
| 1150 | + return raw_input | ||
| 1151 | + return str(parsed.get("command") or parsed.get("cmd") or "") | ||
| 1152 | + | ||
| 1153 | + | ||
| 1154 | +def _is_scalar(value: Any) -> bool: | ||
| 1155 | + return value is None or isinstance(value, str | int | float | bool) | ||
| 1156 | + | ||
| 1157 | + | ||
| 1158 | +def _is_number(value: Any) -> bool: | ||
| 1159 | + return isinstance(value, int | float) and not isinstance(value, bool) | ||
| 1160 | + | ||
| 1161 | + | ||
| 1162 | +def _compact_text(value: Any, limit: int) -> str: | ||
| 1163 | + return _head_tail(str(value or ""), limit) | ||
| 1164 | + | ||
| 1165 | + | ||
| 1166 | +def _head_tail(text: str, limit: int) -> str: | ||
| 1167 | + if len(text) <= limit: | ||
| 1168 | + return text | ||
| 1169 | + head = max(1, int(limit * 0.7)) | ||
| 1170 | + tail = max(1, limit - head) | ||
| 1171 | + omitted = len(text) - head - tail | ||
| 1172 | + marker = _COMPACTION_MARKER.format(omitted=omitted) | ||
| 1173 | + return f"{text[:head]}\n{marker}\n{text[-tail:]}" | ||
| 1174 | + | ||
| 1175 | + | ||
| 1176 | +def _read_mapping(path: Path) -> dict[str, Any]: | ||
| 1177 | + if not path.is_file(): | ||
| 1178 | + return {} | ||
| 1179 | + try: | ||
| 1180 | + value = json.loads(path.read_text(encoding="utf-8")) | ||
| 1181 | + except (OSError, json.JSONDecodeError): | ||
| 1182 | + return {} | ||
| 1183 | + return dict(value) if isinstance(value, Mapping) else {} | ||
| @@ -0,0 +1,1672 @@ | |||
| 1 | +# coding: utf-8 | ||
| 2 | +# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. | ||
| 3 | +"""Bounded, read-only evidence acquisition for causal RSI diagnosis.""" | ||
| 4 | + | ||
| 5 | +from __future__ import annotations | ||
| 6 | + | ||
| 7 | +import ast | ||
| 8 | +import hashlib | ||
| 9 | +import json | ||
| 10 | +import math | ||
| 11 | +import os | ||
| 12 | +import re | ||
| 13 | +import zipfile | ||
| 14 | +from collections.abc import Mapping, Sequence | ||
| 15 | +from pathlib import Path | ||
| 16 | +from typing import Any | ||
| 17 | +from xml.etree import ElementTree | ||
| 18 | + | ||
| 19 | +from openjiuwen.rsi.evaluation_result_analyzer.case_reader import CaseAnalysisInput | ||
| 20 | + | ||
| 21 | +_ALLOWED_OPERATIONS = { | ||
| 22 | + "check_relation", | ||
| 23 | + "compare_numeric_change", | ||
| 24 | + "compare_runs", | ||
| 25 | + "inspect_artifact", | ||
| 26 | + "inspect_evaluation", | ||
| 27 | + "read_artifact_window", | ||
| 28 | + "read_repository_file", | ||
| 29 | + "read_event", | ||
| 30 | + "search_repository", | ||
| 31 | + "search_trace", | ||
| 32 | +} | ||
| 33 | +_MAX_HYPOTHESES = 6 | ||
| 34 | +_MAX_REQUESTS = 12 | ||
| 35 | +_MAX_SEARCH_RESULTS = 5 | ||
| 36 | +_MAX_EVENT_CHARS = 12_000 | ||
| 37 | +_MAX_ARTIFACT_FILE_CHARS = 200_000 | ||
| 38 | +_MAX_ARTIFACT_FILES = 100 | ||
| 39 | +_MAX_ARTIFACT_WINDOW_CHARS = 12_000 | ||
| 40 | +_MAX_AUTOMATIC_ARTIFACT_WINDOWS = 12 | ||
| 41 | +_MAX_AUTOMATIC_ARTIFACT_SOURCES = 4 | ||
| 42 | +_MAX_AUTOMATIC_ARTIFACT_CHARS = 96_000 | ||
| 43 | +_MAX_STRUCTURED_ARTIFACTS_PER_REQUEST = 16 | ||
| 44 | +_MAX_REPOSITORY_FILES = 2_000 | ||
| 45 | +_MAX_STRUCTURED_CELLS = 20_000 | ||
| 46 | +_MAX_STRUCTURED_PAGES = 100 | ||
| 47 | +_TEXT_ARTIFACT_SUFFIXES = { | ||
| 48 | + ".csv", | ||
| 49 | + ".diff", | ||
| 50 | + ".json", | ||
| 51 | + ".log", | ||
| 52 | + ".md", | ||
| 53 | + ".patch", | ||
| 54 | + ".txt", | ||
| 55 | + ".xml", | ||
| 56 | + ".yaml", | ||
| 57 | + ".yml", | ||
| 58 | +} | ||
| 59 | +_STRUCTURED_ARTIFACT_SUFFIXES = {".docx", ".pdf", ".pptx", ".xlsx"} | ||
| 60 | +_STRUCTURED_QUERY_SUFFIX_HINTS = { | ||
| 61 | + ".docx": ("docx", "word", "paragraph", "document", "文档", "段落"), | ||
| 62 | + ".pdf": ("pdf", "page", "document", "文档", "页面"), | ||
| 63 | + ".pptx": ("pptx", "powerpoint", "slide", "deck", "presentation", "幻灯片", "演示"), | ||
| 64 | + ".xlsx": ( | ||
| 65 | + "xlsx", | ||
| 66 | + "excel", | ||
| 67 | + "workbook", | ||
| 68 | + "worksheet", | ||
| 69 | + "spreadsheet", | ||
| 70 | + "cell", | ||
| 71 | + "formula", | ||
| 72 | + "工作簿", | ||
| 73 | + "工作表", | ||
| 74 | + "单元格", | ||
| 75 | + "公式", | ||
| 76 | + ), | ||
| 77 | +} | ||
| 78 | +_TERM_PATTERN = re.compile(r"[A-Za-z][A-Za-z0-9_.:/-]{1,}|\d+(?:\.\d+)?|[\u4e00-\u9fff]{2,8}") | ||
| 79 | +_EXPLICIT_NUMERIC_DELTA_PATTERN = re.compile( | ||
| 80 | + r"(?:\b(?:before[-\s]*(?:vs\.?|versus|and)[-\s]*after|numeric\s+(?:change|delta)|" | ||
| 81 | + r"formula\s+(?:change|delta))\b|(?:数值|公式)(?:变化|差值)|前后(?:数值|公式))", | ||
| 82 | + re.IGNORECASE, | ||
| 83 | +) | ||
| 84 | +_IMPLICIT_NUMERIC_DELTA_PATTERN = re.compile( | ||
| 85 | + r"(?:[%％]|\b(?:percentage\s+points?|percent|delta|difference|increase[sd]?|decrease[sd]?|" | ||
| 86 | + r"subtract(?:ed|ing)?|add(?:ed|ing)?|formula|ratio|rate)\b|(?:百分点|百分比|增(?:加|长)|" | ||
| 87 | + r"减少|差值|公式|比率|比例))", | ||
| 88 | + re.IGNORECASE, | ||
| 89 | +) | ||
| 90 | +_STOPWORDS = { | ||
| 91 | + "about", | ||
| 92 | + "after", | ||
| 93 | + "agent", | ||
| 94 | + "before", | ||
| 95 | + "case", | ||
| 96 | + "complete", | ||
| 97 | + "content", | ||
| 98 | + "document", | ||
| 99 | + "evidence", | ||
| 100 | + "failure", | ||
| 101 | + "file", | ||
| 102 | + "from", | ||
| 103 | + "full", | ||
| 104 | + "inspect", | ||
| 105 | + "into", | ||
| 106 | + "missing", | ||
| 107 | + "output", | ||
| 108 | + "result", | ||
| 109 | + "read", | ||
| 110 | + "show", | ||
| 111 | + "should", | ||
| 112 | + "source", | ||
| 113 | + "task", | ||
| 114 | + "that", | ||
| 115 | + "the", | ||
| 116 | + "this", | ||
| 117 | + "tool", | ||
| 118 | + "trace", | ||
| 119 | + "with", | ||
| 120 | +} | ||
| 121 | +_ABSENCE_CLAIM_PATTERN = re.compile( | ||
| 122 | + r"\b(?:absent|absence|missing|lacks?|without|does\s+not\s+(?:contain|include|show)|" | ||
| 123 | + r"no\s+(?:evidence|record|field|entry|occurrence))\b|(?:缺少|缺失|不存在|未包含|没有)", | ||
| 124 | + re.IGNORECASE, | ||
| 125 | +) | ||
| 126 | +_EXISTENCE_CLAIM_PATTERN = re.compile( | ||
| 127 | + r"\b(?:contains?|includes?|present|exists?|records?|shows?|states?)\b|(?:包含|存在|记录|显示|说明)", | ||
| 128 | + re.IGNORECASE, | ||
| 129 | +) | ||
| 130 | + | ||
| 131 | + | ||
| 132 | +def causal_hypothesis_semantic_id(claim: str, falsified_if: str) -> str: | ||
| 133 | + """Return a stable identity for one causal statement, independent of local labels.""" | ||
| 134 | + canonical = json.dumps( | ||
| 135 | + { | ||
| 136 | + "claim": _normalized_semantics(claim), | ||
| 137 | + "falsified_if": _normalized_semantics(falsified_if), | ||
| 138 | + }, | ||
| 139 | + ensure_ascii=True, | ||
| 140 | + sort_keys=True, | ||
| 141 | + separators=(",", ":"), | ||
| 142 | + ) | ||
| 143 | + return f"chs:{hashlib.sha256(canonical.encode('utf-8')).hexdigest()[:24]}" | ||
| 144 | + | ||
| 145 | + | ||
| 146 | +def normalize_causal_investigation( | ||
| 147 | + value: Mapping[str, Any] | None, | ||
| 148 | + *, | ||
| 149 | + failed_requirement_ids: Sequence[str] = (), | ||
| 150 | + max_requests: int = _MAX_REQUESTS, | ||
| 151 | + max_hypotheses: int = _MAX_HYPOTHESES, | ||
| 152 | + min_hypotheses: int = 1, | ||
| 153 | + min_hypotheses_per_requirement: int = 0, | ||
| 154 | + require_evidence_per_hypothesis: bool = False, | ||
| 155 | +) -> dict[str, Any] | None: | ||
| 156 | + """Validate and bound a model-proposed causal investigation plan.""" | ||
| 157 | + if not isinstance(value, Mapping): | ||
| 158 | + return None | ||
| 159 | + raw = value.get("causal_investigation") | ||
| 160 | + if not isinstance(raw, Mapping): | ||
| 161 | + raw = value.get("investigation") | ||
| 162 | + if not isinstance(raw, Mapping): | ||
| 163 | + raw = value | ||
| 164 | + if not isinstance(raw, Mapping): | ||
| 165 | + return None | ||
| 166 | + hypothesis_limit = max(1, min(_MAX_HYPOTHESES, int(max_hypotheses))) | ||
| 167 | + raw_hypotheses = raw.get("hypotheses") | ||
| 168 | + if not isinstance(raw_hypotheses, list) or not raw_hypotheses: | ||
| 169 | + return None | ||
| 170 | + | ||
| 171 | + allowed_requirements = {str(item) for item in failed_requirement_ids if str(item)} | ||
| 172 | + hypotheses: list[dict[str, Any]] = [] | ||
| 173 | + hypothesis_ids: set[str] = set() | ||
| 174 | + hypothesis_semantics: set[tuple[str, str]] = set() | ||
| 175 | + raw_requests: list[tuple[str, Mapping[str, Any]]] = [] | ||
| 176 | + for index, item in enumerate(raw_hypotheses[:hypothesis_limit], start=1): | ||
| 177 | + if not isinstance(item, Mapping): | ||
| 178 | + continue | ||
| 179 | + claim = str(item.get("claim", "") or "").strip() | ||
| 180 | + falsified_if = str(item.get("falsified_if", "") or "").strip() | ||
| 181 | + if not claim or not falsified_if: | ||
| 182 | + continue | ||
| 183 | + semantics = (_normalized_semantics(claim), _normalized_semantics(falsified_if)) | ||
| 184 | + if semantics in hypothesis_semantics: | ||
| 185 | + continue | ||
| 186 | + hypothesis_semantics.add(semantics) | ||
| 187 | + hypothesis_id = str(item.get("hypothesis_id", "") or f"h{index}").strip() | ||
| 188 | + if not hypothesis_id or hypothesis_id in hypothesis_ids: | ||
| 189 | + hypothesis_id = f"h{index}" | ||
| 190 | + hypothesis_ids.add(hypothesis_id) | ||
| 191 | + explains = _string_list(item.get("explains_requirement_ids")) | ||
| 192 | + if allowed_requirements: | ||
| 193 | + explains = [requirement_id for requirement_id in explains if requirement_id in allowed_requirements] | ||
| 194 | + requests = item.get("evidence_requests") | ||
| 195 | + raw_hypothesis_requests = requests if isinstance(requests, list) else [] | ||
| 196 | + declared_numeric_check = item.get("numeric_change_check_required") | ||
| 197 | + # A model may underestimate its own verification obligation. Explicit | ||
| 198 | + # ``false`` therefore cannot disable a controller-detected numeric | ||
| 199 | + # before/after claim. This is a control-plane decision, not a prompt | ||
| 200 | + # preference: textual arithmetic is never accepted as execution. | ||
| 201 | + hypothesis_text = f"{claim}\n{falsified_if}" | ||
| 202 | + numeric_change_check_required = bool(declared_numeric_check) or bool( | ||
| 203 | + _EXPLICIT_NUMERIC_DELTA_PATTERN.search(hypothesis_text) | ||
| 204 | + or _IMPLICIT_NUMERIC_DELTA_PATTERN.search(hypothesis_text) | ||
| 205 | + ) | ||
| 206 | + if any( | ||
| 207 | + isinstance(request, Mapping) and request.get("operation") == "compare_numeric_change" | ||
| 208 | + for request in raw_hypothesis_requests | ||
| 209 | + ): | ||
| 210 | + numeric_change_check_required = True | ||
| 211 | + hypotheses.append( | ||
| 212 | + { | ||
| 213 | + "hypothesis_id": hypothesis_id, | ||
| 214 | + "hypothesis_semantic_id": causal_hypothesis_semantic_id(claim, falsified_if), | ||
| 215 | + "claim": claim, | ||
| 216 | + "explains_requirement_ids": explains, | ||
| 217 | + "current_support": _string_list(item.get("current_support"))[:8], | ||
| 218 | + "falsified_if": falsified_if, | ||
| 219 | + "numeric_change_check_required": numeric_change_check_required, | ||
| 220 | + } | ||
| 221 | + ) | ||
| 222 | + raw_requests.extend( | ||
| 223 | + (hypothesis_id, request) for request in raw_hypothesis_requests if isinstance(request, Mapping) | ||
| 224 | + ) | ||
| 225 | + | ||
| 226 | + top_level_requests = raw.get("evidence_requests") | ||
| 227 | + if isinstance(top_level_requests, list): | ||
| 228 | + raw_requests.extend(("", request) for request in top_level_requests if isinstance(request, Mapping)) | ||
| 229 | + if len(hypotheses) < max(1, min(hypothesis_limit, int(min_hypotheses))): | ||
| 230 | + return None | ||
| 231 | + required_alternatives = max(0, min(hypothesis_limit, int(min_hypotheses_per_requirement))) | ||
| 232 | + if allowed_requirements and required_alternatives: | ||
| 233 | + hypothesis_coverage = { | ||
| 234 | + requirement_id: sum(requirement_id in hypothesis["explains_requirement_ids"] for hypothesis in hypotheses) | ||
| 235 | + for requirement_id in allowed_requirements | ||
| 236 | + } | ||
| 237 | + if any(count < required_alternatives for count in hypothesis_coverage.values()): | ||
| 238 | + return None | ||
| 239 | + | ||
| 240 | + request_limit = min(_MAX_REQUESTS, max(0, int(max_requests))) | ||
| 241 | + requests: list[dict[str, Any]] = [] | ||
| 242 | + request_index_by_fingerprint: dict[str, int] = {} | ||
| 243 | + used_request_ids: set[str] = set() | ||
| 244 | + for default_hypothesis_id, request in raw_requests: | ||
| 245 | + normalized = _normalize_request( | ||
| 246 | + request, | ||
| 247 | + default_hypothesis_id=default_hypothesis_id, | ||
| 248 | + known_hypothesis_ids=hypothesis_ids, | ||
| 249 | + index=len(requests) + 1, | ||
| 250 | + ) | ||
| 251 | + if normalized is None: | ||
| 252 | + continue | ||
| 253 | + fingerprint = _request_execution_fingerprint(normalized) | ||
| 254 | + duplicate_index = request_index_by_fingerprint.get(fingerprint) | ||
| 255 | + if duplicate_index is not None: | ||
| 256 | + existing = requests[duplicate_index] | ||
| 257 | + existing["hypothesis_ids"] = list( | ||
| 258 | + dict.fromkeys( | ||
| 259 | + [ | ||
| 260 | + *_string_list(existing.get("hypothesis_ids")), | ||
| 261 | + *_string_list(normalized.get("hypothesis_ids")), | ||
| 262 | + ] | ||
| 263 | + ) | ||
| 264 | + ) | ||
| 265 | + continue | ||
| 266 | + if request_limit == 0 or len(requests) >= request_limit: | ||
| 267 | + # Keep scanning so later declarations can still bind another | ||
| 268 | + # hypothesis to an already retained shared probe. | ||
| 269 | + continue | ||
| 270 | + request_id = str(normalized.get("request_id", "") or f"q{len(requests) + 1}") | ||
| 271 | + if request_id in used_request_ids: | ||
| 272 | + stem = request_id | ||
| 273 | + suffix = 2 | ||
| 274 | + while request_id in used_request_ids: | ||
| 275 | + request_id = f"{stem}_{suffix}" | ||
| 276 | + suffix += 1 | ||
| 277 | + normalized["request_id"] = request_id | ||
| 278 | + used_request_ids.add(request_id) | ||
| 279 | + request_index_by_fingerprint[fingerprint] = len(requests) | ||
| 280 | + requests.append(normalized) | ||
| 281 | + | ||
| 282 | + if require_evidence_per_hypothesis: | ||
| 283 | + covered: set[str] = set() | ||
| 284 | + for request in requests: | ||
| 285 | + covered.update(_string_list(request.get("hypothesis_ids"))) | ||
| 286 | + if not hypothesis_ids.issubset(covered): | ||
| 287 | + return None | ||
| 288 | + | ||
| 289 | + hypotheses_by_id = {item["hypothesis_id"]: item for item in hypotheses} | ||
| 290 | + for request in requests: | ||
| 291 | + if request.get("operation") != "inspect_artifact" or request.get("proof_obligation"): | ||
| 292 | + continue | ||
| 293 | + claim_text = " ".join( | ||
| 294 | + str(hypotheses_by_id[hypothesis_id].get("claim", "") or "") | ||
| 295 | + for hypothesis_id in _string_list(request.get("hypothesis_ids")) | ||
| 296 | + if hypothesis_id in hypotheses_by_id | ||
| 297 | + ) | ||
| 298 | + request["proof_obligation"] = ( | ||
| 299 | + "absence" | ||
| 300 | + if _ABSENCE_CLAIM_PATTERN.search(claim_text) | ||
| 301 | + else "existence" | ||
| 302 | + if claim_text and _EXISTENCE_CLAIM_PATTERN.search(claim_text) | ||
| 303 | + else "coverage" | ||
| 304 | + ) | ||
| 305 | + | ||
| 306 | + return { | ||
| 307 | + "schema_version": 1, | ||
| 308 | + "hypotheses": hypotheses, | ||
| 309 | + "evidence_requests": requests, | ||
| 310 | + "ready_without_more_evidence": bool(raw.get("ready_without_more_evidence")) and not requests, | ||
| 311 | + } | ||
| 312 | + | ||
| 313 | + | ||
| 314 | +def _request_execution_fingerprint(request: Mapping[str, Any]) -> str: | ||
| 315 | + """Identify one controller operation independently of model-local labels.""" | ||
| 316 | + execution = {key: value for key, value in request.items() if key not in {"request_id", "hypothesis_ids", "purpose"}} | ||
| 317 | + return json.dumps(execution, ensure_ascii=True, sort_keys=True, separators=(",", ":")) | ||
| 318 | + | ||
| 319 | + | ||
| 320 | +def execute_causal_investigation( | ||
| 321 | + case: CaseAnalysisInput, | ||
| 322 | + investigation: Mapping[str, Any], | ||
| 323 | + *, | ||
| 324 | + prior_candidate_feedback: Mapping[str, Any] | None = None, | ||
| 325 | + evidence_root: str | Path | None = None, | ||
| 326 | +) -> dict[str, Any]: | ||
| 327 | + """Execute only controller-owned evidence operations for one public case.""" | ||
| 328 | + trace_data = _read_json(Path(case.result_path).parent / "judge" / "normalized_trace.json") | ||
| 329 | + events = _trace_events(trace_data) | ||
| 330 | + repository_dir = _repository_dir(evidence_root) | ||
| 331 | + discovered_repository_paths: set[str] = set() | ||
| 332 | + artifact_text_cache: dict[Path, str] = {} | ||
| 333 | + results: list[dict[str, Any]] = [] | ||
| 334 | + for request in investigation.get("evidence_requests", []): | ||
| 335 | + if not isinstance(request, Mapping): | ||
| 336 | + continue | ||
| 337 | + operation = str(request.get("operation", "") or "") | ||
| 338 | + if operation == "check_relation": | ||
| 339 | + evidence = _check_relation(request) | ||
| 340 | + elif operation == "compare_numeric_change": | ||
| 341 | + evidence = _compare_numeric_change(request) | ||
| 342 | + elif operation == "search_trace": | ||
| 343 | + evidence = _search_trace(events, request) | ||
| 344 | + elif operation == "read_event": | ||
| 345 | + evidence = _read_event(events, request) | ||
| 346 | + elif operation == "inspect_artifact": | ||
| 347 | + evidence = _inspect_artifact(case, request, text_cache=artifact_text_cache) | ||
| 348 | + elif operation == "read_artifact_window": | ||
| 349 | + evidence = _read_artifact_window(case, request, text_cache=artifact_text_cache) | ||
| 350 | + elif operation == "inspect_evaluation": | ||
| 351 | + evidence = _inspect_evaluation(case, request) | ||
| 352 | + elif operation == "search_repository": | ||
| 353 | + evidence = _search_repository(repository_dir, request) | ||
| 354 | + discovered_repository_paths.update( | ||
| 355 | + str(item.get("relative_path", "") or "") | ||
| 356 | + for item in evidence.get("files", []) | ||
| 357 | + if isinstance(item, Mapping) and str(item.get("relative_path", "") or "") | ||
| 358 | + ) | ||
| 359 | + elif operation == "read_repository_file": | ||
| 360 | + evidence = _read_repository_file( | ||
| 361 | + repository_dir, | ||
| 362 | + request, | ||
| 363 | + discovered_paths=discovered_repository_paths, | ||
| 364 | + ) | ||
| 365 | + elif operation == "compare_runs": | ||
| 366 | + evidence = _compare_runs(prior_candidate_feedback, request) | ||
| 367 | + else: | ||
| 368 | + continue | ||
| 369 | + results.append( | ||
| 370 | + { | ||
| 371 | + "request_id": str(request.get("request_id", "") or ""), | ||
| 372 | + "hypothesis_ids": _string_list(request.get("hypothesis_ids")), | ||
| 373 | + "operation": operation, | ||
| 374 | + "purpose": str(request.get("purpose", "") or ""), | ||
| 375 | + "proof_obligation": str(request.get("proof_obligation", "") or ""), | ||
| 376 | + **evidence, | ||
| 377 | + } | ||
| 378 | + ) | ||
| 379 | + | ||
| 380 | + automatic_requests, closure_results, closure = _close_incomplete_artifact_evidence( | ||
| 381 | + case, | ||
| 382 | + investigation, | ||
| 383 | + results, | ||
| 384 | + text_cache=artifact_text_cache, | ||
| 385 | + ) | ||
| 386 | + results.extend(closure_results) | ||
| 387 | + | ||
| 388 | + return { | ||
| 389 | + "schema_version": 1, | ||
| 390 | + "policy": { | ||
| 391 | + "controller_owned": True, | ||
| 392 | + "read_only": True, | ||
| 393 | + "arbitrary_shell_or_path_access": False, | ||
| 394 | + "display_omissions_are_not_task_agent_observations": True, | ||
| 395 | + }, | ||
| 396 | + "hypotheses": list(investigation.get("hypotheses", [])), | ||
| 397 | + "request_count": len(investigation.get("evidence_requests", [])) + len(automatic_requests), | ||
| 398 | + "model_request_count": len(investigation.get("evidence_requests", [])), | ||
| 399 | + "automatic_request_count": len(automatic_requests), | ||
| 400 | + "automatic_requests": automatic_requests, | ||
| 401 | + "completed_request_count": len(results), | ||
| 402 | + "artifact_evidence_closure": closure, | ||
| 403 | + "results": results, | ||
| 404 | + } | ||
| 405 | + | ||
| 406 | + | ||
| 407 | +def _close_incomplete_artifact_evidence( | ||
| 408 | + case: CaseAnalysisInput, | ||
| 409 | + investigation: Mapping[str, Any], | ||
| 410 | + initial_results: Sequence[Mapping[str, Any]], | ||
| 411 | + *, | ||
| 412 | + text_cache: dict[Path, str], | ||
| 413 | +) -> tuple[list[dict[str, Any]], list[dict[str, Any]], dict[str, Any]]: | ||
| 414 | + """Deterministically complete bounded artifact sources exposed by search. | ||
| 415 | + | ||
| 416 | + ``inspect_artifact`` is discovery, not proof that content is absent. Once a | ||
| 417 | + causal request selects a physical source but returns an incomplete excerpt, | ||
| 418 | + the controller owns the mechanical continuation. This avoids asking the | ||
| 419 | + model to rediscover an already known source/offset and gives absence claims | ||
| 420 | + continuous coverage from character zero to EOF. | ||
| 421 | + """ | ||
| 422 | + existing_ids = { | ||
| 423 | + str(item.get("request_id", "") or "") for item in initial_results if str(item.get("request_id", "") or "") | ||
| 424 | + } | ||
| 425 | + # Explicit windows count toward coverage and must not be read again. | ||
| 426 | + covered_ranges: dict[str, list[tuple[int, int]]] = {} | ||
| 427 | + source_counts: dict[str, int] = {} | ||
| 428 | + for item in initial_results: | ||
| 429 | + if str(item.get("operation", "") or "") != "read_artifact_window": | ||
| 430 | + continue | ||
| 431 | + if str(item.get("availability", "") or "") != "available": | ||
| 432 | + continue | ||
| 433 | + source = str(item.get("source", "") or "") | ||
| 434 | + start = _optional_nonnegative_int(item.get("source_char_start")) | ||
| 435 | + end = _optional_nonnegative_int(item.get("source_char_end")) | ||
| 436 | + count = _optional_nonnegative_int(item.get("source_char_count")) | ||
| 437 | + if not source or start is None or end is None: | ||
| 438 | + continue | ||
| 439 | + if end >= start: | ||
| 440 | + covered_ranges.setdefault(source, []).append((start, end)) | ||
| 441 | + if count is not None: | ||
| 442 | + source_counts[source] = count | ||
| 443 | + | ||
| 444 | + candidates: list[dict[str, Any]] = [] | ||
| 445 | + candidate_by_source: dict[str, dict[str, Any]] = {} | ||
| 446 | + for item in initial_results: | ||
| 447 | + if str(item.get("operation", "") or "") != "inspect_artifact": | ||
| 448 | + continue | ||
| 449 | + if str(item.get("availability", "") or "") != "available": | ||
| 450 | + continue | ||
| 451 | + hypothesis_ids = _string_list(item.get("hypothesis_ids")) | ||
| 452 | + if not hypothesis_ids: | ||
| 453 | + continue | ||
| 454 | + if str(item.get("proof_obligation", "") or "") == "existence": | ||
| 455 | + # A physical match is a complete witness for an existence claim. | ||
| 456 | + # Reading unrelated tail content cannot strengthen that obligation. | ||
| 457 | + continue | ||
| 458 | + matches = item.get("matches", []) | ||
| 459 | + for match in matches[:1] if isinstance(matches, list) else []: | ||
| 460 | + if not isinstance(match, Mapping): | ||
| 461 | + continue | ||
| 462 | + source = str(match.get("source", "") or "") | ||
| 463 | + if not source: | ||
| 464 | + continue | ||
| 465 | + raw_spans = match.get("exact_spans", []) | ||
| 466 | + spans = raw_spans if isinstance(raw_spans, list) else [] | ||
| 467 | + incomplete = any(isinstance(span, Mapping) and not bool(span.get("window_complete")) for span in spans) | ||
| 468 | + if not incomplete: | ||
| 469 | + continue | ||
| 470 | + parent_request_id = str(item.get("request_id", "") or "") | ||
| 471 | + existing_candidate = candidate_by_source.get(source) | ||
| 472 | + if existing_candidate is not None: | ||
| 473 | + existing_candidate["hypothesis_ids"] = list( | ||
| 474 | + dict.fromkeys([*existing_candidate["hypothesis_ids"], *hypothesis_ids]) | ||
| 475 | + ) | ||
| 476 | + existing_candidate["parent_request_ids"] = list( | ||
| 477 | + dict.fromkeys([*existing_candidate["parent_request_ids"], parent_request_id]) | ||
| 478 | + ) | ||
| 479 | + continue | ||
| 480 | + candidate = { | ||
| 481 | + "source": source, | ||
| 482 | + "logical_source": str(match.get("logical_source", "") or source), | ||
| 483 | + "hypothesis_ids": hypothesis_ids, | ||
| 484 | + "parent_request_id": parent_request_id, | ||
| 485 | + "parent_request_ids": [parent_request_id] if parent_request_id else [], | ||
| 486 | + "purpose": str(item.get("purpose", "") or ""), | ||
| 487 | + } | ||
| 488 | + candidate_by_source[source] = candidate | ||
| 489 | + candidates.append(candidate) | ||
| 490 | + if len(candidates) >= _MAX_AUTOMATIC_ARTIFACT_SOURCES: | ||
| 491 | + break | ||
| 492 | + if len(candidates) >= _MAX_AUTOMATIC_ARTIFACT_SOURCES: | ||
| 493 | + break | ||
| 494 | + | ||
| 495 | + automatic_requests: list[dict[str, Any]] = [] | ||
| 496 | + closure_results: list[dict[str, Any]] = [] | ||
| 497 | + source_records: list[dict[str, Any]] = [] | ||
| 498 | + total_chars = 0 | ||
| 499 | + window_budget = _MAX_AUTOMATIC_ARTIFACT_WINDOWS | ||
| 500 | + for candidate in candidates: | ||
| 501 | + source = candidate["source"] | ||
| 502 | + cursor = _continuous_prefix_end(covered_ranges.get(source, [])) | ||
| 503 | + completed = bool(source_counts.get(source) is not None and cursor >= source_counts[source]) | ||
| 504 | + source_windows = 0 | ||
| 505 | + source_chars = 0 | ||
| 506 | + last_reason = "already_covered" if completed else "" | ||
| 507 | + while not completed and window_budget > 0 and total_chars < _MAX_AUTOMATIC_ARTIFACT_CHARS: | ||
| 508 | + remaining_chars = _MAX_AUTOMATIC_ARTIFACT_CHARS - total_chars | ||
| 509 | + max_chars = min(_MAX_ARTIFACT_WINDOW_CHARS, remaining_chars) | ||
| 510 | + request_id = _unique_automatic_request_id( | ||
| 511 | + candidate["parent_request_id"], | ||
| 512 | + source, | ||
| 513 | + cursor, | ||
| 514 | + existing_ids, | ||
| 515 | + ) | ||
| 516 | + request = { | ||
| 517 | + "request_id": request_id, | ||
| 518 | + "hypothesis_ids": list(candidate["hypothesis_ids"]), | ||
| 519 | + "operation": "read_artifact_window", | ||
| 520 | + "relative_path": source, | ||
| 521 | + "source_char_start": cursor, | ||
| 522 | + "max_chars": max_chars, | ||
| 523 | + "purpose": ( | ||
| 524 | + "controller-owned continuation of an incomplete artifact search; " | ||
| 525 | + "establish continuous source coverage before reasoning about absence" | ||
| 526 | + ), | ||
| 527 | + "automatic": True, | ||
| 528 | + "parent_request_id": candidate["parent_request_id"], | ||
| 529 | + "parent_request_ids": list(candidate["parent_request_ids"]), | ||
| 530 | + } | ||
| 531 | + evidence = _read_artifact_window(case, request, text_cache=text_cache) | ||
| 532 | + result = { | ||
| 533 | + **request, | ||
| 534 | + **evidence, | ||
| 535 | + } | ||
| 536 | + automatic_requests.append(request) | ||
| 537 | + closure_results.append(result) | ||
| 538 | + existing_ids.add(request_id) | ||
| 539 | + window_budget -= 1 | ||
| 540 | + source_windows += 1 | ||
| 541 | + if str(evidence.get("availability", "") or "") != "available": | ||
| 542 | + last_reason = str(evidence.get("reason", "") or evidence.get("availability", "") or "unavailable") | ||
| 543 | + break | ||
| 544 | + start = _optional_nonnegative_int(evidence.get("source_char_start")) | ||
| 545 | + end = _optional_nonnegative_int(evidence.get("source_char_end")) | ||
| 546 | + count = _optional_nonnegative_int(evidence.get("source_char_count")) | ||
| 547 | + if start is None or end is None: | ||
| 548 | + last_reason = "non_contiguous_controller_window" | ||
| 549 | + break | ||
| 550 | + if count is None: | ||
| 551 | + last_reason = "non_contiguous_controller_window" | ||
| 552 | + break | ||
| 553 | + if start != cursor or end <= start: | ||
| 554 | + last_reason = "non_contiguous_controller_window" | ||
| 555 | + break | ||
| 556 | + read_chars = end - start | ||
| 557 | + total_chars += read_chars | ||
| 558 | + source_chars += read_chars | ||
| 559 | + covered_ranges.setdefault(source, []).append((start, end)) | ||
| 560 | + source_counts[source] = count | ||
| 561 | + next_cursor = _continuous_prefix_end(covered_ranges[source]) | ||
| 562 | + if next_cursor <= cursor: | ||
| 563 | + last_reason = "no_forward_progress" | ||
| 564 | + break | ||
| 565 | + cursor = next_cursor | ||
| 566 | + completed = cursor >= count | ||
| 567 | + last_reason = "complete" if completed else "continuation_required" | ||
| 568 | + if not completed and not last_reason: | ||
| 569 | + last_reason = "budget_exhausted" | ||
| 570 | + if not completed and (window_budget <= 0 or total_chars >= _MAX_AUTOMATIC_ARTIFACT_CHARS): | ||
| 571 | + last_reason = "budget_exhausted" | ||
| 572 | + source_records.append( | ||
| 573 | + { | ||
| 574 | + "source": source, | ||
| 575 | + "logical_source": candidate["logical_source"], | ||
| 576 | + "parent_request_id": candidate["parent_request_id"], | ||
| 577 | + "continuous_source_char_end": cursor, | ||
| 578 | + "source_char_count": source_counts.get(source), | ||
| 579 | + "window_count": source_windows, | ||
| 580 | + "read_char_count": source_chars, | ||
| 581 | + "complete": completed, | ||
| 582 | + "status": last_reason, | ||
| 583 | + } | ||
| 584 | + ) | ||
| 585 | + | ||
| 586 | + attempted = bool(candidates) | ||
| 587 | + all_complete = attempted and all(bool(item.get("complete")) for item in source_records) | ||
| 588 | + budget_exhausted = any(item.get("status") == "budget_exhausted" for item in source_records) | ||
| 589 | + return ( | ||
| 590 | + automatic_requests, | ||
| 591 | + closure_results, | ||
| 592 | + { | ||
| 593 | + "attempted": attempted, | ||
| 594 | + "status": ( | ||
| 595 | + "completed" | ||
| 596 | + if all_complete | ||
| 597 | + else "budget_exhausted" | ||
| 598 | + if budget_exhausted | ||
| 599 | + else "incomplete" | ||
| 600 | + if attempted | ||
| 601 | + else "not_needed" | ||
| 602 | + ), | ||
| 603 | + "candidate_source_count": len(candidates), | ||
| 604 | + "completed_source_count": sum(bool(item.get("complete")) for item in source_records), | ||
| 605 | + "automatic_window_count": len(automatic_requests), | ||
| 606 | + "read_char_count": total_chars, | ||
| 607 | + "limits": { | ||
| 608 | + "max_sources": _MAX_AUTOMATIC_ARTIFACT_SOURCES, | ||
| 609 | + "max_windows": _MAX_AUTOMATIC_ARTIFACT_WINDOWS, | ||
| 610 | + "max_chars": _MAX_AUTOMATIC_ARTIFACT_CHARS, | ||
| 611 | + "max_chars_per_window": _MAX_ARTIFACT_WINDOW_CHARS, | ||
| 612 | + }, | ||
| 613 | + "sources": source_records, | ||
| 614 | + }, | ||
| 615 | + ) | ||
| 616 | + | ||
| 617 | + | ||
| 618 | +def _continuous_prefix_end(ranges: Sequence[tuple[int, int]]) -> int: | ||
| 619 | + cursor = 0 | ||
| 620 | + for start, end in sorted(ranges): | ||
| 621 | + if start > cursor: | ||
| 622 | + break | ||
| 623 | + if end > cursor: | ||
| 624 | + cursor = end | ||
| 625 | + return cursor | ||
| 626 | + | ||
| 627 | + | ||
| 628 | +def _unique_automatic_request_id( | ||
| 629 | + parent_request_id: str, | ||
| 630 | + source: str, | ||
| 631 | + start: int, | ||
| 632 | + existing_ids: set[str], | ||
| 633 | +) -> str: | ||
| 634 | + stem = re.sub(r"[^a-zA-Z0-9_.-]+", "_", parent_request_id).strip("_") or "artifact" | ||
| 635 | + digest = hashlib.sha256(source.encode("utf-8")).hexdigest()[:8] | ||
| 636 | + candidate = f"{stem}.auto.{digest}.{start}" | ||
| 637 | + suffix = 2 | ||
| 638 | + while candidate in existing_ids: | ||
| 639 | + candidate = f"{stem}.auto.{digest}.{start}.{suffix}" | ||
| 640 | + suffix += 1 | ||
| 641 | + return candidate | ||
| 642 | + | ||
| 643 | + | ||
| 644 | +def _normalize_request( | ||
| 645 | + request: Mapping[str, Any], | ||
| 646 | + *, | ||
| 647 | + default_hypothesis_id: str, | ||
| 648 | + known_hypothesis_ids: set[str], | ||
| 649 | + index: int, | ||
| 650 | +) -> dict[str, Any] | None: | ||
| 651 | + operation = str(request.get("operation", "") or "").strip().casefold() | ||
| 652 | + if operation not in _ALLOWED_OPERATIONS: | ||
| 653 | + return None | ||
| 654 | + query = str(request.get("query", "") or "").strip() | ||
| 655 | + if operation in {"search_trace", "inspect_artifact", "inspect_evaluation", "search_repository"} and not query: | ||
| 656 | + return None | ||
| 657 | + requested_path = request.get("relative_path") | ||
| 658 | + if operation == "read_artifact_window" and not _safe_relative_path(requested_path): | ||
| 659 | + # Inspection results expose the selected identity as ``source``. Accept | ||
| 660 | + # that exact controller-issued identity when a refinement model feeds it | ||
| 661 | + # back, while retaining the same bounded relative-path validation. | ||
| 662 | + requested_path = request.get("source") | ||
| 663 | + if operation in {"read_artifact_window", "read_repository_file"} and not _safe_relative_path(requested_path): | ||
| 664 | + return None | ||
| 665 | + if operation == "check_relation" and not str(request.get("expression", "") or "").strip(): | ||
| 666 | + return None | ||
| 667 | + if operation == "compare_numeric_change" and not all( | ||
| 668 | + str(request.get(key, "") or "").strip() for key in ("before_expression", "after_expression") | ||
| 669 | + ): | ||
| 670 | + return None | ||
| 671 | + hypothesis_ids = _string_list(request.get("hypothesis_ids")) | ||
| 672 | + if default_hypothesis_id: | ||
| 673 | + hypothesis_ids.append(default_hypothesis_id) | ||
| 674 | + hypothesis_ids = list(dict.fromkeys(item for item in hypothesis_ids if item in known_hypothesis_ids)) | ||
| 675 | + normalized: dict[str, Any] = { | ||
| 676 | + "request_id": str(request.get("request_id", "") or f"q{index}").strip() or f"q{index}", | ||
| 677 | + "hypothesis_ids": hypothesis_ids, | ||
| 678 | + "operation": operation, | ||
| 679 | + "purpose": str(request.get("purpose", "") or "").strip(), | ||
| 680 | + } | ||
| 681 | + if query: | ||
| 682 | + normalized["query"] = query[:1_000] | ||
| 683 | + if operation == "inspect_artifact": | ||
| 684 | + artifact_hint = _safe_relative_path(request.get("relative_path") or request.get("source")) | ||
| 685 | + if artifact_hint: | ||
| 686 | + normalized["relative_path"] = artifact_hint | ||
| 687 | + proof_obligation = str(request.get("proof_obligation", "") or "").strip().casefold() | ||
| 688 | + if proof_obligation in {"existence", "absence", "coverage"}: | ||
| 689 | + normalized["proof_obligation"] = proof_obligation | ||
| 690 | + if operation == "check_relation": | ||
| 691 | + expected = _finite_number(request.get("expected")) | ||
| 692 | + if expected is None: | ||
| 693 | + return None | ||
| 694 | + normalized.update( | ||
| 695 | + { | ||
| 696 | + "expression": str(request.get("expression", "") or "").strip()[:200], | ||
| 697 | + "operator": str(request.get("operator", "approximately_equal") or "approximately_equal") | ||
| 698 | + .strip() | ||
| 699 | + .casefold(), | ||
| 700 | + "expected": expected, | ||
| 701 | + "tolerance": _bounded_tolerance(request.get("tolerance")), | ||
| 702 | + } | ||
| 703 | + ) | ||
| 704 | + if operation == "compare_numeric_change": | ||
| 705 | + expected_delta = _finite_number(request.get("expected_delta")) | ||
| 706 | + if expected_delta is None: | ||
| 707 | + return None | ||
| 708 | + normalized.update( | ||
| 709 | + { | ||
| 710 | + "before_expression": str(request.get("before_expression", "") or "").strip()[:200], | ||
| 711 | + "after_expression": str(request.get("after_expression", "") or "").strip()[:200], | ||
| 712 | + "expected_delta": expected_delta, | ||
| 713 | + "tolerance": _bounded_tolerance(request.get("tolerance")), | ||
| 714 | + } | ||
| 715 | + ) | ||
| 716 | + if operation == "search_trace": | ||
| 717 | + normalized["max_results"] = min( | ||
| 718 | + _MAX_SEARCH_RESULTS, | ||
| 719 | + max(1, _positive_int(request.get("max_results"), default=3)), | ||
| 720 | + ) | ||
| 721 | + if operation == "search_repository": | ||
| 722 | + normalized["max_results"] = min( | ||
| 723 | + _MAX_SEARCH_RESULTS, | ||
| 724 | + max(1, _positive_int(request.get("max_results"), default=3)), | ||
| 725 | + ) | ||
| 726 | + if operation == "read_repository_file": | ||
| 727 | + normalized["relative_path"] = _safe_relative_path(request.get("relative_path")) | ||
| 728 | + if operation == "read_artifact_window": | ||
| 729 | + normalized["relative_path"] = _safe_relative_path(requested_path) | ||
| 730 | + source_char_start = _optional_nonnegative_int(request.get("source_char_start")) or 0 | ||
| 731 | + source_char_end = _optional_nonnegative_int(request.get("source_char_end")) | ||
| 732 | + requested_max_chars = request.get("max_chars") | ||
| 733 | + if requested_max_chars is None and source_char_end is not None and source_char_end > source_char_start: | ||
| 734 | + requested_max_chars = source_char_end - source_char_start | ||
| 735 | + normalized["source_char_start"] = source_char_start | ||
| 736 | + normalized["max_chars"] = min( | ||
| 737 | + _MAX_ARTIFACT_WINDOW_CHARS, | ||
| 738 | + max(1, _positive_int(requested_max_chars, default=_MAX_ARTIFACT_WINDOW_CHARS)), | ||
| 739 | + ) | ||
| 740 | + if operation == "read_event": | ||
| 741 | + message_index = _optional_nonnegative_int(request.get("message_index")) | ||
| 742 | + if message_index is None: | ||
| 743 | + return None | ||
| 744 | + normalized["trace_id"] = str(request.get("trace_id", "") or "").strip() | ||
| 745 | + normalized["message_index"] = message_index | ||
| 746 | + tool_call_index = _optional_nonnegative_int(request.get("tool_call_index")) | ||
| 747 | + if tool_call_index is not None: | ||
| 748 | + normalized["tool_call_index"] = tool_call_index | ||
| 749 | + return normalized | ||
| 750 | + | ||
| 751 | + | ||
| 752 | +def _check_relation(request: Mapping[str, Any]) -> dict[str, Any]: | ||
| 753 | + """Evaluate a small numeric discriminator without executing model code.""" | ||
| 754 | + expression = str(request.get("expression", "") or "").strip() | ||
| 755 | + expected = _finite_number(request.get("expected")) | ||
| 756 | + operator = str(request.get("operator", "approximately_equal") or "approximately_equal").casefold() | ||
| 757 | + tolerance = _bounded_tolerance(request.get("tolerance")) | ||
| 758 | + if expected is None: | ||
| 759 | + return {"availability": "invalid", "reason": "expected_must_be_a_finite_number"} | ||
| 760 | + try: | ||
| 761 | + value = _safe_numeric_expression(expression) | ||
| 762 | + except (SyntaxError, TypeError, ValueError, ZeroDivisionError, OverflowError) as exc: | ||
| 763 | + return { | ||
| 764 | + "availability": "invalid", | ||
| 765 | + "expression": expression, | ||
| 766 | + "reason": f"invalid_numeric_expression:{type(exc).__name__}", | ||
| 767 | + } | ||
| 768 | + | ||
| 769 | + if operator in {"approximately_equal", "equal"}: | ||
| 770 | + holds = math.isclose(value, expected, rel_tol=0.0, abs_tol=tolerance) | ||
| 771 | + elif operator == "not_equal": | ||
| 772 | + holds = not math.isclose(value, expected, rel_tol=0.0, abs_tol=tolerance) | ||
| 773 | + elif operator == "less_than": | ||
| 774 | + holds = value < expected | ||
| 775 | + elif operator == "less_than_or_equal": | ||
| 776 | + holds = value <= expected or math.isclose(value, expected, rel_tol=0.0, abs_tol=tolerance) | ||
| 777 | + elif operator == "greater_than": | ||
| 778 | + holds = value > expected | ||
| 779 | + elif operator == "greater_than_or_equal": | ||
| 780 | + holds = value >= expected or math.isclose(value, expected, rel_tol=0.0, abs_tol=tolerance) | ||
| 781 | + else: | ||
| 782 | + return { | ||
| 783 | + "availability": "invalid", | ||
| 784 | + "expression": expression, | ||
| 785 | + "value": value, | ||
| 786 | + "reason": "unsupported_relation_operator", | ||
| 787 | + } | ||
| 788 | + return { | ||
| 789 | + "availability": "available", | ||
| 790 | + "expression": expression, | ||
| 791 | + "value": value, | ||
| 792 | + "operator": operator, | ||
| 793 | + "expected": expected, | ||
| 794 | + "tolerance": tolerance, | ||
| 795 | + "holds": holds, | ||
| 796 | + } | ||
| 797 | + | ||
| 798 | + | ||
| 799 | +def _compare_numeric_change(request: Mapping[str, Any]) -> dict[str, Any]: | ||
| 800 | + """Compute after-before so formula changes are compared to their real baseline.""" | ||
| 801 | + before_expression = str(request.get("before_expression", "") or "").strip() | ||
| 802 | + after_expression = str(request.get("after_expression", "") or "").strip() | ||
| 803 | + expected_delta = _finite_number(request.get("expected_delta")) | ||
| 804 | + tolerance = _bounded_tolerance(request.get("tolerance")) | ||
| 805 | + if expected_delta is None: | ||
| 806 | + return {"availability": "invalid", "reason": "expected_delta_must_be_a_finite_number"} | ||
| 807 | + try: | ||
| 808 | + before = _safe_numeric_expression(before_expression) | ||
| 809 | + after = _safe_numeric_expression(after_expression) | ||
| 810 | + except (SyntaxError, TypeError, ValueError, ZeroDivisionError, OverflowError) as exc: | ||
| 811 | + return { | ||
| 812 | + "availability": "invalid", | ||
| 813 | + "before_expression": before_expression, | ||
| 814 | + "after_expression": after_expression, | ||
| 815 | + "reason": f"invalid_numeric_expression:{type(exc).__name__}", | ||
| 816 | + } | ||
| 817 | + delta = after - before | ||
| 818 | + return { | ||
| 819 | + "availability": "available", | ||
| 820 | + "before_expression": before_expression, | ||
| 821 | + "after_expression": after_expression, | ||
| 822 | + "before_value": before, | ||
| 823 | + "after_value": after, | ||
| 824 | + "computed_delta": delta, | ||
| 825 | + "expected_delta": expected_delta, | ||
| 826 | + "tolerance": tolerance, | ||
| 827 | + "holds": math.isclose(delta, expected_delta, rel_tol=0.0, abs_tol=tolerance), | ||
| 828 | + } | ||
| 829 | + | ||
| 830 | + | ||
| 831 | +def _safe_numeric_expression(expression: str) -> float: | ||
| 832 | + """Evaluate only finite numeric literals and bounded arithmetic operators.""" | ||
| 833 | + if not expression or len(expression) > 200: | ||
| 834 | + raise ValueError("expression_length") | ||
| 835 | + tree = ast.parse(expression, mode="eval") | ||
| 836 | + if sum(1 for _ in ast.walk(tree)) > 40: | ||
| 837 | + raise ValueError("expression_complexity") | ||
| 838 | + | ||
| 839 | + def _evaluate(node: ast.AST) -> float: | ||
| 840 | + if isinstance(node, ast.Expression): | ||
| 841 | + return _evaluate(node.body) | ||
| 842 | + if isinstance(node, ast.Constant) and isinstance(node.value, (int, float)) and not isinstance(node.value, bool): | ||
| 843 | + value = float(node.value) | ||
| 844 | + elif isinstance(node, ast.UnaryOp) and isinstance(node.op, (ast.UAdd, ast.USub)): | ||
| 845 | + operand = _evaluate(node.operand) | ||
| 846 | + value = operand if isinstance(node.op, ast.UAdd) else -operand | ||
| 847 | + elif isinstance(node, ast.BinOp) and isinstance(node.op, (ast.Add, ast.Sub, ast.Mult, ast.Div)): | ||
| 848 | + left = _evaluate(node.left) | ||
| 849 | + right = _evaluate(node.right) | ||
| 850 | + if isinstance(node.op, ast.Add): | ||
| 851 | + value = left + right | ||
| 852 | + elif isinstance(node.op, ast.Sub): | ||
| 853 | + value = left - right | ||
| 854 | + elif isinstance(node.op, ast.Mult): | ||
| 855 | + value = left * right | ||
| 856 | + else: | ||
| 857 | + value = left / right | ||
| 858 | + else: | ||
| 859 | + raise ValueError("unsupported_expression_node") | ||
| 860 | + if not math.isfinite(value) or abs(value) > 1e100: | ||
| 861 | + raise ValueError("non_finite_or_unbounded_result") | ||
| 862 | + return value | ||
| 863 | + | ||
| 864 | + return _evaluate(tree) | ||
| 865 | + | ||
| 866 | + | ||
| 867 | +def _finite_number(value: Any) -> float | None: | ||
| 868 | + if isinstance(value, bool) or not isinstance(value, (int, float)): | ||
| 869 | + return None | ||
| 870 | + number = float(value) | ||
| 871 | + return number if math.isfinite(number) else None | ||
| 872 | + | ||
| 873 | + | ||
| 874 | +def _bounded_tolerance(value: Any) -> float: | ||
| 875 | + number = _finite_number(value) | ||
| 876 | + if number is None: | ||
| 877 | + return 1e-9 | ||
| 878 | + return min(max(number, 0.0), 1.0) | ||
| 879 | + | ||
| 880 | + | ||
| 881 | +def _trace_events(trace_data: Mapping[str, Any]) -> list[dict[str, Any]]: | ||
| 882 | + events: list[dict[str, Any]] = [] | ||
| 883 | + traces = trace_data.get("traces") | ||
| 884 | + if not isinstance(traces, list): | ||
| 885 | + return events | ||
| 886 | + for trace in traces: | ||
| 887 | + if not isinstance(trace, Mapping): | ||
| 888 | + continue | ||
| 889 | + trace_id = str(trace.get("trace_id", "") or "") | ||
| 890 | + messages = trace.get("messages") | ||
| 891 | + if not isinstance(messages, list): | ||
| 892 | + continue | ||
| 893 | + for sequence, message in enumerate(messages): | ||
| 894 | + if not isinstance(message, Mapping): | ||
| 895 | + continue | ||
| 896 | + raw_index = message.get("message_index", sequence) | ||
| 897 | + message_index = _optional_nonnegative_int(raw_index) | ||
| 898 | + if message_index is None: | ||
| 899 | + message_index = sequence | ||
| 900 | + tool_calls = [dict(call) for call in message.get("tool_calls", []) if isinstance(call, Mapping)] | ||
| 901 | + events.append( | ||
| 902 | + { | ||
| 903 | + "trace_id": trace_id, | ||
| 904 | + "message_index": message_index, | ||
| 905 | + "step_pointer": str(message.get("step_pointer", "") or ""), | ||
| 906 | + "role": str(message.get("role", "") or ""), | ||
| 907 | + "content": str(message.get("content", "") or ""), | ||
| 908 | + "tool_calls": tool_calls, | ||
| 909 | + } | ||
| 910 | + ) | ||
| 911 | + return events | ||
| 912 | + | ||
| 913 | + | ||
| 914 | +def _search_trace(events: list[dict[str, Any]], request: Mapping[str, Any]) -> dict[str, Any]: | ||
| 915 | + query = str(request.get("query", "") or "") | ||
| 916 | + terms = _query_terms(query) | ||
| 917 | + ranked: list[tuple[int, int, dict[str, Any]]] = [] | ||
| 918 | + for sequence, event in enumerate(events): | ||
| 919 | + searchable = json.dumps(event, ensure_ascii=False, separators=(",", ":")).casefold() | ||
| 920 | + matched = [term for term in terms if term in searchable] | ||
| 921 | + if matched: | ||
| 922 | + ranked.append((len(set(matched)), sequence, event)) | ||
| 923 | + limit = min(_MAX_SEARCH_RESULTS, max(1, _positive_int(request.get("max_results"), default=3))) | ||
| 924 | + selected = sorted(ranked, key=lambda item: (-item[0], item[1]))[:limit] | ||
| 925 | + return { | ||
| 926 | + "availability": "available" if selected else "not_found", | ||
| 927 | + "query": query, | ||
| 928 | + "matched_event_count": len(ranked), | ||
| 929 | + "events": [_search_event_view(event, terms) for _, _, event in selected], | ||
| 930 | + } | ||
| 931 | + | ||
| 932 | + | ||
| 933 | +def _read_event(events: list[dict[str, Any]], request: Mapping[str, Any]) -> dict[str, Any]: | ||
| 934 | + trace_id = str(request.get("trace_id", "") or "") | ||
| 935 | + message_index = _optional_nonnegative_int(request.get("message_index")) | ||
| 936 | + matches = [ | ||
| 937 | + event | ||
| 938 | + for event in events | ||
| 939 | + if event["message_index"] == message_index and (not trace_id or event["trace_id"] == trace_id) | ||
| 940 | + ] | ||
| 941 | + if not matches: | ||
| 942 | + return { | ||
| 943 | + "availability": "not_found", | ||
| 944 | + "trace_id": trace_id, | ||
| 945 | + "message_index": message_index, | ||
| 946 | + } | ||
| 947 | + if not trace_id and len(matches) > 1: | ||
| 948 | + return { | ||
| 949 | + "availability": "ambiguous", | ||
| 950 | + "reason": "trace_id_required_for_non_unique_message_index", | ||
| 951 | + "message_index": message_index, | ||
| 952 | + "candidate_trace_ids": sorted({str(item.get("trace_id", "") or "") for item in matches}), | ||
| 953 | + } | ||
| 954 | + event = matches[0] | ||
| 955 | + tool_call_index = _optional_nonnegative_int(request.get("tool_call_index")) | ||
| 956 | + calls = event["tool_calls"] | ||
| 957 | + if tool_call_index is not None: | ||
| 958 | + calls = [calls[tool_call_index]] if tool_call_index < len(calls) else [] | ||
| 959 | + return { | ||
| 960 | + "availability": "available", | ||
| 961 | + "event": { | ||
| 962 | + "trace_id": event["trace_id"], | ||
| 963 | + "message_index": event["message_index"], | ||
| 964 | + "step_pointer": event["step_pointer"], | ||
| 965 | + "role": event["role"], | ||
| 966 | + "content": _bounded_exact_text(event["content"], _MAX_EVENT_CHARS), | ||
| 967 | + "tool_calls": [ | ||
| 968 | + { | ||
| 969 | + "tool_call_index": index, | ||
| 970 | + "name": str(call.get("name", "") or ""), | ||
| 971 | + "input": _bounded_exact_text(str(call.get("input", "") or ""), _MAX_EVENT_CHARS), | ||
| 972 | + "output": _bounded_exact_text(str(call.get("output", "") or ""), _MAX_EVENT_CHARS), | ||
| 973 | + "error": _bounded_exact_text(str(call.get("error", "") or ""), 4_000), | ||
| 974 | + } | ||
| 975 | + for index, call in enumerate(calls) | ||
| 976 | + ], | ||
| 977 | + }, | ||
| 978 | + } | ||
| 979 | + | ||
| 980 | + | ||
| 981 | +def _inspect_artifact( | ||
| 982 | + case: CaseAnalysisInput, | ||
| 983 | + request: Mapping[str, Any], | ||
| 984 | + *, | ||
| 985 | + text_cache: dict[Path, str] | None = None, | ||
| 986 | +) -> dict[str, Any]: | ||
| 987 | + """Search only physically materialized task artifacts. | ||
| 988 | + | ||
| 989 | + Evaluation metadata is deliberately excluded. Previously a criterion or | ||
| 990 | + judge explanation could satisfy an ``inspect_artifact`` request even when | ||
| 991 | + the source document/spreadsheet was absent, which turned a repeated outcome | ||
| 992 | + description into apparent causal evidence. | ||
| 993 | + """ | ||
| 994 | + query = str(request.get("query", "") or "") | ||
| 995 | + artifact_hint = str(request.get("relative_path", "") or "").strip() | ||
| 996 | + purpose = str(request.get("purpose", "") or "").strip() | ||
| 997 | + selection_query = f"{query} {artifact_hint} {purpose}".strip() | ||
| 998 | + terms = _query_terms(query) | ||
| 999 | + identity_terms = _query_terms(f"{artifact_hint} {purpose}".strip()) | ||
| 1000 | + case_dir = _windows_long_path(Path(case.result_path).parent) | ||
| 1001 | + sources: list[tuple[str, str, str]] = [] | ||
| 1002 | + artifacts_dir = case_dir / "artifacts" | ||
| 1003 | + if artifacts_dir.is_dir(): | ||
| 1004 | + aliases = _artifact_path_aliases(case, artifacts_dir) | ||
| 1005 | + files = [ | ||
| 1006 | + path | ||
| 1007 | + for path in sorted(artifacts_dir.rglob("*")) | ||
| 1008 | + if path.is_file() and path.suffix.casefold() in _TEXT_ARTIFACT_SUFFIXES | ||
| 1009 | + ][:_MAX_ARTIFACT_FILES] | ||
| 1010 | + for path in files: | ||
| 1011 | + resolved = path.resolve() | ||
| 1012 | + if not resolved.is_relative_to(case_dir): | ||
| 1013 | + continue | ||
| 1014 | + source = f"artifacts/{path.relative_to(artifacts_dir).as_posix()}" | ||
| 1015 | + sources.append((source, aliases.get(resolved, source), _read_text(path, _MAX_ARTIFACT_FILE_CHARS))) | ||
| 1016 | + structured_files = [ | ||
| 1017 | + path | ||
| 1018 | + for path in sorted(artifacts_dir.rglob("*")) | ||
| 1019 | + if path.is_file() and path.suffix.casefold() in _STRUCTURED_ARTIFACT_SUFFIXES | ||
| 1020 | + ] | ||
| 1021 | + structured_files = _select_structured_artifact_files(structured_files, selection_query, aliases=aliases) | ||
| 1022 | + for path in structured_files: | ||
| 1023 | + resolved = path.resolve() | ||
| 1024 | + if not resolved.is_relative_to(case_dir): | ||
| 1025 | + continue | ||
| 1026 | + if text_cache is not None and resolved in text_cache: | ||
| 1027 | + text = text_cache[resolved] | ||
| 1028 | + else: | ||
| 1029 | + text = _structured_artifact_text(path) | ||
| 1030 | + if text_cache is not None: | ||
| 1031 | + text_cache[resolved] = text | ||
| 1032 | + if text: | ||
| 1033 | + source = f"artifacts/{path.relative_to(artifacts_dir).as_posix()}" | ||
| 1034 | + sources.append((source, aliases.get(resolved, source), text)) | ||
| 1035 | + | ||
| 1036 | + if not sources: | ||
| 1037 | + return { | ||
| 1038 | + "availability": "not_available", | ||
| 1039 | + "reason": "physical_artifact_snapshot_not_available", | ||
| 1040 | + "query": query, | ||
| 1041 | + "matches": [], | ||
| 1042 | + } | ||
| 1043 | + | ||
| 1044 | + if artifact_hint: | ||
| 1045 | + hint_terms = _query_terms(artifact_hint) | ||
| 1046 | + identity_scores = [ | ||
| 1047 | + (_query_match_score(logical_source, hint_terms), logical_source) for _, logical_source, _ in sources | ||
| 1048 | + ] | ||
| 1049 | + best_identity_score = max((score for score, _ in identity_scores), default=0.0) | ||
| 1050 | + if best_identity_score > 0: | ||
| 1051 | + best_logical_sources = { | ||
| 1052 | + logical_source for score, logical_source in identity_scores if score == best_identity_score | ||
| 1053 | + } | ||
| 1054 | + sources = [item for item in sources if item[1] in best_logical_sources] | ||
| 1055 | + | ||
| 1056 | + matches: list[tuple[float, str, str, str]] = [] | ||
| 1057 | + for source, logical_source, text in sources: | ||
| 1058 | + lowered = text.casefold() | ||
| 1059 | + matched = [term for term in terms if term in lowered] | ||
| 1060 | + path_matched = [term for term in terms if term in logical_source.casefold()] | ||
| 1061 | + if matched or path_matched: | ||
| 1062 | + score = ( | ||
| 1063 | + _query_match_score(text, terms) | ||
| 1064 | + + 2.0 * _query_match_score(logical_source, terms) | ||
| 1065 | + + 4.0 * _query_match_score(logical_source, identity_terms) | ||
| 1066 | + ) | ||
| 1067 | + matches.append((score, source, logical_source, text)) | ||
| 1068 | + selected = sorted(matches, key=lambda item: (-item[0], item[1]))[:_MAX_SEARCH_RESULTS] | ||
| 1069 | + return { | ||
| 1070 | + "availability": "available" if selected else "not_found", | ||
| 1071 | + "query": query, | ||
| 1072 | + "matches": [ | ||
| 1073 | + { | ||
| 1074 | + "source": source, | ||
| 1075 | + "logical_source": logical_source, | ||
| 1076 | + "exact_spans": _exact_match_spans(text, terms, max_spans=3) or _leading_text_span(text), | ||
| 1077 | + } | ||
| 1078 | + for _, source, logical_source, text in selected | ||
| 1079 | + ], | ||
| 1080 | + } | ||
| 1081 | + | ||
| 1082 | + | ||
| 1083 | +def _artifact_path_aliases(case: CaseAnalysisInput, artifacts_dir: Path) -> dict[Path, str]: | ||
| 1084 | + """Map bounded snapshot paths back to their original logical names.""" | ||
| 1085 | + snapshot = case.evaluation_metadata.get("analysis_artifact_snapshot", {}) | ||
| 1086 | + rows = snapshot.get("files", []) if isinstance(snapshot, Mapping) else [] | ||
| 1087 | + aliases: dict[Path, str] = {} | ||
| 1088 | + for row in rows if isinstance(rows, list) else []: | ||
| 1089 | + if not isinstance(row, Mapping): | ||
| 1090 | + continue | ||
| 1091 | + stored = _safe_relative_path(row.get("path")) | ||
| 1092 | + logical = str(row.get("source_path", "") or "").strip() | ||
| 1093 | + if not stored or not logical: | ||
| 1094 | + continue | ||
| 1095 | + for candidate in (artifacts_dir / stored, artifacts_dir / "workspace" / stored): | ||
| 1096 | + resolved = candidate.resolve() | ||
| 1097 | + if resolved.is_relative_to(artifacts_dir) and resolved.is_file(): | ||
| 1098 | + aliases[resolved] = logical | ||
| 1099 | + break | ||
| 1100 | + return aliases | ||
| 1101 | + | ||
| 1102 | + | ||
| 1103 | +def _select_structured_artifact_files( | ||
| 1104 | + files: Sequence[Path], | ||
| 1105 | + query: str, | ||
| 1106 | + *, | ||
| 1107 | + aliases: Mapping[Path, str] | None = None, | ||
| 1108 | +) -> list[Path]: | ||
| 1109 | + """Bound structured parsing and prefer file types named by the request.""" | ||
| 1110 | + lowered_query = query.casefold() | ||
| 1111 | + hinted_suffixes = { | ||
| 1112 | + suffix | ||
| 1113 | + for suffix, hints in _STRUCTURED_QUERY_SUFFIX_HINTS.items() | ||
| 1114 | + if any(hint in lowered_query for hint in hints) | ||
| 1115 | + } | ||
| 1116 | + candidates = [path for path in files if not hinted_suffixes or path.suffix.casefold() in hinted_suffixes] | ||
| 1117 | + if len(candidates) <= _MAX_STRUCTURED_ARTIFACTS_PER_REQUEST: | ||
| 1118 | + return candidates | ||
| 1119 | + | ||
| 1120 | + query_terms = _query_terms(query) | ||
| 1121 | + ranked = sorted( | ||
| 1122 | + candidates, | ||
| 1123 | + key=lambda path: ( | ||
| 1124 | + -_query_match_score( | ||
| 1125 | + f"{path.as_posix()} {(aliases or {}).get(path.resolve(), '')}", | ||
| 1126 | + query_terms, | ||
| 1127 | + ), | ||
| 1128 | + path.stat().st_size, | ||
| 1129 | + path.as_posix(), | ||
| 1130 | + ), | ||
| 1131 | + ) | ||
| 1132 | + return ranked[:_MAX_STRUCTURED_ARTIFACTS_PER_REQUEST] | ||
| 1133 | + | ||
| 1134 | + | ||
| 1135 | +def _read_artifact_window( | ||
| 1136 | + case: CaseAnalysisInput, | ||
| 1137 | + request: Mapping[str, Any], | ||
| 1138 | + *, | ||
| 1139 | + text_cache: dict[Path, str] | None = None, | ||
| 1140 | +) -> dict[str, Any]: | ||
| 1141 | + """Read one exact, bounded window from a previously named task artifact.""" | ||
| 1142 | + relative_path = _safe_relative_path(request.get("relative_path")) | ||
| 1143 | + if relative_path.startswith("artifacts/"): | ||
| 1144 | + relative_path = relative_path.removeprefix("artifacts/") | ||
| 1145 | + if not relative_path: | ||
| 1146 | + return {"availability": "invalid", "reason": "invalid_relative_path"} | ||
| 1147 | + case_dir = _windows_long_path(Path(case.result_path).parent) | ||
| 1148 | + artifacts_dir = (case_dir / "artifacts").resolve() | ||
| 1149 | + aliases = _artifact_path_aliases(case, artifacts_dir) | ||
| 1150 | + path = (artifacts_dir / relative_path).resolve() | ||
| 1151 | + logical_source = aliases.get(path) | ||
| 1152 | + if not path.is_relative_to(artifacts_dir) or not path.is_file(): | ||
| 1153 | + workspace_path = (artifacts_dir / "workspace" / relative_path).resolve() | ||
| 1154 | + if workspace_path.is_relative_to(artifacts_dir) and workspace_path.is_file(): | ||
| 1155 | + path = workspace_path | ||
| 1156 | + logical_source = aliases.get(path) | ||
| 1157 | + else: | ||
| 1158 | + requested = relative_path.removeprefix("workspace/") | ||
| 1159 | + logical_matches = [ | ||
| 1160 | + (stored, logical) | ||
| 1161 | + for stored, logical in aliases.items() | ||
| 1162 | + if logical.replace("\\", "/").removeprefix("workspace/") == requested | ||
| 1163 | + ] | ||
| 1164 | + if not logical_matches: | ||
| 1165 | + logical_matches = _unambiguous_logical_artifact_matches(requested, aliases) | ||
| 1166 | + if len(logical_matches) != 1: | ||
| 1167 | + return {"availability": "not_found", "relative_path": relative_path} | ||
| 1168 | + path, logical_source = logical_matches[0] | ||
| 1169 | + if not path.is_relative_to(artifacts_dir) or not path.is_file(): | ||
| 1170 | + return {"availability": "not_found", "relative_path": relative_path} | ||
| 1171 | + suffix = path.suffix.casefold() | ||
| 1172 | + if suffix in _STRUCTURED_ARTIFACT_SUFFIXES: | ||
| 1173 | + if text_cache is not None and path in text_cache: | ||
| 1174 | + content = text_cache[path] | ||
| 1175 | + else: | ||
| 1176 | + content = _structured_artifact_text(path) | ||
| 1177 | + if text_cache is not None: | ||
| 1178 | + text_cache[path] = content | ||
| 1179 | + elif suffix in _TEXT_ARTIFACT_SUFFIXES: | ||
| 1180 | + content = _read_text(path, _MAX_ARTIFACT_FILE_CHARS) | ||
| 1181 | + else: | ||
| 1182 | + return {"availability": "unsupported", "relative_path": relative_path} | ||
| 1183 | + if not content: | ||
| 1184 | + return {"availability": "not_available", "relative_path": relative_path} | ||
| 1185 | + start = min( | ||
| 1186 | + len(content), | ||
| 1187 | + _optional_nonnegative_int(request.get("source_char_start")) or 0, | ||
| 1188 | + ) | ||
| 1189 | + max_chars = min( | ||
| 1190 | + _MAX_ARTIFACT_WINDOW_CHARS, | ||
| 1191 | + max(1, _positive_int(request.get("max_chars"), default=_MAX_ARTIFACT_WINDOW_CHARS)), | ||
| 1192 | + ) | ||
| 1193 | + end = min(len(content), start + max_chars) | ||
| 1194 | + physical_relative_path = path.relative_to(artifacts_dir).as_posix() | ||
| 1195 | + return { | ||
| 1196 | + "availability": "available", | ||
| 1197 | + "source": f"artifacts/{physical_relative_path}", | ||
| 1198 | + "logical_source": logical_source or aliases.get(path, f"artifacts/{physical_relative_path}"), | ||
| 1199 | + "source_char_start": start, | ||
| 1200 | + "source_char_end": end, | ||
| 1201 | + "source_char_count": len(content), | ||
| 1202 | + "text": content[start:end], | ||
| 1203 | + "window_complete": start == 0 and end == len(content), | ||
| 1204 | + "next_source_char_start": end if end < len(content) else None, | ||
| 1205 | + "omission_origin": "controller_read_window" if start or end < len(content) else "none", | ||
| 1206 | + } | ||
| 1207 | + | ||
| 1208 | + | ||
| 1209 | +def _unambiguous_logical_artifact_matches( | ||
| 1210 | + requested: str, | ||
| 1211 | + aliases: Mapping[Path, str], | ||
| 1212 | +) -> list[tuple[Path, str]]: | ||
| 1213 | + """Recover a named artifact when presentation-only path encoding changed.""" | ||
| 1214 | + requested_terms = {term for term in _query_terms(Path(requested).name) if len(term) >= 3} | ||
| 1215 | + if len(requested_terms) < 2: | ||
| 1216 | + return [] | ||
| 1217 | + ranked: list[tuple[int, Path, str]] = [] | ||
| 1218 | + for stored, logical in aliases.items(): | ||
| 1219 | + logical_terms = {term for term in _query_terms(Path(logical).name) if len(term) >= 3} | ||
| 1220 | + overlap = len(requested_terms & logical_terms) | ||
| 1221 | + if overlap: | ||
| 1222 | + ranked.append((overlap, stored, logical)) | ||
| 1223 | + if not ranked: | ||
| 1224 | + return [] | ||
| 1225 | + best = max(score for score, _, _ in ranked) | ||
| 1226 | + minimum = max(2, (len(requested_terms) + 1) // 2) | ||
| 1227 | + winners = [(stored, logical) for score, stored, logical in ranked if score == best and score >= minimum] | ||
| 1228 | + return winners if len(winners) == 1 else [] | ||
| 1229 | + | ||
| 1230 | + | ||
| 1231 | +def _inspect_evaluation(case: CaseAnalysisInput, request: Mapping[str, Any]) -> dict[str, Any]: | ||
| 1232 | + """Search evaluator-owned result metadata without calling it an artifact.""" | ||
| 1233 | + query = str(request.get("query", "") or "") | ||
| 1234 | + terms = _query_terms(query) | ||
| 1235 | + sources = [ | ||
| 1236 | + ("case.evaluation_metadata", json.dumps(case.evaluation_metadata, ensure_ascii=False, indent=2)), | ||
| 1237 | + ("case.result", _read_text(Path(case.result_path), _MAX_ARTIFACT_FILE_CHARS)), | ||
| 1238 | + ] | ||
| 1239 | + matches: list[tuple[int, str, str]] = [] | ||
| 1240 | + for source, content in sources: | ||
| 1241 | + lowered = content.casefold() | ||
| 1242 | + matched = [term for term in terms if term in lowered] | ||
| 1243 | + if matched: | ||
| 1244 | + matches.append((len(set(matched)), source, content)) | ||
| 1245 | + selected = sorted(matches, key=lambda item: (-item[0], item[1]))[:_MAX_SEARCH_RESULTS] | ||
| 1246 | + return { | ||
| 1247 | + "availability": "available" if selected else "not_found", | ||
| 1248 | + "evidence_class": "evaluation_metadata", | ||
| 1249 | + "query": query, | ||
| 1250 | + "matches": [ | ||
| 1251 | + { | ||
| 1252 | + "source": source, | ||
| 1253 | + "exact_spans": _exact_match_spans(content, terms, max_spans=3), | ||
| 1254 | + } | ||
| 1255 | + for _, source, content in selected | ||
| 1256 | + ], | ||
| 1257 | + } | ||
| 1258 | + | ||
| 1259 | + | ||
| 1260 | +def _repository_dir(evidence_root: str | Path | None) -> Path | None: | ||
| 1261 | + if evidence_root is None: | ||
| 1262 | + return None | ||
| 1263 | + root = Path(evidence_root).expanduser().resolve() | ||
| 1264 | + repository = (root / "repository").resolve() | ||
| 1265 | + return repository if repository.is_dir() and repository.is_relative_to(root) else None | ||
| 1266 | + | ||
| 1267 | + | ||
| 1268 | +def _search_repository(repository_dir: Path | None, request: Mapping[str, Any]) -> dict[str, Any]: | ||
| 1269 | + if repository_dir is None: | ||
| 1270 | + return {"availability": "not_available", "reason": "repository_snapshot_not_available"} | ||
| 1271 | + query = str(request.get("query", "") or "") | ||
| 1272 | + terms = _query_terms(query) | ||
| 1273 | + ranked: list[tuple[int, str, str]] = [] | ||
| 1274 | + for path in _repository_text_files(repository_dir): | ||
| 1275 | + text = _read_text(path, _MAX_ARTIFACT_FILE_CHARS) | ||
| 1276 | + lowered = text.casefold() | ||
| 1277 | + matched = [term for term in terms if term in lowered] | ||
| 1278 | + if matched: | ||
| 1279 | + relative = path.relative_to(repository_dir).as_posix() | ||
| 1280 | + ranked.append((len(set(matched)), relative, text)) | ||
| 1281 | + limit = min(_MAX_SEARCH_RESULTS, max(1, _positive_int(request.get("max_results"), default=3))) | ||
| 1282 | + selected = sorted(ranked, key=lambda item: (-item[0], item[1]))[:limit] | ||
| 1283 | + return { | ||
| 1284 | + "availability": "available" if selected else "not_found", | ||
| 1285 | + "query": query, | ||
| 1286 | + "matched_file_count": len(ranked), | ||
| 1287 | + "files": [ | ||
| 1288 | + {"relative_path": relative, "exact_spans": _exact_match_spans(text, terms, max_spans=3)} | ||
| 1289 | + for _, relative, text in selected | ||
| 1290 | + ], | ||
| 1291 | + } | ||
| 1292 | + | ||
| 1293 | + | ||
| 1294 | +def _read_repository_file( | ||
| 1295 | + repository_dir: Path | None, | ||
| 1296 | + request: Mapping[str, Any], | ||
| 1297 | + *, | ||
| 1298 | + discovered_paths: set[str], | ||
| 1299 | +) -> dict[str, Any]: | ||
| 1300 | + if repository_dir is None: | ||
| 1301 | + return {"availability": "not_available", "reason": "repository_snapshot_not_available"} | ||
| 1302 | + relative_path = _safe_relative_path(request.get("relative_path")) | ||
| 1303 | + if not relative_path: | ||
| 1304 | + return {"availability": "invalid", "reason": "invalid_relative_path"} | ||
| 1305 | + if relative_path not in discovered_paths: | ||
| 1306 | + return { | ||
| 1307 | + "availability": "invalid", | ||
| 1308 | + "reason": "relative_path_not_returned_by_prior_search_repository", | ||
| 1309 | + "relative_path": relative_path, | ||
| 1310 | + } | ||
| 1311 | + path = (repository_dir / relative_path).resolve() | ||
| 1312 | + if not path.is_relative_to(repository_dir) or not path.is_file(): | ||
| 1313 | + return {"availability": "not_found", "relative_path": relative_path} | ||
| 1314 | + if path.suffix.casefold() not in _TEXT_ARTIFACT_SUFFIXES | { | ||
| 1315 | + ".c", | ||
| 1316 | + ".cc", | ||
| 1317 | + ".cpp", | ||
| 1318 | + ".go", | ||
| 1319 | + ".h", | ||
| 1320 | + ".hpp", | ||
| 1321 | + ".java", | ||
| 1322 | + ".js", | ||
| 1323 | + ".jsx", | ||
| 1324 | + ".py", | ||
| 1325 | + ".rs", | ||
| 1326 | + ".sh", | ||
| 1327 | + ".ts", | ||
| 1328 | + ".tsx", | ||
| 1329 | + }: | ||
| 1330 | + return {"availability": "unsupported", "relative_path": relative_path} | ||
| 1331 | + return { | ||
| 1332 | + "availability": "available", | ||
| 1333 | + "relative_path": relative_path, | ||
| 1334 | + "content": _bounded_exact_text(_read_text(path, _MAX_ARTIFACT_FILE_CHARS), _MAX_EVENT_CHARS), | ||
| 1335 | + } | ||
| 1336 | + | ||
| 1337 | + | ||
| 1338 | +def _repository_text_files(repository_dir: Path) -> list[Path]: | ||
| 1339 | + suffixes = _TEXT_ARTIFACT_SUFFIXES | { | ||
| 1340 | + ".c", | ||
| 1341 | + ".cc", | ||
| 1342 | + ".cpp", | ||
| 1343 | + ".go", | ||
| 1344 | + ".h", | ||
| 1345 | + ".hpp", | ||
| 1346 | + ".java", | ||
| 1347 | + ".js", | ||
| 1348 | + ".jsx", | ||
| 1349 | + ".py", | ||
| 1350 | + ".rs", | ||
| 1351 | + ".sh", | ||
| 1352 | + ".ts", | ||
| 1353 | + ".tsx", | ||
| 1354 | + } | ||
| 1355 | + return [ | ||
| 1356 | + path | ||
| 1357 | + for path in sorted(repository_dir.rglob("*")) | ||
| 1358 | + if path.is_file() and path.suffix.casefold() in suffixes and path.resolve().is_relative_to(repository_dir) | ||
| 1359 | + ][:_MAX_REPOSITORY_FILES] | ||
| 1360 | + | ||
| 1361 | + | ||
| 1362 | +def _safe_relative_path(value: Any) -> str: | ||
| 1363 | + text = str(value or "").strip().replace("\\", "/") | ||
| 1364 | + if not text or text.startswith("/") or re.match(r"^[A-Za-z]:", text): | ||
| 1365 | + return "" | ||
| 1366 | + parts = [part for part in text.split("/") if part not in {"", "."}] | ||
| 1367 | + if not parts or any(part == ".." for part in parts): | ||
| 1368 | + return "" | ||
| 1369 | + return "/".join(parts) | ||
| 1370 | + | ||
| 1371 | + | ||
| 1372 | +def _normalized_semantics(value: str) -> str: | ||
| 1373 | + return " ".join(re.findall(r"[a-z0-9_]+|[\u4e00-\u9fff]", value.casefold())) | ||
| 1374 | + | ||
| 1375 | + | ||
| 1376 | +def _structured_artifact_text(path: Path) -> str: | ||
| 1377 | + try: | ||
| 1378 | + suffix = path.suffix.casefold() | ||
| 1379 | + if suffix == ".xlsx": | ||
| 1380 | + return _xlsx_text(path) | ||
| 1381 | + if suffix == ".docx": | ||
| 1382 | + return _docx_text(path) | ||
| 1383 | + if suffix == ".pdf": | ||
| 1384 | + return _pdf_text(path) | ||
| 1385 | + if suffix == ".pptx": | ||
| 1386 | + return _pptx_text(path) | ||
| 1387 | + except Exception: # malformed optional artifacts are unavailable evidence | ||
| 1388 | + return "" | ||
| 1389 | + return "" | ||
| 1390 | + | ||
| 1391 | + | ||
| 1392 | +def _xlsx_text(path: Path) -> str: | ||
| 1393 | + from openpyxl import load_workbook | ||
| 1394 | + | ||
| 1395 | + workbook = load_workbook(path, read_only=True, data_only=False) | ||
| 1396 | + lines: list[str] = [] | ||
| 1397 | + total_chars = 0 | ||
| 1398 | + cell_count = 0 | ||
| 1399 | + try: | ||
| 1400 | + for worksheet in workbook.worksheets: | ||
| 1401 | + lines.append(f"[sheet:{worksheet.title}]") | ||
| 1402 | + for row in worksheet.iter_rows( | ||
| 1403 | + max_row=min(worksheet.max_row, 2_000), | ||
| 1404 | + max_col=min(worksheet.max_column, 200), | ||
| 1405 | + ): | ||
| 1406 | + for cell in row: | ||
| 1407 | + if cell.value is not None: | ||
| 1408 | + line = f"{cell.coordinate}={cell.value}" | ||
| 1409 | + lines.append(line) | ||
| 1410 | + total_chars += len(line) | ||
| 1411 | + cell_count += 1 | ||
| 1412 | + if total_chars >= _MAX_ARTIFACT_FILE_CHARS or cell_count >= _MAX_STRUCTURED_CELLS: | ||
| 1413 | + return "\n".join(lines) | ||
| 1414 | + finally: | ||
| 1415 | + workbook.close() | ||
| 1416 | + return "\n".join(lines) | ||
| 1417 | + | ||
| 1418 | + | ||
| 1419 | +def _docx_text(path: Path) -> str: | ||
| 1420 | + from docx import Document | ||
| 1421 | + | ||
| 1422 | + document = Document(path) | ||
| 1423 | + lines: list[str] = [] | ||
| 1424 | + total_chars = 0 | ||
| 1425 | + for paragraph in document.paragraphs: | ||
| 1426 | + if not paragraph.text: | ||
| 1427 | + continue | ||
| 1428 | + lines.append(paragraph.text) | ||
| 1429 | + total_chars += len(paragraph.text) | ||
| 1430 | + if total_chars >= _MAX_ARTIFACT_FILE_CHARS: | ||
| 1431 | + return "\n".join(lines)[:_MAX_ARTIFACT_FILE_CHARS] | ||
| 1432 | + for table_index, table in enumerate(document.tables, start=1): | ||
| 1433 | + for row_index, row in enumerate(table.rows, start=1): | ||
| 1434 | + values = [cell.text for cell in row.cells] | ||
| 1435 | + line = f"[table:{table_index}:row:{row_index}] " + " | ".join(values) | ||
| 1436 | + lines.append(line) | ||
| 1437 | + total_chars += len(line) | ||
| 1438 | + if total_chars >= _MAX_ARTIFACT_FILE_CHARS: | ||
| 1439 | + return "\n".join(lines)[:_MAX_ARTIFACT_FILE_CHARS] | ||
| 1440 | + return "\n".join(lines)[:_MAX_ARTIFACT_FILE_CHARS] | ||
| 1441 | + | ||
| 1442 | + | ||
| 1443 | +def _pdf_text(path: Path) -> str: | ||
| 1444 | + import pdfplumber | ||
| 1445 | + | ||
| 1446 | + lines: list[str] = [] | ||
| 1447 | + with pdfplumber.open(path) as document: | ||
| 1448 | + for page_index, page in enumerate(document.pages[:_MAX_STRUCTURED_PAGES], start=1): | ||
| 1449 | + lines.append(f"[page:{page_index}]") | ||
| 1450 | + lines.append(page.extract_text() or "") | ||
| 1451 | + if sum(len(item) for item in lines) >= _MAX_ARTIFACT_FILE_CHARS: | ||
| 1452 | + break | ||
| 1453 | + return "\n".join(lines)[:_MAX_ARTIFACT_FILE_CHARS] | ||
| 1454 | + | ||
| 1455 | + | ||
| 1456 | +def _pptx_text(path: Path) -> str: | ||
| 1457 | + lines: list[str] = [] | ||
| 1458 | + with zipfile.ZipFile(path) as archive: | ||
| 1459 | + slide_names = sorted(name for name in archive.namelist() if re.fullmatch(r"ppt/slides/slide\d+\.xml", name))[ | ||
| 1460 | + :_MAX_STRUCTURED_PAGES | ||
| 1461 | + ] | ||
| 1462 | + for slide_name in slide_names: | ||
| 1463 | + root = ElementTree.fromstring(archive.read(slide_name)) | ||
| 1464 | + texts = [element.text or "" for element in root.iter() if element.tag.endswith("}t")] | ||
| 1465 | + lines.append(f"[{slide_name}] " + " ".join(texts)) | ||
| 1466 | + return "\n".join(lines)[:_MAX_ARTIFACT_FILE_CHARS] | ||
| 1467 | + | ||
| 1468 | + | ||
| 1469 | +def _compare_runs( | ||
| 1470 | + prior_candidate_feedback: Mapping[str, Any] | None, | ||
| 1471 | + request: Mapping[str, Any], | ||
| 1472 | +) -> dict[str, Any]: | ||
| 1473 | + if not prior_candidate_feedback: | ||
| 1474 | + return {"availability": "not_available", "reason": "no_paired_candidate_feedback"} | ||
| 1475 | + rendered = json.dumps(prior_candidate_feedback, ensure_ascii=False, indent=2) | ||
| 1476 | + query = str(request.get("query", "") or "") | ||
| 1477 | + terms = _query_terms(query) if query else [] | ||
| 1478 | + return { | ||
| 1479 | + "availability": "available", | ||
| 1480 | + "query": query, | ||
| 1481 | + "paired_feedback": ( | ||
| 1482 | + {"exact_spans": _exact_match_spans(rendered, terms, max_spans=5)} | ||
| 1483 | + if terms | ||
| 1484 | + else _bounded_exact_text(rendered, _MAX_EVENT_CHARS) | ||
| 1485 | + ), | ||
| 1486 | + } | ||
| 1487 | + | ||
| 1488 | + | ||
| 1489 | +def _search_event_view(event: Mapping[str, Any], terms: Sequence[str]) -> dict[str, Any]: | ||
| 1490 | + return { | ||
| 1491 | + "trace_id": event.get("trace_id"), | ||
| 1492 | + "message_index": event.get("message_index"), | ||
| 1493 | + "step_pointer": event.get("step_pointer"), | ||
| 1494 | + "role": event.get("role"), | ||
| 1495 | + "content_spans": _exact_match_spans(str(event.get("content", "") or ""), terms, max_spans=2), | ||
| 1496 | + "tool_calls": [ | ||
| 1497 | + { | ||
| 1498 | + "tool_call_index": index, | ||
| 1499 | + "name": str(call.get("name", "") or ""), | ||
| 1500 | + "input_spans": _exact_match_spans(str(call.get("input", "") or ""), terms, max_spans=2), | ||
| 1501 | + "output_spans": _exact_match_spans(str(call.get("output", "") or ""), terms, max_spans=3), | ||
| 1502 | + "error_spans": _exact_match_spans(str(call.get("error", "") or ""), terms, max_spans=2), | ||
| 1503 | + } | ||
| 1504 | + for index, call in enumerate(event.get("tool_calls", [])) | ||
| 1505 | + ], | ||
| 1506 | + } | ||
| 1507 | + | ||
| 1508 | + | ||
| 1509 | +def _exact_match_spans( | ||
| 1510 | + text: str, | ||
| 1511 | + terms: Sequence[str], | ||
| 1512 | + *, | ||
| 1513 | + max_spans: int, | ||
| 1514 | + span_chars: int = 2_000, | ||
| 1515 | +) -> list[dict[str, Any]]: | ||
| 1516 | + if not text or not terms: | ||
| 1517 | + return [] | ||
| 1518 | + lowered = text.casefold() | ||
| 1519 | + candidates: list[tuple[float, int, int, list[str]]] = [] | ||
| 1520 | + for term in terms: | ||
| 1521 | + start_at = 0 | ||
| 1522 | + for _ in range(16): | ||
| 1523 | + position = lowered.find(term, start_at) | ||
| 1524 | + if position < 0: | ||
| 1525 | + break | ||
| 1526 | + start = max(0, position - span_chars // 3) | ||
| 1527 | + end = min(len(text), start + span_chars) | ||
| 1528 | + start = max(0, end - span_chars) | ||
| 1529 | + window = lowered[start:end] | ||
| 1530 | + matched_terms = [candidate for candidate in terms if candidate in window] | ||
| 1531 | + score = sum(_term_weight(candidate) for candidate in set(matched_terms)) | ||
| 1532 | + score += min(1.0, len(matched_terms) / 10) | ||
| 1533 | + candidates.append((score, start, end, matched_terms)) | ||
| 1534 | + start_at = position + max(1, len(term)) | ||
| 1535 | + spans: list[dict[str, Any]] = [] | ||
| 1536 | + occupied: list[tuple[int, int]] = [] | ||
| 1537 | + for _, start, end, matched_terms in sorted(candidates, key=lambda item: (-item[0], item[1])): | ||
| 1538 | + if any(start < old_end and end > old_start for old_start, old_end in occupied): | ||
| 1539 | + continue | ||
| 1540 | + occupied.append((start, end)) | ||
| 1541 | + spans.append( | ||
| 1542 | + { | ||
| 1543 | + "source_char_start": start, | ||
| 1544 | + "source_char_end": end, | ||
| 1545 | + "source_char_count": len(text), | ||
| 1546 | + "matched_term": matched_terms[0], | ||
| 1547 | + "matched_terms": list(dict.fromkeys(matched_terms)), | ||
| 1548 | + "text": text[start:end], | ||
| 1549 | + "window_complete": start == 0 and end == len(text), | ||
| 1550 | + "omission_origin": "controller_search_window" if start or end < len(text) else "none", | ||
| 1551 | + } | ||
| 1552 | + ) | ||
| 1553 | + if len(spans) >= max_spans: | ||
| 1554 | + break | ||
| 1555 | + return spans | ||
| 1556 | + | ||
| 1557 | + | ||
| 1558 | +def _leading_text_span(text: str, *, span_chars: int = 2_000) -> list[dict[str, Any]]: | ||
| 1559 | + if not text: | ||
| 1560 | + return [] | ||
| 1561 | + end = min(len(text), span_chars) | ||
| 1562 | + return [ | ||
| 1563 | + { | ||
| 1564 | + "source_char_start": 0, | ||
| 1565 | + "source_char_end": end, | ||
| 1566 | + "source_char_count": len(text), | ||
| 1567 | + "matched_term": "logical_source", | ||
| 1568 | + "matched_terms": ["logical_source"], | ||
| 1569 | + "text": text[:end], | ||
| 1570 | + "window_complete": end == len(text), | ||
| 1571 | + "omission_origin": "controller_search_window" if end < len(text) else "none", | ||
| 1572 | + } | ||
| 1573 | + ] | ||
| 1574 | + | ||
| 1575 | + | ||
| 1576 | +def _query_match_score(text: str, terms: Sequence[str]) -> float: | ||
| 1577 | + lowered = text.casefold() | ||
| 1578 | + return sum(_term_weight(term) for term in set(terms) if term in lowered) | ||
| 1579 | + | ||
| 1580 | + | ||
| 1581 | +def _term_weight(term: str) -> float: | ||
| 1582 | + weight = 1.0 + min(len(term), 24) / 24 | ||
| 1583 | + if any(character.isdigit() for character in term): | ||
| 1584 | + weight += 1.5 | ||
| 1585 | + if any(character in term for character in "_./:-"): | ||
| 1586 | + weight += 0.5 | ||
| 1587 | + return weight | ||
| 1588 | + | ||
| 1589 | + | ||
| 1590 | +def _bounded_exact_text(text: str, limit: int) -> dict[str, Any]: | ||
| 1591 | + if len(text) <= limit: | ||
| 1592 | + return { | ||
| 1593 | + "text": text, | ||
| 1594 | + "source_char_count": len(text), | ||
| 1595 | + "complete": True, | ||
| 1596 | + "omission_origin": "none", | ||
| 1597 | + } | ||
| 1598 | + return { | ||
| 1599 | + "text": text[:limit], | ||
| 1600 | + "source_char_count": len(text), | ||
| 1601 | + "complete": False, | ||
| 1602 | + "omission_origin": "controller_bound", | ||
| 1603 | + "omitted_source_chars": len(text) - limit, | ||
| 1604 | + } | ||
| 1605 | + | ||
| 1606 | + | ||
| 1607 | +def _query_terms(query: str) -> list[str]: | ||
| 1608 | + terms: list[str] = [] | ||
| 1609 | + seen: set[str] = set() | ||
| 1610 | + for match in _TERM_PATTERN.findall(query): | ||
| 1611 | + normalized = match.casefold() | ||
| 1612 | + if normalized in _STOPWORDS or normalized in seen: | ||
| 1613 | + continue | ||
| 1614 | + seen.add(normalized) | ||
| 1615 | + terms.append(normalized) | ||
| 1616 | + if len(terms) >= 24: | ||
| 1617 | + break | ||
| 1618 | + return terms | ||
| 1619 | + | ||
| 1620 | + | ||
| 1621 | +def _read_json(path: Path) -> dict[str, Any]: | ||
| 1622 | + try: | ||
| 1623 | + value = json.loads(path.read_text(encoding="utf-8")) | ||
| 1624 | + except (OSError, json.JSONDecodeError): | ||
| 1625 | + return {} | ||
| 1626 | + return value if isinstance(value, dict) else {} | ||
| 1627 | + | ||
| 1628 | + | ||
| 1629 | +def _read_text(path: Path, limit: int) -> str: | ||
| 1630 | + try: | ||
| 1631 | + with path.open("r", encoding="utf-8", errors="replace") as stream: | ||
| 1632 | + return stream.read(limit) | ||
| 1633 | + except OSError: | ||
| 1634 | + return "" | ||
| 1635 | + | ||
| 1636 | + | ||
| 1637 | +def _windows_long_path(path: Path) -> Path: | ||
| 1638 | + """Return an extended Windows path for deep evaluation artifact trees.""" | ||
| 1639 | + resolved = str(path.resolve(strict=False)) | ||
| 1640 | + if os.name != "nt" or resolved.startswith("\\\\?\\"): | ||
| 1641 | + return Path(resolved) | ||
| 1642 | + if resolved.startswith("\\\\"): | ||
| 1643 | + return Path("".join(("\\\\?\\UNC\\", resolved.lstrip("\\")))) | ||
| 1644 | + return Path("".join(("\\\\?\\", resolved))) | ||
| 1645 | + | ||
| 1646 | + | ||
| 1647 | +def _string_list(value: Any) -> list[str]: | ||
| 1648 | + if not isinstance(value, (list, tuple, set)): | ||
| 1649 | + return [] | ||
| 1650 | + return [str(item).strip() for item in value if str(item).strip()] | ||
| 1651 | + | ||
| 1652 | + | ||
| 1653 | +def _positive_int(value: Any, *, default: int) -> int: | ||
| 1654 | + try: | ||
| 1655 | + parsed = int(value) | ||
| 1656 | + except (TypeError, ValueError): | ||
| 1657 | + return default | ||
| 1658 | + return parsed if parsed > 0 else default | ||
| 1659 | + | ||
| 1660 | + | ||
| 1661 | +def _optional_nonnegative_int(value: Any) -> int | None: | ||
| 1662 | + try: | ||
| 1663 | + parsed = int(value) | ||
| 1664 | + except (TypeError, ValueError): | ||
| 1665 | + return None | ||
| 1666 | + return parsed if parsed >= 0 else None | ||
| 1667 | + | ||
| 1668 | + | ||
| 1669 | +__all__ = [ | ||
| 1670 | + "execute_causal_investigation", | ||
| 1671 | + "normalize_causal_investigation", | ||
| 1672 | +] | ||