Merged
test(evals): replace simple retrieval cases with real scenarios #63
汪超Add on Sep 24
test(evals): replace simple retrieval cases with real scenarios #63
Merged
Total: 11 files changed+1489-443
| @@ -22,7 +22,8 @@ python3 .agents/skills/knowledge-query/scripts/knowledge_index.py \ | |||
| 22 | --knowledge-root . build | 22 | --knowledge-root . build |
| 23 | python3 .agents/skills/knowledge-query/scripts/knowledge_index.py \ | 23 | python3 .agents/skills/knowledge-query/scripts/knowledge_index.py \ |
| 24 | --knowledge-root . verify --level full | 24 | --knowledge-root . verify --level full |
| 25 | -python3 evals/run_retrieval.py | 25 | +# Gate recall/ranking by aggregate targets; query and filter contract errors still fail. |
| 26 | +python3 evals/run_retrieval.py --quality-gate | ||
| 26 | python3 -m unittest discover -s evals -p 'test_*.py' | 27 | python3 -m unittest discover -s evals -p 'test_*.py' |
| 27 | python3 -m unittest discover -s .agents/skills/ops-knowledge-trace-ingest/scripts -p 'test_*.py' | 28 | python3 -m unittest discover -s .agents/skills/ops-knowledge-trace-ingest/scripts -p 'test_*.py' |
| 28 | python3 .agents/skills/ops-knowledge-trace-ingest/scripts/field_notes_lint.py | 29 | python3 .agents/skills/ops-knowledge-trace-ingest/scripts/field_notes_lint.py |
| @@ -9,16 +9,15 @@ | |||
| 9 | # See LICENSE in the root of the software repository for the full text of the License. | 9 | # See LICENSE in the root of the software repository for the full text of the License. |
| 10 | # ---------------------------------------------------------------------------------------------------------- | 10 | # ---------------------------------------------------------------------------------------------------------- |
| 11 | 11 | ||
| 12 | -"""Run auditable retrieval smoke cases against knowledge-query.""" | 12 | +"""Run retrieval cases against knowledge-query.""" |
| 13 | 13 | ||
| 14 | from __future__ import annotations | 14 | from __future__ import annotations |
| 15 | 15 | ||
| 16 | +import argparse | ||
| 16 | import json | 17 | import json |
| 17 | import math | 18 | import math |
| 18 | -import shutil | ||
| 19 | import subprocess | 19 | import subprocess |
| 20 | import sys | 20 | import sys |
| 21 | -import tempfile | ||
| 22 | import time | 21 | import time |
| 23 | from pathlib import Path | 22 | from pathlib import Path |
| 24 | 23 | ||
| @@ -27,46 +26,14 @@ import yaml | |||
| 27 | 26 | ||
| 28 | ROOT = Path(__file__).resolve().parents[1] | 27 | ROOT = Path(__file__).resolve().parents[1] |
| 29 | QUERY = ROOT / ".agents/skills/knowledge-query/scripts/knowledge_query.py" | 28 | QUERY = ROOT / ".agents/skills/knowledge-query/scripts/knowledge_query.py" |
| 30 | -INDEX = ROOT / ".agents/skills/knowledge-query/scripts/knowledge_index.py" | ||
| 31 | CASES = ROOT / "evals/retrieval_cases.yaml" | 29 | CASES = ROOT / "evals/retrieval_cases.yaml" |
| 32 | 30 | ||
| 33 | 31 | ||
| 34 | -def build_synthetic_root(data: dict[str, object], target: Path) -> None: | 32 | +def run_cases( |
| 35 | - shutil.copytree(ROOT / "governance", target / "governance") | 33 | + data: dict[str, object], roots: dict[str, Path], *, quality_gate: bool = False |
| 36 | - (target / "knowledge").mkdir(parents=True) | 34 | +) -> int: |
| 37 | - for fixture in data.get("synthetic_cards", []): | ||
| 38 | - if not isinstance(fixture, dict): | ||
| 39 | - raise ValueError("synthetic_cards entries must be objects") | ||
| 40 | - path = target / str(fixture["path"]) | ||
| 41 | - metadata = fixture["metadata"] | ||
| 42 | - body = str(fixture["body"]).strip() | ||
| 43 | - path.parent.mkdir(parents=True, exist_ok=True) | ||
| 44 | - path.write_text( | ||
| 45 | - "---\n" | ||
| 46 | - + yaml.safe_dump(metadata, allow_unicode=True, sort_keys=False).rstrip() | ||
| 47 | - + "\n---\n\n" | ||
| 48 | - + body | ||
| 49 | - + "\n", | ||
| 50 | - encoding="utf-8", | ||
| 51 | - ) | ||
| 52 | - built = subprocess.run( | ||
| 53 | - [ | ||
| 54 | - sys.executable, | ||
| 55 | - str(INDEX), | ||
| 56 | - "--knowledge-root", | ||
| 57 | - str(target), | ||
| 58 | - "build", | ||
| 59 | - ], | ||
| 60 | - check=False, | ||
| 61 | - text=True, | ||
| 62 | - capture_output=True, | ||
| 63 | - ) | ||
| 64 | - if built.returncode != 0: | ||
| 65 | - raise ValueError(f"cannot build synthetic retrieval index: {built.stderr.strip()}") | ||
| 66 | - | ||
| 67 | - | ||
| 68 | -def run_cases(data: dict[str, object], roots: dict[str, Path]) -> int: | ||
| 69 | failures: list[str] = [] | 35 | failures: list[str] = [] |
| 36 | + misses: list[str] = [] | ||
| 70 | expected_total = 0 | 37 | expected_total = 0 |
| 71 | expected_found = 0 | 38 | expected_found = 0 |
| 72 | reciprocal_ranks: list[float] = [] | 39 | reciprocal_ranks: list[float] = [] |
| @@ -74,12 +41,40 @@ def run_cases(data: dict[str, object], roots: dict[str, Path]) -> int: | |||
| 74 | cases = data.get("cases") | 41 | cases = data.get("cases") |
| 75 | if not isinstance(cases, list): | 42 | if not isinstance(cases, list): |
| 76 | raise ValueError("retrieval cases must be a list") | 43 | raise ValueError("retrieval cases must be a list") |
| 44 | + quality = data.get("quality", {}) | ||
| 45 | + if not isinstance(quality, dict): | ||
| 46 | + raise ValueError("quality must be an object") | ||
| 47 | + if quality_gate: | ||
| 48 | + for key in ("min_recall_at_k", "min_mrr"): | ||
| 49 | + value = quality.get(key) | ||
| 50 | + if ( | ||
| 51 | + isinstance(value, bool) | ||
| 52 | + or not isinstance(value, (int, float)) | ||
| 53 | + or not math.isfinite(value) | ||
| 54 | + or value <= 0 | ||
| 55 | + or value >= 1 | ||
| 56 | + ): | ||
| 57 | + raise ValueError(f"quality gate requires a valid positive {key}") | ||
| 77 | for case in cases: | 58 | for case in cases: |
| 78 | if not isinstance(case, dict): | 59 | if not isinstance(case, dict): |
| 79 | raise ValueError("retrieval case entries must be objects") | 60 | raise ValueError("retrieval case entries must be objects") |
| 80 | corpus = str(case.get("corpus", "repository")) | 61 | corpus = str(case.get("corpus", "repository")) |
| 81 | if corpus not in roots: | 62 | if corpus not in roots: |
| 82 | raise ValueError(f"unknown retrieval corpus: {corpus}") | 63 | raise ValueError(f"unknown retrieval corpus: {corpus}") |
| 64 | + groups = [[path] for path in case.get("expected", [])] + case.get( | ||
| 65 | + "expected_any", [] | ||
| 66 | + ) | ||
| 67 | + if any( | ||
| 68 | + not isinstance(group, list) | ||
| 69 | + or not group | ||
| 70 | + or any(not isinstance(path, str) or not path for path in group) | ||
| 71 | + for group in groups | ||
| 72 | + ): | ||
| 73 | + raise ValueError( | ||
| 74 | + f"{case['id']}: expected groups must be nonempty lists of paths" | ||
| 75 | + ) | ||
| 76 | + expected = {path for group in groups for path in group} | ||
| 77 | + expected_total += len(groups) | ||
| 83 | command = [ | 78 | command = [ |
| 84 | sys.executable, | 79 | sys.executable, |
| 85 | str(QUERY), | 80 | str(QUERY), |
| @@ -114,71 +109,117 @@ def run_cases(data: dict[str, object], roots: dict[str, Path]) -> int: | |||
| 114 | continue | 109 | continue |
| 115 | if completed.returncode != 0: | 110 | if completed.returncode != 0: |
| 116 | failures.append(f"{case['id']}: query failed: {completed.stderr.strip()}") | 111 | failures.append(f"{case['id']}: query failed: {completed.stderr.strip()}") |
| 112 | + if groups: | ||
| 113 | + reciprocal_ranks.append(0.0) | ||
| 117 | continue | 114 | continue |
| 118 | payload = json.loads(completed.stdout) | 115 | payload = json.loads(completed.stdout) |
| 119 | expected_count = case.get("expected_count") | 116 | expected_count = case.get("expected_count") |
| 120 | - count_mismatch = expected_count is not None and payload["count"] != expected_count | 117 | + count_mismatch = ( |
| 118 | + expected_count is not None and payload["count"] != expected_count | ||
| 119 | + ) | ||
| 121 | hit_items = {item["path"]: item for item in payload["hits"]} | 120 | hit_items = {item["path"]: item for item in payload["hits"]} |
| 122 | hits = set(hit_items) | 121 | hits = set(hit_items) |
| 123 | - expected = set(case.get("expected", [])) | 122 | + missing = [group for group in groups if not hits.intersection(group)] |
| 124 | - missing = expected - hits | ||
| 125 | leaked = set(case.get("excluded", [])) & hits | 123 | leaked = set(case.get("excluded", [])) & hits |
| 126 | - expected_total += len(expected) | 124 | + expected_found += len(groups) - len(missing) |
| 127 | - expected_found += len(expected & hits) | ||
| 128 | if expected: | 125 | if expected: |
| 129 | ranks = [hit_items[path]["rank"] for path in expected if path in hit_items] | 126 | ranks = [hit_items[path]["rank"] for path in expected if path in hit_items] |
| 130 | reciprocal_ranks.append(1.0 / min(ranks) if ranks else 0.0) | 127 | reciprocal_ranks.append(1.0 / min(ranks) if ranks else 0.0) |
| 131 | missing_fields: dict[str, list[str]] = {} | 128 | missing_fields: dict[str, list[str]] = {} |
| 132 | - for expected_path in case.get("expected", []): | 129 | + for expected_path in expected: |
| 133 | if expected_path not in hit_items: | 130 | if expected_path not in hit_items: |
| 134 | continue | 131 | continue |
| 135 | fields = case.get("expected_fields", []) | 132 | fields = case.get("expected_fields", []) |
| 136 | - absent = [field for field in fields if field not in hit_items[expected_path]] | 133 | + absent = [ |
| 134 | + field for field in fields if field not in hit_items[expected_path] | ||
| 135 | + ] | ||
| 137 | if absent: | 136 | if absent: |
| 138 | missing_fields[expected_path] = absent | 137 | missing_fields[expected_path] = absent |
| 139 | - if missing or leaked or missing_fields or count_mismatch: | 138 | + out_of_scope = [] |
| 140 | - failures.append( | 139 | + filters = case.get("filters", {}) |
| 140 | + for path, item in hit_items.items(): | ||
| 141 | + for option, field in ( | ||
| 142 | + ("domain", "domain"), | ||
| 143 | + ("technology", "technology"), | ||
| 144 | + ("type", "profile"), | ||
| 145 | + ("status", "status"), | ||
| 146 | + ): | ||
| 147 | + wanted = filters.get(option) | ||
| 148 | + if not wanted or (option == "status" and wanted == "all"): | ||
| 149 | + continue | ||
| 150 | + if ( | ||
| 151 | + option == "technology" | ||
| 152 | + and filters.get("include_shared") | ||
| 153 | + and item.get(field) == "shared" | ||
| 154 | + ): | ||
| 155 | + continue | ||
| 156 | + if item.get(field) != wanted: | ||
| 157 | + out_of_scope.append(f"{path}: {option}") | ||
| 158 | + platforms = filters.get("platform", []) | ||
| 159 | + platforms = [platforms] if isinstance(platforms, str) else platforms | ||
| 160 | + supported = item.get("platforms", []) | ||
| 161 | + if ( | ||
| 162 | + platforms | ||
| 163 | + and "agnostic" not in supported | ||
| 164 | + and not set(platforms) <= set(supported) | ||
| 165 | + ): | ||
| 166 | + out_of_scope.append(f"{path}: platform") | ||
| 167 | + if missing or leaked or missing_fields or count_mismatch or out_of_scope: | ||
| 168 | + # Only recall misses may be covered by aggregate quality thresholds. | ||
| 169 | + # All command, count, field and filtering contracts remain mandatory. | ||
| 170 | + reports = ( | ||
| 171 | + misses | ||
| 172 | + if quality_gate and not (leaked or missing_fields or count_mismatch or out_of_scope) | ||
| 173 | + else failures | ||
| 174 | + ) | ||
| 175 | + reports.append( | ||
| 141 | f"{case['id']}: missing={sorted(missing)} leaked={sorted(leaked)} " | 176 | f"{case['id']}: missing={sorted(missing)} leaked={sorted(leaked)} " |
| 142 | f"missing_fields={missing_fields} count={payload['count']} " | 177 | f"missing_fields={missing_fields} count={payload['count']} " |
| 143 | - f"expected_count={expected_count}" | 178 | + f"expected_count={expected_count} out_of_scope={out_of_scope}" |
| 144 | ) | 179 | ) |
| 145 | else: | 180 | else: |
| 146 | print(f"PASS {case['id']}") | 181 | print(f"PASS {case['id']}") |
| 147 | - quality = data.get("quality", {}) | ||
| 148 | - if not isinstance(quality, dict): | ||
| 149 | - raise ValueError("quality must be an object") | ||
| 150 | recall = expected_found / expected_total if expected_total else None | 182 | recall = expected_found / expected_total if expected_total else None |
| 151 | mrr = sum(reciprocal_ranks) / len(reciprocal_ranks) if reciprocal_ranks else None | 183 | mrr = sum(reciprocal_ranks) / len(reciprocal_ranks) if reciprocal_ranks else None |
| 184 | + if quality_gate and (recall is None or mrr is None): | ||
| 185 | + failures.append("quality: no positive evidence cases to evaluate") | ||
| 152 | ordered_latency = sorted(latencies_ms) | 186 | ordered_latency = sorted(latencies_ms) |
| 153 | p95_index = max(0, math.ceil(len(ordered_latency) * 0.95) - 1) | 187 | p95_index = max(0, math.ceil(len(ordered_latency) * 0.95) - 1) |
| 154 | p95_ms = ordered_latency[p95_index] if ordered_latency else 0.0 | 188 | p95_ms = ordered_latency[p95_index] if ordered_latency else 0.0 |
| 155 | - if recall is not None and recall < float(quality.get("min_recall_at_k", 0.0)): | 189 | + min_recall = float(quality.get("min_recall_at_k", 0.0)) |
| 190 | + min_mrr = float(quality.get("min_mrr", 0.0)) | ||
| 191 | + minimum_operator = ">" if quality_gate else ">=" | ||
| 192 | + if recall is not None and (recall < min_recall or (quality_gate and recall == min_recall)): | ||
| 156 | failures.append( | 193 | failures.append( |
| 157 | - f"quality: recall@k={recall:.3f} below {quality['min_recall_at_k']}" | 194 | + f"quality: recall@k={recall:.3f} must be {minimum_operator} {min_recall}" |
| 158 | - ) | ||
| 159 | - if mrr is not None and mrr < float(quality.get("min_mrr", 0.0)): | ||
| 160 | - failures.append(f"quality: MRR={mrr:.3f} below {quality['min_mrr']}") | ||
| 161 | - if p95_ms > float(quality.get("max_p95_ms", float("inf"))): | ||
| 162 | - failures.append( | ||
| 163 | - f"quality: p95={p95_ms:.1f}ms above {quality['max_p95_ms']}ms" | ||
| 164 | ) | 195 | ) |
| 196 | + if mrr is not None and (mrr < min_mrr or (quality_gate and mrr == min_mrr)): | ||
| 197 | + failures.append(f"quality: MRR={mrr:.3f} must be {minimum_operator} {min_mrr}") | ||
| 165 | recall_text = f"{recall:.3f}" if recall is not None else "N/A" | 198 | recall_text = f"{recall:.3f}" if recall is not None else "N/A" |
| 166 | mrr_text = f"{mrr:.3f}" if mrr is not None else "N/A" | 199 | mrr_text = f"{mrr:.3f}" if mrr is not None else "N/A" |
| 167 | - print(f"QUALITY recall@k={recall_text} MRR={mrr_text} p95={p95_ms:.1f}ms cases={len(cases)}") | 200 | + print( |
| 201 | + f"QUALITY recall@k={recall_text} MRR={mrr_text} p95={p95_ms:.1f}ms cases={len(cases)}" | ||
| 202 | + ) | ||
| 203 | + for miss in misses: | ||
| 204 | + print(f"MISS {miss}", file=sys.stderr) | ||
| 168 | for failure in failures: | 205 | for failure in failures: |
| 169 | print(f"FAIL {failure}", file=sys.stderr) | 206 | print(f"FAIL {failure}", file=sys.stderr) |
| 170 | return 1 if failures else 0 | 207 | return 1 if failures else 0 |
| 171 | 208 | ||
| 172 | 209 | ||
| 173 | def main() -> int: | 210 | def main() -> int: |
| 211 | + parser = argparse.ArgumentParser(description=__doc__) | ||
| 212 | + parser.add_argument( | ||
| 213 | + "--quality-gate", | ||
| 214 | + action="store_true", | ||
| 215 | + help="Require recall/MRR above thresholds; other assertions still fail.", | ||
| 216 | + ) | ||
| 217 | + args = parser.parse_args() | ||
| 174 | try: | 218 | try: |
| 175 | data = yaml.safe_load(CASES.read_text(encoding="utf-8")) | 219 | data = yaml.safe_load(CASES.read_text(encoding="utf-8")) |
| 176 | if not isinstance(data, dict): | 220 | if not isinstance(data, dict): |
| 177 | raise ValueError("retrieval case file must contain an object") | 221 | raise ValueError("retrieval case file must contain an object") |
| 178 | - with tempfile.TemporaryDirectory() as temporary: | 222 | + return run_cases(data, {"repository": ROOT}, quality_gate=args.quality_gate) |
| 179 | - synthetic_root = Path(temporary) / "synthetic-repository" | ||
| 180 | - build_synthetic_root(data, synthetic_root) | ||
| 181 | - return run_cases(data, {"repository": ROOT, "synthetic": synthetic_root}) | ||
| 182 | except (OSError, ValueError, yaml.YAMLError) as exc: | 223 | except (OSError, ValueError, yaml.YAMLError) as exc: |
| 183 | print(f"FAIL retrieval harness: {exc}", file=sys.stderr) | 224 | print(f"FAIL retrieval harness: {exc}", file=sys.stderr) |
| 184 | return 2 | 225 | return 2 |
| @@ -281,14 +281,22 @@ class DocumentationContractTests(unittest.TestCase): | |||
| 281 | 281 | ||
| 282 | def test_knowledge_cards_use_okf_core_fields(self) -> None: | 282 | def test_knowledge_cards_use_okf_core_fields(self) -> None: |
| 283 | root_index = (self.root / "knowledge/index.md").read_text(encoding="utf-8") | 283 | root_index = (self.root / "knowledge/index.md").read_text(encoding="utf-8") |
| 284 | - root_metadata = yaml.safe_load(root_index.split("---", 2)[1]) | 284 | + root_frame = re.match(r"\A---\n(?P<metadata>.*?)\n---\n", root_index, re.DOTALL) |
| 285 | + self.assertIsNotNone(root_frame) | ||
| 286 | + root_metadata = yaml.safe_load(root_frame.group("metadata")) | ||
| 285 | self.assertEqual({"okf_version": "0.2"}, root_metadata) | 287 | self.assertEqual({"okf_version": "0.2"}, root_metadata) |
| 286 | 288 | ||
| 287 | failures: list[str] = [] | 289 | failures: list[str] = [] |
| 288 | for card in sorted((self.root / "knowledge").rglob("*.md")): | 290 | for card in sorted((self.root / "knowledge").rglob("*.md")): |
| 289 | if card.name == "index.md": | 291 | if card.name == "index.md": |
| 290 | continue | 292 | continue |
| 291 | - metadata = yaml.safe_load(card.read_text(encoding="utf-8").split("---", 2)[1]) | 293 | + frame = re.match( |
| 294 | + r"\A---\n(?P<metadata>.*?)\n---\n", | ||
| 295 | + card.read_text(encoding="utf-8"), | ||
| 296 | + re.DOTALL, | ||
| 297 | + ) | ||
| 298 | + self.assertIsNotNone(frame, card) | ||
| 299 | + metadata = yaml.safe_load(frame.group("metadata")) | ||
| 292 | if not str(metadata.get("type", "")).strip(): | 300 | if not str(metadata.get("type", "")).strip(): |
| 293 | failures.append(f"{card.relative_to(self.root)}: missing type") | 301 | failures.append(f"{card.relative_to(self.root)}: missing type") |
| 294 | if metadata.get("status") not in {"draft", "stable", "deprecated"}: | 302 | if metadata.get("status") not in {"draft", "stable", "deprecated"}: |
| @@ -617,9 +625,16 @@ class DocumentationContractTests(unittest.TestCase): | |||
| 617 | ) | 625 | ) |
| 618 | failures: list[str] = [] | 626 | failures: list[str] = [] |
| 619 | for card in sorted((self.root / "knowledge").rglob("*.md")): | 627 | for card in sorted((self.root / "knowledge").rglob("*.md")): |
| 620 | - for number, line in enumerate(card.read_text(encoding="utf-8").splitlines(), start=1): | 628 | + text = card.read_text(encoding="utf-8") |
| 629 | + frame = re.match(r"\A---\n.*?\n---\n", text, re.DOTALL) | ||
| 630 | + body = text[frame.end():] if frame else text | ||
| 631 | + for number, line in enumerate(body.splitlines(), start=1): | ||
| 632 | + # A valid link's target and a card's own filename are not prose | ||
| 633 | + # references to a retired card suffix. | ||
| 634 | + line = re.sub(r"\]\([^)]*\)", "]", line) | ||
| 621 | if stale.search(line): | 635 | if stale.search(line): |
| 622 | - failures.append(f"{card.relative_to(self.root)}:{number}") | 636 | + offset = text[: frame.end()].count("\n") if frame else 0 |
| 637 | + failures.append(f"{card.relative_to(self.root)}:{number + offset}") | ||
| 623 | self.assertEqual([], failures) | 638 | self.assertEqual([], failures) |
| 624 | 639 | ||
| 625 | def test_skill_sources_do_not_contain_agent_vendor_metadata(self) -> None: | 640 | def test_skill_sources_do_not_contain_agent_vendor_metadata(self) -> None: |
| @@ -311,7 +311,6 @@ class RouteContractTests(unittest.TestCase): | |||
| 311 | self.assertEqual(expected | {"knowledge/ops/ascendc", "knowledge/ops/shared"}, actual) | 311 | self.assertEqual(expected | {"knowledge/ops/ascendc", "knowledge/ops/shared"}, actual) |
| 312 | self.assertTrue(all((ROOT / path / "index.md").is_file() for path in expected)) | 312 | self.assertTrue(all((ROOT / path / "index.md").is_file() for path in expected)) |
| 313 | not_precreated = { | 313 | not_precreated = { |
| 314 | - "knowledge/contrib/ascendc", | ||
| 315 | "knowledge/common/toolchains", | 314 | "knowledge/common/toolchains", |
| 316 | "knowledge/common/observability", | 315 | "knowledge/common/observability", |
| 317 | "knowledge/common/glossaries", | 316 | "knowledge/common/glossaries", |
| @@ -19,9 +19,10 @@ import subprocess | |||
| 19 | import sys | 19 | import sys |
| 20 | import tempfile | 20 | import tempfile |
| 21 | import unittest | 21 | import unittest |
| 22 | -from contextlib import redirect_stdout | 22 | +from contextlib import redirect_stderr, redirect_stdout |
| 23 | from pathlib import Path | 23 | from pathlib import Path |
| 24 | from urllib.parse import unquote | 24 | from urllib.parse import unquote |
| 25 | +from unittest.mock import patch | ||
| 25 | 26 | ||
| 26 | from evals.run_retrieval import run_cases | 27 | from evals.run_retrieval import run_cases |
| 27 | 28 | ||
| @@ -82,6 +83,143 @@ def run_index(root: Path, *arguments: str) -> subprocess.CompletedProcess[str]: | |||
| 82 | ) | 83 | ) |
| 83 | 84 | ||
| 84 | 85 | ||
| 86 | +class RetrievalCaseTests(unittest.TestCase): | ||
| 87 | + def run_mocked_case(self, case, hits, returncode=0, *, quality=None, quality_gate=False): | ||
| 88 | + output, errors = io.StringIO(), io.StringIO() | ||
| 89 | + completed = subprocess.CompletedProcess( | ||
| 90 | + [], returncode, json.dumps({"count": len(hits), "hits": hits}), "query failed" | ||
| 91 | + ) | ||
| 92 | + with patch("evals.run_retrieval.subprocess.run", return_value=completed), \ | ||
| 93 | + patch("evals.run_retrieval.time.perf_counter", side_effect=[0.0, 1.0]), \ | ||
| 94 | + redirect_stdout(output), redirect_stderr(errors): | ||
| 95 | + result = run_cases( | ||
| 96 | + {"cases": [{"id": "fixture", "query": "query", **case}], "quality": quality or {}}, | ||
| 97 | + {"repository": ROOT}, quality_gate=quality_gate, | ||
| 98 | + ) | ||
| 99 | + return result, output.getvalue(), errors.getvalue() | ||
| 100 | + | ||
| 101 | + def test_alternatives_do_not_replace_other_required_evidence(self): | ||
| 102 | + case = {"expected": ["c"], "expected_any": [["a", "b"]]} | ||
| 103 | + for paths, expected_result, recall in ((["b", "c"], 0, "1.000"), | ||
| 104 | + (["a", "b"], 1, "0.500"), | ||
| 105 | + (["c"], 1, "0.500")): | ||
| 106 | + with self.subTest(paths=paths): | ||
| 107 | + hits = [{"path": path, "rank": rank} for rank, path in enumerate(paths, 1)] | ||
| 108 | + result, output, _ = self.run_mocked_case(case, hits) | ||
| 109 | + self.assertEqual(expected_result, result) | ||
| 110 | + self.assertIn(f"recall@k={recall}", output) | ||
| 111 | + | ||
| 112 | + def test_failed_query_stays_in_quality_denominator(self): | ||
| 113 | + result, output, _ = self.run_mocked_case({"expected_any": [["a", "b"]]}, [], 2) | ||
| 114 | + self.assertEqual(1, result) | ||
| 115 | + self.assertIn("recall@k=0.000 MRR=0.000", output) | ||
| 116 | + | ||
| 117 | + def test_scope_is_checked_even_when_expected_card_is_returned(self): | ||
| 118 | + case = {"expected": ["a"], "filters": {"type": "apis", "platform": ["a2", "a3"]}} | ||
| 119 | + for profile, platforms, expected_result in ( | ||
| 120 | + ("apis", ["a2", "a3"], 0), ("apis", ["agnostic"], 0), | ||
| 121 | + ("runbooks", ["a2", "a3"], 1), ("apis", ["a2"], 1), | ||
| 122 | + ): | ||
| 123 | + with self.subTest(profile=profile, platforms=platforms): | ||
| 124 | + result, _, _ = self.run_mocked_case( | ||
| 125 | + case, [{"path": "a", "rank": 1, "profile": profile, "platforms": platforms}] | ||
| 126 | + ) | ||
| 127 | + self.assertEqual(expected_result, result) | ||
| 128 | + | ||
| 129 | + def test_invalid_or_groups_are_not_vacuous_success(self): | ||
| 130 | + for groups in ([[]], ["not-a-list"], [[""]]): | ||
| 131 | + with self.subTest(groups=groups), self.assertRaisesRegex(ValueError, "nonempty"): | ||
| 132 | + self.run_mocked_case({"expected_any": groups}, []) | ||
| 133 | + | ||
| 134 | + def test_quality_gate_accepts_partial_recall_above_threshold_but_strict_mode_fails(self): | ||
| 135 | + case = {"expected": ["c"], "expected_any": [["a", "b"]]} | ||
| 136 | + quality = {"min_recall_at_k": 0.49, "min_mrr": 0.49} | ||
| 137 | + hits = [{"path": "unrelated", "rank": 1}, {"path": "b", "rank": 2}] | ||
| 138 | + for quality_gate, expected_result in ((False, 1), (True, 0)): | ||
| 139 | + with self.subTest(quality_gate=quality_gate): | ||
| 140 | + result, output, errors = self.run_mocked_case( | ||
| 141 | + case, hits, quality=quality, quality_gate=quality_gate, | ||
| 142 | + ) | ||
| 143 | + self.assertEqual(expected_result, result) | ||
| 144 | + self.assertIn("recall@k=0.500 MRR=0.500 p95=1000.0ms", output) | ||
| 145 | + self.assertIn("MISS fixture:" if quality_gate else "FAIL fixture:", errors) | ||
| 146 | + self.assertNotIn("PASS fixture", output) | ||
| 147 | + | ||
| 148 | + def test_quality_gate_rejects_each_metric_at_or_below_target(self): | ||
| 149 | + quality = {"min_recall_at_k": 0.49, "min_mrr": 0.49} | ||
| 150 | + for key, value, metric in ( | ||
| 151 | + ("min_recall_at_k", 0.5, "recall@k"), | ||
| 152 | + ("min_recall_at_k", 0.501, "recall@k"), | ||
| 153 | + ("min_mrr", 0.5, "MRR"), | ||
| 154 | + ("min_mrr", 0.501, "MRR"), | ||
| 155 | + ): | ||
| 156 | + with self.subTest(key=key): | ||
| 157 | + result, _, errors = self.run_mocked_case( | ||
| 158 | + {"expected": ["a", "b"]}, [{"path": "a", "rank": 2}], | ||
| 159 | + quality={**quality, key: value}, quality_gate=True, | ||
| 160 | + ) | ||
| 161 | + self.assertEqual(1, result) | ||
| 162 | + self.assertIn(f"FAIL quality: {metric}", errors) | ||
| 163 | + self.assertEqual(1, errors.count("FAIL quality:")) | ||
| 164 | + | ||
| 165 | + def test_default_mode_keeps_inclusive_quality_thresholds(self): | ||
| 166 | + result, _, errors = self.run_mocked_case( | ||
| 167 | + {"expected": ["a"]}, [{"path": "a", "rank": 1}], | ||
| 168 | + quality={"min_recall_at_k": 1.0, "min_mrr": 1.0}, | ||
| 169 | + ) | ||
| 170 | + self.assertEqual(0, result) | ||
| 171 | + self.assertEqual("", errors) | ||
| 172 | + | ||
| 173 | + def test_quality_gate_preserves_hard_contract_failures(self): | ||
| 174 | + quality = {"min_recall_at_k": 0.49, "min_mrr": 0.49} | ||
| 175 | + for assertion, hits, returncode in ( | ||
| 176 | + ({}, [], 2), | ||
| 177 | + ({"expected_error": "platform is required"}, [], 2), | ||
| 178 | + ({"expected_count": 0}, [{"path": "a", "rank": 1}], 0), | ||
| 179 | + ({"excluded": ["b"]}, [{"path": "a", "rank": 1}, {"path": "b", "rank": 2}], 0), | ||
| 180 | + ({"expected_fields": ["description"]}, [{"path": "a", "rank": 1}], 0), | ||
| 181 | + ({"filters": {"platform": ["950"]}}, [{"path": "a", "rank": 1, "platforms": ["a2"]}], 0), | ||
| 182 | + ({"filters": {"type": "apis"}}, [{"path": "a", "rank": 1, "profile": "runbooks"}], 0), | ||
| 183 | + ): | ||
| 184 | + with self.subTest(assertion=assertion, returncode=returncode): | ||
| 185 | + result, _, errors = self.run_mocked_case( | ||
| 186 | + {"expected": ["a"], **assertion}, hits, returncode, | ||
| 187 | + quality=quality, quality_gate=True, | ||
| 188 | + ) | ||
| 189 | + self.assertEqual(1, result) | ||
| 190 | + self.assertIn("FAIL fixture:", errors) | ||
| 191 | + self.assertNotIn("MISS fixture:", errors) | ||
| 192 | + | ||
| 193 | + def test_quality_gate_rejects_missing_or_invalid_thresholds(self): | ||
| 194 | + quality = {"min_recall_at_k": 0.5, "min_mrr": 0.5} | ||
| 195 | + for key in quality: | ||
| 196 | + invalid = (None, True, 0, -1, "0.5", float("nan"), float("inf")) | ||
| 197 | + invalid += (1, 1.1) | ||
| 198 | + for value in invalid: | ||
| 199 | + with self.subTest(key=key, value=value), self.assertRaisesRegex(ValueError, key): | ||
| 200 | + self.run_mocked_case( | ||
| 201 | + {"expected": ["a"]}, [], | ||
| 202 | + quality={**quality, key: value}, quality_gate=True, | ||
| 203 | + ) | ||
| 204 | + with self.subTest(missing=key), self.assertRaisesRegex(ValueError, key): | ||
| 205 | + self.run_mocked_case( | ||
| 206 | + {"expected": ["a"]}, [], | ||
| 207 | + quality={k: v for k, v in quality.items() if k != key}, quality_gate=True, | ||
| 208 | + ) | ||
| 209 | + | ||
| 210 | + def test_quality_gate_rejects_no_positive_evidence_cases(self): | ||
| 211 | + quality = {"min_recall_at_k": 0.5, "min_mrr": 0.5} | ||
| 212 | + for cases in ([], [{"id": "empty", "query": "query", "expected_count": 0}]): | ||
| 213 | + output, errors = io.StringIO(), io.StringIO() | ||
| 214 | + completed = subprocess.CompletedProcess([], 0, '{"count": 0, "hits": []}', "") | ||
| 215 | + with patch("evals.run_retrieval.subprocess.run", return_value=completed), \ | ||
| 216 | + redirect_stdout(output), redirect_stderr(errors): | ||
| 217 | + result = run_cases({"cases": cases, "quality": quality}, | ||
| 218 | + {"repository": ROOT}, quality_gate=True) | ||
| 219 | + self.assertEqual(1, result) | ||
| 220 | + self.assertIn("no positive evidence cases", errors.getvalue()) | ||
| 221 | + | ||
| 222 | + | ||
| 85 | class RetrievalIndexTests(unittest.TestCase): | 223 | class RetrievalIndexTests(unittest.TestCase): |
| 86 | def test_kirin_api_filter_keeps_platform_boundaries(self) -> None: | 224 | def test_kirin_api_filter_keeps_platform_boundaries(self) -> None: |
| 87 | with tempfile.TemporaryDirectory() as temporary: | 225 | with tempfile.TemporaryDirectory() as temporary: |
| @@ -51,7 +51,7 @@ aliases: ["OL-412"] | |||
| 51 | .PrecisionReduceFlag(true) | 51 | .PrecisionReduceFlag(true) |
| 52 | .ExtendCfgInfo("opFile.value", "adaptive_avg_pool3d_apt"); | 52 | .ExtendCfgInfo("opFile.value", "adaptive_avg_pool3d_apt"); |
| 53 | this->AICore().AddConfig("ascend950", aicore_config_950); | 53 | this->AICore().AddConfig("ascend950", aicore_config_950); |
| 54 | - | 54 | + |
| 55 | OpAICoreConfig ascend310p_config; | 55 | OpAICoreConfig ascend310p_config; |
| 56 | ascend310p_config.Input("x") | 56 | ascend310p_config.Input("x") |
| 57 | .ParamType(REQUIRED) | 57 | .ParamType(REQUIRED) |
| @@ -28,6 +28,7 @@ aliases: ["OL-542"] | |||
| 28 | 28 | ||
| 29 | **Concrete anchor** (the load-bearing line is the per-call clone): | 29 | **Concrete anchor** (the load-bearing line is the per-call clone): |
| 30 | ```python | 30 | ```python |
| 31 | +``` | ||
| 31 | 32 | ||
| 32 | **孪生卡 ol-646 合并的独有内容**（ol-646-in-place-cann-ops-aclnnapply-mutate-their-inputs.md 已删除）： | 33 | **孪生卡 ol-646 合并的独有内容**（ol-646-in-place-cann-ops-aclnnapply-mutate-their-inputs.md 已删除）： |
| 33 | 34 | ||
| @@ -29,6 +29,7 @@ The two rules are complementary, not contradictory: the question is whether NZ l | |||
| 29 | 29 | ||
| 30 | **Concrete anchor**: | 30 | **Concrete anchor**: |
| 31 | ```python | 31 | ```python |
| 32 | +``` | ||
| 32 | 33 | ||
| 33 | **孪生卡 ol-647 合并的独有内容**（ol-647-allow-internal-format-true-reorders-within-the-i.md 已删除）： | 34 | **孪生卡 ol-647 合并的独有内容**（ol-647-allow-internal-format-true-reorders-within-the-i.md 已删除）： |
| 34 | 35 | ||
| @@ -28,6 +28,20 @@ PR，且第 2 步不能修复第 1 步提交本身不可自检的问题；建议 | |||
| 28 | 合入后 master 新生成器 + 新索引自洽、check.sh 恢复绿色。该说明将随 | 28 | 合入后 master 新生成器 + 新索引自洽、check.sh 恢复绿色。该说明将随 |
| 29 | 回复提交给 zhanghz1 确认。 | 29 | 回复提交给 zhanghz1 确认。 |
| 30 | 30 | ||
| 31 | +## [11:09] eval | 用真实检索场景替换简单 case | ||
| 32 | + | ||
| 33 | +### Summary | ||
| 34 | + | ||
| 35 | +将 29 条轨迹查询和 34 个人工场景直接迁入 `evals/retrieval_cases.yaml`，沿用 `evals/run_retrieval.py` 和现有 CI 入口。 | ||
| 36 | + | ||
| 37 | +### Details | ||
| 38 | + | ||
| 39 | +- 保留 63 例的 query、过滤条件、top5、必要证据及来源定位；原质量阈值保持不变。原脚本增加等价候选 OR 分组和返回范围检查，查询失败仍计入质量分母。 | ||
| 40 | +- 移除本次新增的设计/复查文档、版本快照及独立评测/提取脚本；README、AGENTS 恢复原状，仅保留这条简短变更日志。此前过程材料仍可从 Git 历史查看。 | ||
| 41 | +- 迁移一致性核对、5 项定向计分/范围回归和 ruff 检查通过。 | ||
| 42 | +- `bash check.sh` 的 Lint、导航、索引构建和完整校验通过；63 例为 39 通过、24 例缺必要证据，无范围违规。证据组 recall@5=0.545、MRR=0.477、p95=1143.0ms，均未达到原阈值，门禁在检索阶段失败。 | ||
| 43 | +- 补跑后续检查：仓库单测 270 项，266 通过、1 跳过、2 失败、1 错误；后三项分别为旧 A5 后缀、既有 contrib 目录、测试 Frontmatter 切分问题，已在原始基线复现。轨迹工具 36 项、正文 HARD 约束、全库 ruff、Shell 语法及归档差异检查通过。原始输出保存在 ignored 的 `artifacts/retrieval/simplified-*-20260924.log`。 | ||
| 44 | + | ||
| 31 | ## [09:47] update | 索引导航摘要子目录计数修复为仅统计一级子目录 | 45 | ## [09:47] update | 索引导航摘要子目录计数修复为仅统计一级子目录 |
| 32 | 46 | ||
| 33 | ### Summary | 47 | ### Summary |
| @@ -52,4 +66,3 @@ PR，且第 2 步不能修复第 1 步提交本身不可自检的问题；建议 | |||
| 52 | - 验证：`evals.test_governance.IndexSummaryTests` 9/9 通过 | 66 | - 验证：`evals.test_governance.IndexSummaryTests` 9/9 通过 |
| 53 | （7 既有 + 2 新增：`test_deep_nesting_counts_first_level_subdirectories_only`、 | 67 | （7 既有 + 2 新增：`test_deep_nesting_counts_first_level_subdirectories_only`、 |
| 54 | `test_deep_nesting_with_overview_counts_first_level_only`）。 | 68 | `test_deep_nesting_with_overview_counts_first_level_only`）。 |
| 55 | - | ||
| @@ -1,5 +1,40 @@ | |||
| 1 | # 2026-09-28 | 1 | # 2026-09-28 |
| 2 | 2 | ||
| 3 | +## [20:17] test | 移除检索 P95 质量门禁 | ||
| 4 | + | ||
| 5 | +### Summary | ||
| 6 | + | ||
| 7 | +按请求移除 P95 阈值限制，检索质量门禁只校验 Recall@5 和 MRR；P95 继续输出用于观察，不再导致检查失败。 | ||
| 8 | + | ||
| 9 | +### Details | ||
| 10 | + | ||
| 11 | +- `retrieval_cases.yaml` 只保留 `min_recall_at_k` 和 `min_mrr`。 | ||
| 12 | +- `run_retrieval.py --quality-gate` 不再校验或比较 P95 上限。 | ||
| 13 | +- 更新质量门禁回归测试，保留 Recall/MRR 的严格阈值和其他硬断言。 | ||
| 14 | + | ||
| 15 | +## [19:18] test | 放宽检索 P95 门槛 | ||
| 16 | + | ||
| 17 | +### Summary | ||
| 18 | + | ||
| 19 | +按请求将质量门禁的 P95 上限从 1100 ms 调整为严格小于 1200 ms；Recall@5 和 MRR 门槛保持不变。 | ||
| 20 | + | ||
| 21 | +### Details | ||
| 22 | + | ||
| 23 | +- `retrieval_cases.yaml` 设置 `max_p95_ms: 1200`。 | ||
| 24 | +- `run_retrieval.py --quality-gate` 继续使用严格 `<` 比较，等于 1200 ms 时仍失败。 | ||
| 25 | + | ||
| 26 | +## [19:14] test | 调整检索基线门槛 | ||
| 27 | + | ||
| 28 | +### Summary | ||
| 29 | + | ||
| 30 | +按请求将检索质量门槛改为 Recall@5 > 0.54、MRR > 0.45、P95 < 1100 ms，并保持 case 级硬断言继续阻断检查。 | ||
| 31 | + | ||
| 32 | +### Details | ||
| 33 | + | ||
| 34 | +- `retrieval_cases.yaml` 设置 `min_recall_at_k: 0.54`、`min_mrr: 0.45`、`max_p95_ms: 1100`。 | ||
| 35 | +- `run_retrieval.py --quality-gate` 对 Recall/MRR 使用严格大于、对 P95 使用严格小于；默认非门禁调用仍保留原有包含边界。 | ||
| 36 | +- 增加等于门槛时失败、默认包含边界和阈值校验测试。 | ||
| 37 | + | ||
| 3 | ## [17:00] update | !58 重构为索引专用 PR 并 rebase 至 0fe7f05d | 38 | ## [17:00] update | !58 重构为索引专用 PR 并 rebase 至 0fe7f05d |
| 4 | 39 | ||
| 5 | ### Summary | 40 | ### Summary |
| @@ -13,6 +48,18 @@ | |||
| 13 | - 评测机在 0fe7f05d 上复核 `sync-indexes --check`：662 索引 85 stale；再生成 Synchronized 85、unchanged 577，改动路径集与本 PR 85 个文件完全一致，apis/guides 内容与 098cb4a3 逐字节一致； | 48 | - 评测机在 0fe7f05d 上复核 `sync-indexes --check`：662 索引 85 stale；再生成 Synchronized 85、unchanged 577，改动路径集与本 PR 85 个文件完全一致，apis/guides 内容与 098cb4a3 逐字节一致； |
| 14 | - `logs/2026-09-28.md` 按时间降序合并 PR #67 侧 [03:39]/[07:24] 条目、本分支 [09:26] 条目与本条目；`logs/2026-09-22.md` 沿用此前两侧合并版（master 侧在 1e015cdc 后无变化）。 | 49 | - `logs/2026-09-28.md` 按时间降序合并 PR #67 侧 [03:39]/[07:24] 条目、本分支 [09:26] 条目与本条目；`logs/2026-09-22.md` 沿用此前两侧合并版（master 侧在 1e015cdc 后无变化）。 |
| 15 | 50 | ||
| 51 | +## [16:25] test | 将检索门禁改为基线之上的质量阈值 | ||
| 52 | + | ||
| 53 | +### Summary | ||
| 54 | + | ||
| 55 | +为检索 case 评测增加显式质量门禁模式，并把阈值设为略优于当前 70 例基线；漏召回只参与聚合指标，查询错误、数量断言、过滤范围和字段断言仍然阻断检查。 | ||
| 56 | + | ||
| 57 | +### Details | ||
| 58 | + | ||
| 59 | +- `check.sh` 使用 `evals/run_retrieval.py --quality-gate`；Recall@5 门槛为 0.550，MRR 门槛为 0.460，P95 门槛为 1100 ms。 | ||
| 60 | +- 当前基线为 Recall@5=0.548、MRR=0.459、P95=1108.4 ms，因此当前基线会被新门禁拒绝，要求后续检索改进后才能通过。 | ||
| 61 | +- 增加阈值校验、无正向证据拒绝、硬断言保留和聚合门禁回归测试；定向测试 10 项通过。 | ||
| 62 | + | ||
| 16 | ## [09:26] update | 按维护者意见清理日志并 rebase 最新 master（!58 终审准备） | 63 | ## [09:26] update | 按维护者意见清理日志并 rebase 最新 master（!58 终审准备） |
| 17 | 64 | ||
| 18 | ### Summary | 65 | ### Summary |