"""Compare two bootstrap ComparisonReport.json files and produce a cross-machine Markdown diff report.
Cross-machine comparison semantics:
- Each ComparisonReport contains two tracks (A and B) from the same machine.
- Cross-machine comparison compares the SAME track across reports:
Report A.track_a vs Report B.track_a, Report A.track_b vs Report B.track_b.
- File differences are cross-report: all files modified in Report A vs all in Report B.
- Diff pattern comparison compares the between-track diffs found on each machine.
Usage:
python3 scripts/compare_reports.py <report_a.json> <report_b.json> [--output <path>] [--verbose]
exit code:
0 — comparison completed (even if task IDs differ, if --continue-on-mismatch)
2 — task_id mismatch (default, stderr + exit 2)
"""
import json
import sys
import argparse
DIFF_CATEGORIES = {"prompt", "tool", "orchestration", "model", "token"}
DIFF_SEVERITIES = {"critical", "major", "minor", "info"}
def load_report(path: str) -> dict:
"""Load and validate a ComparisonReport JSON file."""
with open(path) as f:
report = json.load(f)
if "task" not in report or "track_a" not in report or "track_b" not in report:
raise ValueError(f"{path}: missing required field (task/track_a/track_b)")
if "task_id" not in report.get("task", {}):
raise ValueError(f"{path}: task missing task_id")
return report
def extract_track_metrics(track: dict, label: str) -> dict:
"""Extract metrics from a single track ExecutionResult."""
meta = track.get("comparison_metadata") or {}
total = track.get("total_count", 0)
completed = track.get("completed_count", 0)
tokens = track.get("total_tokens") or 0
files = set(track.get("modified_files", []))
timed_out = "_timed_out" in meta.get("provider_type", "")
return {
"label": label,
"completed": completed,
"total": total,
"completion_rate": completed / max(total, 1),
"tokens": tokens,
"provider": meta.get("provider_type", "unknown"),
"model": meta.get("model_version") or meta.get("provider_type", "unknown"),
"commit": meta.get("commit_hash"),
"files": files,
"timed_out": timed_out,
}
def extract_diff_metrics(report: dict) -> dict:
"""Extract diff-specific metrics from a ComparisonReport."""
diffs = report.get("diffs", [])
categories = {}
severities = {}
for d in diffs:
cat = d.get("category", "unknown")
sev = d.get("severity", "info")
categories[cat] = categories.get(cat, 0) + 1
severities[sev] = severities.get(sev, 0) + 1
return {"total": len(diffs), "categories": categories, "severities": severities}
def extract_report_metrics(report: dict) -> dict:
"""Extract full metrics from a ComparisonReport."""
track_a = extract_track_metrics(report.get("track_a", {}), "A")
track_b = extract_track_metrics(report.get("track_b", {}), "B")
diff = extract_diff_metrics(report)
all_files = track_a["files"] | track_b["files"]
return {
"task_id": report.get("task", {}).get("task_id", "unknown"),
"description": report.get("task", {}).get("description", ""),
"track_a": track_a,
"track_b": track_b,
"diff": diff,
"all_files": all_files,
}
def _severity_key(level: str) -> int:
"""Map severity level to numeric for max comparison."""
return {"": 0, "info": 1, "warn": 2}.get(level, 0)
def check_magnitude(ma: dict, mb: dict) -> tuple[str, str]:
"""Check magnitude divergence between two reports.
Compares token consumption, completion rounds, and completion rate
across all same-track cross-report combinations. Takes the highest
severity across all dimensions. Returns (level, detail) where level
is one of: "", "info", "warn". "" means no meaningful divergence.
Thresholds (三级, per spec diagnosis-output-contract.md §2):
Token: >30% warn | >10% and ≤30% info
Completion (rounds): ≥2 warn | 1 info
Completion rate: ≥1 warn (no info level)
"""
level: str = ""
details: list[str] = []
token_values = [
ma['track_a']['tokens'], mb['track_a']['tokens'],
ma['track_b']['tokens'], mb['track_b']['tokens'],
]
max_token = max(token_values)
max_token_diff = max(
abs(ma['track_a']['tokens'] - mb['track_a']['tokens']),
abs(ma['track_b']['tokens'] - mb['track_b']['tokens']),
)
if max_token == 0:
pass
else:
pct = max_token_diff / max_token * 100
if pct > 30:
level = "warn"
details.append(f"token {pct:.0f}%(>30%)")
elif pct > 10:
level = "info"
details.append(f"token {pct:.0f}%(>10%)")
max_comp_diff = max(
abs(ma['track_a']['completed'] - mb['track_a']['completed']),
abs(ma['track_b']['completed'] - mb['track_b']['completed']),
)
if max_comp_diff >= 2:
level = max(level, "warn", key=_severity_key)
details.append(f"completion {max_comp_diff}(≥2)")
elif max_comp_diff == 1:
level = max(level, "info", key=_severity_key)
details.append(f"completion 1")
max_rate_diff = max(
abs(ma['track_a']['completion_rate'] - mb['track_a']['completion_rate']),
abs(ma['track_b']['completion_rate'] - mb['track_b']['completion_rate']),
)
if max_rate_diff >= 1:
level = max(level, "warn", key=_severity_key)
details.append(f"rate {max_rate_diff:.0f}(≥1)")
if not level:
return ("", "")
elif level == "info":
return ("info", ", ".join(details))
else:
return ("warn", ", ".join(details))
def render_comparison(ma: dict, mb: dict, verbose: bool = False) -> str:
"""Render cross-machine comparison as Markdown.
Args:
ma: metrics from Report A (PC/master)
mb: metrics from Report B (remote/experiment)
"""
lines = []
lines.append("# Cross-Machine Bootstrap Comparison Report\n\n")
lines.append("## Overview\n\n")
lines.append(f"- **Report A (master / PC)**: task `{ma['task_id']}`\n")
lines.append(f"- **Report B (experiment / remote)**: task `{mb['task_id']}`\n")
lines.append(f"- **Description**: {ma['description']}\n")
lines.append("\n")
if ma["task_id"] != mb["task_id"]:
lines.append(
"> ⚠️ **Task ID mismatch** — these reports may not be from the same task. "
"Comparison is best-effort.\n\n"
)
lines.append("---\n")
lines.append("## Track A — Cross-Report Comparison\n\n")
lines.append("Comparing the same track (A) across both machines.\n\n")
lines.append("| Metric | Master (Report A) | Remote (Report B) | Difference |\n")
lines.append("|--------|:-----------------:|:------------------:|:----------:|\n")
_render_track_comparison(ma["track_a"], mb["track_a"], lines)
lines.append("---\n")
lines.append("## Track B — Cross-Report Comparison\n\n")
lines.append("Comparing the same track (B) across both machines.\n\n")
lines.append("| Metric | Master (Report A) | Remote (Report B) | Difference |\n")
lines.append("|--------|:-----------------:|:------------------:|:----------:|\n")
_render_track_comparison(ma["track_b"], mb["track_b"], lines)
lines.append("---\n")
lines.append("## Modified Files — Cross-Report\n\n")
all_a = ma["all_files"]
all_b = mb["all_files"]
only_in_a = all_a - all_b
only_in_b = all_b - all_a
in_both = all_a & all_b
lines.append(f"| Set | Count |\n")
lines.append(f"|-----|:----:|\n")
lines.append(f"| Only in Master (A) | {len(only_in_a)} |\n")
lines.append(f"| Only in Remote (B) | {len(only_in_b)} |\n")
lines.append(f"| In Both | {len(in_both)} |\n\n")
if only_in_a:
lines.append("### Only in Master (Report A)\n\n")
for f in sorted(only_in_a):
lines.append(f"- `{f}`\n")
lines.append("\n")
if only_in_b:
lines.append("### Only in Remote (Report B)\n\n")
for f in sorted(only_in_b):
lines.append(f"- `{f}`\n")
lines.append("\n")
if not only_in_a and not only_in_b:
lines.append("Both reports modified the exact same set of files.\n\n")
lines.append("---\n")
lines.append("## Diff Pattern Comparison\n\n")
lines.append(
"Compares the _between-track_ differences detected on each machine. "
"This shows whether both machines observed the same behavioral divergence.\n\n"
)
da, db = ma["diff"], mb["diff"]
lines.append("| Metric | Master (Report A) | Remote (Report B) |\n")
lines.append("|--------|:-----------------:|:------------------:|\n")
lines.append(f"| Total Diffs | {da['total']} | {db['total']} |\n")
all_cats = sorted(set(da["categories"]) | set(db["categories"]))
lines.append("\n### By Category\n\n")
lines.append("| Category | Master (A) | Remote (B) |\n")
lines.append("|----------|:----------:|:----------:|\n")
for cat in all_cats:
ca = da["categories"].get(cat, 0)
cb = db["categories"].get(cat, 0)
if ca != 0 or cb != 0:
lines.append(f"| {cat} | {ca} | {cb} |\n")
all_sevs = sorted(
set(da["severities"]) | set(db["severities"]),
key=lambda s: {"critical": 0, "major": 1, "minor": 2, "info": 3}.get(s, 99),
)
lines.append("\n### By Severity\n\n")
lines.append("| Severity | Master (A) | Remote (B) |\n")
lines.append("|----------|:----------:|:----------:|\n")
for sev in all_sevs:
sa = da["severities"].get(sev, 0)
sb = db["severities"].get(sev, 0)
if sa != 0 or sb != 0:
lines.append(f"| {sev} | {sa} | {sb} |\n")
lines.append("\n## Diagnosis\n\n")
if ma["task_id"] != mb["task_id"]:
lines.append("⚠️ Task IDs differ — reports may not be comparable.\n")
if da["total"] == 0 and db["total"] == 0:
pattern = "✅ consistent"
elif da["categories"] == db["categories"]:
pattern = "✅ consistent"
else:
pattern = "⚠️ differs"
mag_level, mag_detail = check_magnitude(ma, mb)
if mag_level == "warn":
magnitude = f"⚠️ divergence ({mag_detail})"
elif mag_level == "info":
magnitude = f"ℹ️ divergence ({mag_detail})"
else:
magnitude = "✅ consistent"
lines.append(f"Pattern: {pattern} | Magnitude: {magnitude}\n")
commits_a = {ma['track_a'].get('commit'), ma['track_b'].get('commit')}
commits_b = {mb['track_a'].get('commit'), mb['track_b'].get('commit')}
all_commits = (commits_a | commits_b) - {None}
if len(all_commits) > 1:
lines.append(
"⚠️ **Commit hashes differ** between reports — "
"the two machines may be running different code versions.\n"
)
elif len(all_commits) == 0:
lines.append(
"ℹ️ Commit hashes not recorded (git unavailable during bootstrap).\n"
)
if verbose:
lines.append("\n## Verbose Details\n\n")
lines.append("```json\n")
lines.append("Report A (Master):\n")
lines.append(json.dumps(ma, indent=2, default=str))
lines.append("\n```\n\n")
lines.append("```json\n")
lines.append("Report B (Remote):\n")
lines.append(json.dumps(mb, indent=2, default=str))
lines.append("\n```\n")
return "".join(lines)
def _render_track_comparison(track_a: dict, track_b: dict, lines: list):
"""Append track comparison rows to lines list."""
lines.append(
f"| Completion | {track_a['completed']}/{track_a['total']} "
f"({track_a['completion_rate']:.0%}) | "
f"{track_b['completed']}/{track_b['total']} "
f"({track_b['completion_rate']:.0%}) | "
f"{track_a['completed'] - track_b['completed']:+d} |\n"
)
ta_tok = track_a["tokens"]
tb_tok = track_b["tokens"]
tok_diff = ta_tok - tb_tok
tok_diff_pct = abs(tok_diff) / max(max(ta_tok, tb_tok), 1) * 100
lines.append(
f"| Token Consumption | {ta_tok} | {tb_tok} | "
f"{tok_diff:+d} ({tok_diff_pct:.1f}%) |\n"
)
lines.append(
f"| Model | {track_a['model']} | {track_b['model']} | "
f"{'same' if track_a['model'] == track_b['model'] else 'different'} |\n"
)
ca = track_a.get("commit")
cb = track_b.get("commit")
if ca and cb:
commit_status = (
"✅ same" if ca == cb
else "⚠️ DIFFERENT"
)
elif ca or cb:
commit_status = "⚠️ one report missing commit hash"
else:
commit_status = "— (not recorded)"
ca_display = ca[:12] if ca else "—"
cb_display = cb[:12] if cb else "—"
lines.append(
f"| Commit | `{ca_display}` | `{cb_display}` | {commit_status} |\n"
)
lines.append(
f"| Provider | {track_a['provider']} | {track_b['provider']} | — |\n"
)
if track_a["timed_out"] or track_b["timed_out"]:
status_a = "⏱️ TIMEOUT" if track_a["timed_out"] else "OK"
status_b = "⏱️ TIMEOUT" if track_b["timed_out"] else "OK"
lines.append(f"| Timeout | {status_a} | {status_b} | — |\n")
files_only_a = track_a["files"] - track_b["files"]
files_only_b = track_b["files"] - track_a["files"]
lines.append(f"| Unique Files | {len(files_only_a)} | {len(files_only_b)} | — |\n")
def main():
parser = argparse.ArgumentParser(
description="Compare two bootstrap ComparisonReport.json files"
)
parser.add_argument("report_a", help="Path to first ComparisonReport.json (master/PC)")
parser.add_argument("report_b", help="Path to second ComparisonReport.json (experiment/remote)")
parser.add_argument("--output", "-o", help="Write report to file instead of stdout")
parser.add_argument(
"--verbose", "-v", action="store_true", help="Output verbose JSON details"
)
parser.add_argument(
"--continue-on-mismatch",
action="store_true",
help="Continue with exit 0 even if task IDs differ (default: exit 2)",
)
args = parser.parse_args()
report_a = load_report(args.report_a)
report_b = load_report(args.report_b)
task_ids_match = report_a.get("task", {}).get("task_id") == report_b.get("task", {}).get(
"task_id"
)
if not task_ids_match:
msg = (
f"ERROR: task_id mismatch — "
f"Report A: {report_a.get('task', {}).get('task_id')}, "
f"Report B: {report_b.get('task', {}).get('task_id')}\n"
)
if not args.continue_on_mismatch:
sys.stderr.write(msg)
sys.exit(2)
sys.stderr.write(msg)
sys.stderr.write("Continuing with --continue-on-mismatch\n")
ma = extract_report_metrics(report_a)
mb = extract_report_metrics(report_b)
report_md = render_comparison(ma, mb, verbose=args.verbose)
if args.output:
with open(args.output, "w") as f:
f.write(report_md)
print(f"Report saved to {args.output}")
else:
print(report_md)
def _make_report(task_id: str, commit_a: str | None, commit_b: str | None) -> dict:
"""Build a minimal valid ComparisonReport dict with controlled commit hashes."""
return {
"task": {"task_id": task_id, "description": ""},
"track_a": {
"total_count": 0,
"completed_count": 0,
"total_tokens": 0,
"modified_files": [],
"comparison_metadata": {
"provider_type": "mock",
"model_version": "mock",
"commit_hash": commit_a,
},
},
"track_b": {
"total_count": 0,
"completed_count": 0,
"total_tokens": 0,
"modified_files": [],
"comparison_metadata": {
"provider_type": "mock",
"model_version": "mock",
"commit_hash": commit_b,
},
},
"diffs": [],
}
def test_commit_hash_same():
"""两报告相同 commit_hash → Markdown 含 '✅ same'"""
report = _make_report("test-001", "abc123def456", "abc123def456")
ma = extract_report_metrics(report)
mb = extract_report_metrics(report)
output = render_comparison(ma, mb)
assert "✅ same" in output, f"Expected '✅ same' in commit row, got:\n{output}"
def test_commit_hash_different():
"""不同 commit_hash → Markdown 含 '⚠️ DIFFERENT'"""
report_a = _make_report("test-001", "abc123def456", "abc123def456")
report_b = _make_report("test-001", "xyz789ghi012", "xyz789ghi012")
ma = extract_report_metrics(report_a)
mb = extract_report_metrics(report_b)
output = render_comparison(ma, mb)
assert "⚠️ DIFFERENT" in output, f"Expected '⚠️ DIFFERENT' in commit row, got:\n{output}"
def test_commit_hash_missing():
"""commit_hash=None → Markdown 含 'not recorded'"""
report = _make_report("test-001", None, None)
ma = extract_report_metrics(report)
mb = extract_report_metrics(report)
output = render_comparison(ma, mb)
assert "not recorded" in output, f"Expected 'not recorded' in commit row, got:\n{output}"
def _make_metrics(
task_id: str,
ta_tok: int, ta_comp: int, ta_total: int,
tb_tok: int, tb_comp: int, tb_total: int,
commit: str = "abc123",
diff: dict | None = None,
) -> dict:
"""Build a minimal metrics dict for check_magnitude / render_comparison tests."""
return {
"task_id": task_id,
"description": "",
"track_a": {
"label": "A",
"completed": ta_comp,
"total": ta_total,
"completion_rate": ta_comp / max(ta_total, 1),
"tokens": ta_tok,
"provider": "mock",
"model": "mock",
"commit": commit,
"files": set(),
"timed_out": False,
},
"track_b": {
"label": "B",
"completed": tb_comp,
"total": tb_total,
"completion_rate": tb_comp / max(tb_total, 1),
"tokens": tb_tok,
"provider": "mock",
"model": "mock",
"commit": commit,
"files": set(),
"timed_out": False,
},
"diff": diff or {"total": 0, "categories": {}, "severities": {}},
"all_files": set(),
}
def test_magnitude_token_warn():
"""Token diff >30% → warn (T1-8)"""
ma = _make_metrics("t", 10000, 1, 1, 10000, 1, 1)
mb = _make_metrics("t", 5000, 1, 1, 5000, 1, 1)
level, detail = check_magnitude(ma, mb)
assert level == "warn", f"Expected warn for 50% token diff, got ({level}, {detail})"
def test_magnitude_token_info():
"""Token diff >10% and ≤30% → info (T1-9)"""
ma = _make_metrics("t", 100, 1, 1, 100, 1, 1)
mb = _make_metrics("t", 80, 1, 1, 80, 1, 1)
level, detail = check_magnitude(ma, mb)
assert level == "info", f"Expected info for 20% token diff, got ({level}, {detail})"
def test_magnitude_completion_warn():
"""Completion diff ≥2 → warn (T1-10)"""
ma = _make_metrics("t", 100, 2, 2, 100, 1, 1)
mb = _make_metrics("t", 100, 6, 6, 100, 1, 1)
level, detail = check_magnitude(ma, mb)
assert level == "warn", f"Expected warn for completion diff 4, got ({level}, {detail})"
def test_magnitude_completion_info():
"""Completion diff ==1 → info (T1-11)"""
ma = _make_metrics("t", 100, 1, 1, 100, 1, 1)
mb = _make_metrics("t", 100, 2, 2, 100, 1, 1)
level, detail = check_magnitude(ma, mb)
assert level == "info", f"Expected info for completion diff 1, got ({level}, {detail})"
def test_magnitude_completion_rate_warn():
"""Completion rate diff ≥1 → warn (T1-12, B1 fix)"""
ma = _make_metrics("t", 100, 1, 1, 100, 0, 1)
mb = _make_metrics("t", 100, 0, 1, 100, 0, 1)
level, detail = check_magnitude(ma, mb)
assert level == "warn", f"Expected warn for rate 1.0 vs 0.0, got ({level}, {detail})"
def test_magnitude_multi_dim_highest():
"""Multi-dimension: take highest severity (T1-13)"""
ma = _make_metrics("t", 100, 2, 2, 100, 1, 1)
mb = _make_metrics("t", 85, 3, 3, 100, 1, 1)
level, detail = check_magnitude(ma, mb)
assert level == "info", f"Expected info for multi-dim info+info, got ({level}, {detail})"
def test_magnitude_no_divergence():
"""Identical reports → no divergence (T1 consistent baseline)"""
ma = _make_metrics("t", 100, 1, 1, 100, 1, 1)
mb = _make_metrics("t", 100, 1, 1, 100, 1, 1)
level, detail = check_magnitude(ma, mb)
assert level == "", f"Expected no divergence, got ({level}, {detail})"
def test_magnitude_zero_tokens_all():
"""All four token values are 0 → no crash, consistent (T1-14, B1)"""
ma = _make_metrics("t", 0, 0, 0, 0, 0, 0)
mb = _make_metrics("t", 0, 0, 0, 0, 0, 0)
level, detail = check_magnitude(ma, mb)
assert level == "", f"Expected consistent for all-zero tokens, got ({level}, {detail})"
def test_magnitude_zero_tokens_one_side():
"""One report tokens=0, other=1000 → 100% warn (T1-15, B1/TS1)"""
ma = _make_metrics("t", 0, 0, 1, 0, 0, 1)
mb = _make_metrics("t", 1000, 0, 1, 1000, 0, 1)
level, detail = check_magnitude(ma, mb)
assert level == "warn", f"Expected warn for 0 vs 1000 (100%), got ({level}, {detail})"
def test_magnitude_zero_tokens_single_track():
"""Single track tokens=0, other track >0 → no crash (T1-16)"""
ma = _make_metrics("t", 0, 0, 1, 5000, 2, 2)
mb = _make_metrics("t", 1000, 1, 1, 5000, 2, 2)
level, detail = check_magnitude(ma, mb)
assert level in ("", "info", "warn"), f"Unexpected level for single-track zero: ({level}, {detail})"
def test_magnitude_golden_regression():
"""Golden regression: PC 5766tok/2轮 vs km 12010tok/6轮 → warn (T1-17/T1-18)
Source: v0.80.0 real self-bootstrap commit c69692b4, 52% token diff + 4 completion diff."""
ma = _make_metrics("t", 5766, 2, 2, 5766, 2, 2, commit="c69692b4")
mb = _make_metrics("t", 12010, 6, 6, 12010, 6, 6, commit="c69692b4")
level, detail = check_magnitude(ma, mb)
assert level == "warn", f"Golden regression: expected warn, got ({level}, {detail})"
output = render_comparison(ma, mb)
assert "Pattern: ✅ consistent" in output, f"Expected Pattern consistent for same diff categories"
assert "Magnitude: ⚠️ divergence" in output, f"Expected Magnitude divergence for 52%/4"
def test_magnitude_pattern_differs_with_warn():
"""Pattern differs + magnitude warn → both appear in Diagnosis (T1-6)"""
ma = _make_metrics("t", 5766, 2, 2, 5766, 2, 2,
diff={"total": 1, "categories": {"prompt": 1}, "severities": {"info": 1}})
mb = _make_metrics("t", 12010, 6, 6, 12010, 6, 6,
diff={"total": 1, "categories": {"token": 1}, "severities": {"info": 1}})
output = render_comparison(ma, mb)
assert "Pattern: ⚠️ differs" in output, f"Expected Pattern differs"
assert "Magnitude: ⚠️ divergence" in output, f"Expected Magnitude divergence"
if __name__ == "__main__":
main()