- F10 bm25 section search: stdlib SQLite FTS5 over page sections, with title/heading path/tldr in a separate context column; cached in tmp/ckb-fts.sqlite, refreshed by mtime then sha256; section anchors per candidate. New benchmark tests/bench_retrieval.py over tests/fixtures/bench-kb (20 questions): hit@1 14/20, hit@3 20/20 with and without bm25; weight kept at 1.0 (D-0015) - F12 token estimate per candidate; layered reading in ckb-retrieve - F16 --opened access log in tmp/access.log; lint_report.py --cold proposes demotion to an index `## Archive` section, never moves - F17 ckb-retrieve offers a table/chart/deck/HTML once when it fits Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
42 lines
1.6 KiB
Python
42 lines
1.6 KiB
Python
#!/usr/bin/env python3
|
|
"""Retrieval benchmark: hit@1 and hit@3 over tests/bench/retrieval.tsv against
|
|
tests/fixtures/bench-kb, with and without the bm25 section signal.
|
|
|
|
python3 tests/bench_retrieval.py # print the table
|
|
python3 tests/bench_retrieval.py --check # exit 1 if bm25 makes hit@3 worse
|
|
"""
|
|
import json
|
|
import subprocess
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
HERE = Path(__file__).resolve().parent
|
|
RETR = HERE.parent / ".agents/skills/ckb-retrieve/scripts/retrieve_candidates.py"
|
|
KB = HERE / "fixtures/bench-kb"
|
|
|
|
|
|
def run(question, extra):
|
|
out = subprocess.run([sys.executable, str(RETR), question, "--root", str(KB), "--today", "2026-11-02", "--json"] + extra,
|
|
capture_output=True, text=True, check=True).stdout
|
|
return [c["path"] for c in json.loads(out)["candidates"] if c["path"].startswith("wiki/entities/") and not c["path"].endswith("index.md")]
|
|
|
|
|
|
def main():
|
|
cases = [l.split("\t") for l in (HERE / "bench/retrieval.tsv").read_text().splitlines() if l and not l.startswith("#")]
|
|
scores = {}
|
|
for label, extra in (("without bm25", ["--no-bm25"]), ("with bm25", [])):
|
|
h1 = h3 = 0
|
|
for q, want in cases:
|
|
got = run(q, extra)
|
|
h1 += got[:1] == [want]
|
|
h3 += want in got[:3]
|
|
scores[label] = (h1, h3)
|
|
print(f"{label:<13} hit@1 {h1}/{len(cases)} hit@3 {h3}/{len(cases)}")
|
|
if "--check" in sys.argv and scores["with bm25"][1] < scores["without bm25"][1]:
|
|
print("bm25 made hit@3 worse")
|
|
return 1
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|