ckb/.agents/skills/ckb-retrieve/scripts/retrieve_candidates.py
Michał Kopeć 1d7a489323 Template 1.24.0: release 3 batch 1, retrieval
- F10 bm25 section search: stdlib SQLite FTS5 over page sections, with
  title/heading path/tldr in a separate context column; cached in
  tmp/ckb-fts.sqlite, refreshed by mtime then sha256; section anchors
  per candidate. New benchmark tests/bench_retrieval.py over
  tests/fixtures/bench-kb (20 questions): hit@1 14/20, hit@3 20/20 with
  and without bm25; weight kept at 1.0 (D-0015)
- F12 token estimate per candidate; layered reading in ckb-retrieve
- F16 --opened access log in tmp/access.log; lint_report.py --cold
  proposes demotion to an index `## Archive` section, never moves
- F17 ckb-retrieve offers a table/chart/deck/HTML once when it fits

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-02 20:25:26 +01:00

526 lines
22 KiB
Python
Executable file
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

#!/usr/bin/env python3
"""Mechanical half of ckb-retrieve: gather ranked candidate lists from several
local signals and fuse them with reciprocal rank fusion.
python3 retrieve_candidates.py "question text" [--root DIR] [--k 10]
[--top 12] [--scope SLUG] [--tag TAG ...] [--no-bm25] [--json]
python3 retrieve_candidates.py --opened PATH [PATH ...] # access log: pages actually read
Signals, each its own ranked list (weight in brackets):
exact rg/grep hits on literal tokens, across wiki/, raw/archive/, raw/inbox/,
outputs/, linked/*/ and connector-backed libs/*/ [1.0, or 2.0
when the question carries literal-looking tokens]
tldr title + tldr token overlap for every frontmatter page [1.0]
scope pages listed by a matching wiki/projects/ scope page [1.0]
graph one-hop neighbours of the text-matched shortlist [0.7]
indegree how many edges point at a shortlisted page [0.5]
trust confidence x quality x freshness, over the shortlist [0.3]
bm25 SQLite FTS5 over page sections, each chunk prefixed with
"title › heading path — tldr" (contextual retrieval); a page
ranks by its best section, and the hit sections are returned [1.0]
The bm25 index lives in tmp/ckb-fts.sqlite (in memory when tmp/ is absent)
and refreshes incrementally by mtime, then sha256. Without FTS5 in this
Python's sqlite3 the list is skipped and the header says so.
--tag keeps only wiki pages carrying every named tag (schema 1.6 `tags:`).
Fused score = sum(weight / (k + rank)) over the lists a candidate appears in.
`k` defaults to 10: a local wiki yields a dozen candidates, not hundreds, and
k=60 flattens them into near-identical scores.
The script ranks; it does not judge. Dedupe-by-claim, rerank against the literal
question, and source verification stay with the skill. It never writes.
"""
import argparse
import hashlib
import json
import os
import re
import shutil
import sqlite3
import subprocess
import sys
from collections import Counter, defaultdict
from datetime import date, datetime
from pathlib import Path
STOP = set("""a an and are as at be by for from how in is it of on or that the this to
was we what when where which who why with do does did our your their about into
""".split())
MD_EXCLUDE = {"index.md", "log.md"}
# ---------------------------------------------------------------- parsing ---
def parse_frontmatter(text):
if not text.startswith("---\n"):
return {}, text
end = text.find("\n---", 4)
if end < 0:
return {}, text
fm = {}
for line in text[4:end].splitlines():
m = re.match(r"^([A-Za-z_][\w-]*):\s*(.*)$", line)
if m:
fm[m.group(1)] = m.group(2).strip().strip('"').strip("'")
return fm, text[end + 4:]
def title_of(body, fallback):
m = re.search(r"^#\s+(.+)$", body, re.M)
return m.group(1).strip() if m else fallback
def tokens(s):
return [t for t in re.findall(r"[a-z0-9][a-z0-9_./:-]{1,}", s.lower()) if t not in STOP and len(t) > 1]
def literal_tokens(q):
"""Things that should match exactly: quoted phrases, paths, flags, ids, error-ish strings."""
lits = re.findall(r'"([^"]{3,})"|\'([^\']{3,})\'|`([^`]{2,})`', q)
out = [next(x for x in t if x) for t in lits]
for t in re.findall(r"\S+", q):
if re.search(r"[/\\.\-_]\w|^--?\w|\d{2,}|[A-Z]{2,}-\d+|^[A-Z][A-Z0-9_]{3,}$", t) and len(t) > 2:
out.append(t.strip(",.;:?!()"))
return list(dict.fromkeys(out))
def is_fresh(fm, today):
lu, win = fm.get("last_updated"), fm.get("freshness_window_days")
if not lu or not win:
return None
try:
return (today - datetime.strptime(lu, "%Y-%m-%d").date()).days <= int(win)
except ValueError:
return None
# ---------------------------------------------------------------- corpus ----
def page_roots(root):
yield root / "wiki"
linked = root / "linked"
if linked.is_dir():
for d in sorted(linked.iterdir()):
if d.is_dir():
yield d
libs = root / "libs"
if libs.is_dir():
for d in sorted(libs.iterdir()):
if d.is_dir() and (d / "source.yaml").exists():
yield d # connector index only; a git-copy lib is not indexed here
def load_pages(root):
pages = {}
for base in page_roots(root):
for p in sorted(base.rglob("*.md")):
if p.name in MD_EXCLUDE or "/.git/" in p.as_posix():
continue
try:
text = p.read_text(encoding="utf-8", errors="replace")
except OSError:
continue
fm, body = parse_frontmatter(text)
if not fm.get("type") and base.name == "wiki":
continue
rel = p.relative_to(root).as_posix()
pages[rel] = {
"path": rel, "fm": fm, "title": title_of(body, p.stem),
"sources": re.findall(r"^\s*-\s+`([^`]+)`", body.split("## Sources", 1)[1].split("\n## ", 1)[0], re.M)
if "## Sources" in body else [],
}
return pages
def load_edges(root):
edges = []
for f in [root / "wiki/graph/edges.json", *root.glob("libs/*/graph/edges.json")]:
if f.exists():
try:
edges += json.loads(f.read_text(encoding="utf-8")).get("edges", [])
except (OSError, ValueError):
pass
return edges
def norm_ref(ref):
return ref.lstrip("/").removeprefix("wiki/").removesuffix(".md")
# ---------------------------------------------------------------- signals ---
def signal_exact(root, toks, lits):
"""Ranked by (distinct tokens matched, total hits)."""
targets = [d for d in ["wiki", "raw/archive", "raw/inbox", "outputs", "linked"] if (root / d).exists()]
targets += [d.relative_to(root).as_posix() for d in (root / "libs").glob("*") if (d / "source.yaml").exists()]
if not targets:
return [], set()
needles = list(dict.fromkeys(lits + [t for t in toks if len(t) >= 4]))[:12]
per_file = defaultdict(lambda: [set(), 0])
rg = shutil.which("rg")
for n in needles:
if rg:
cmd = [rg, "-c", "-i", "-F", "--follow", "--no-messages", "-g", "!*.json", "-g", "!.git", "--", n, *targets]
else:
cmd = ["grep", "-r", "-c", "-i", "-F", "--exclude=*.json", "--exclude-dir=.git", "--", n, *targets]
try:
out = subprocess.run(cmd, cwd=root, capture_output=True, text=True, timeout=30).stdout
except (OSError, subprocess.TimeoutExpired):
continue
for line in out.splitlines():
path, _, cnt = line.rpartition(":")
if not path or not cnt.isdigit() or int(cnt) == 0:
continue
per_file[path][0].add(n)
per_file[path][1] += int(cnt)
ranked = sorted(per_file.items(), key=lambda kv: (-len(kv[1][0]), -kv[1][1], kv[0]))
inbox = {p for p in per_file if p.startswith("raw/inbox/")}
return [p for p, _ in ranked], inbox
def signal_tldr(pages, toks):
qs = set(toks)
scored = []
for rel, pg in pages.items():
hay = set(tokens(pg["title"] + " " + pg["fm"].get("tldr", "")))
hit = len(qs & hay)
if hit:
scored.append((-(hit / max(len(qs), 1)), -hit, rel))
return [r for _, _, r in sorted(scored)]
def signal_scope(root, pages, toks, forced):
proj = root / "wiki/projects"
if not proj.is_dir():
return [], None
qs = set(toks)
best, best_hit = None, 0
for p in proj.glob("*.md"):
if p.name == "index.md":
continue
if forced and p.stem == forced:
best = p
break
pg = pages.get(p.relative_to(root).as_posix())
hit = len(qs & set(tokens((pg["title"] if pg else "") + " " + (pg["fm"].get("tldr", "") if pg else "") + " " + p.stem)))
if hit > best_hit:
best, best_hit = p, hit
if not best:
return [], None
body = best.read_text(encoding="utf-8", errors="replace")
refs = re.findall(r"\]\(([^)]+\.md)\)|\[\[([^\]]+)\]\]|`((?:wiki|raw|libs|outputs)/[^`]+)`", body)
listed = []
for a, b, c in refs:
r = norm_ref(a or b or c)
cand = next((rel for rel in pages if norm_ref(rel) == r or rel.endswith(r + ".md")), None)
if cand and cand not in listed:
listed.append(cand)
return listed, best.relative_to(root).as_posix()
def signal_graph(edges, pages, shortlist):
short = {norm_ref(s) for s in shortlist}
neigh = Counter()
for e in edges:
a, b = norm_ref(str(e.get("from", ""))), norm_ref(str(e.get("to", "")))
if a in short and b not in short:
neigh[b] += 1
if b in short and a not in short:
neigh[a] += 1
out = []
for n, _ in neigh.most_common():
cand = next((rel for rel in pages if norm_ref(rel) == n or rel.endswith("/" + n + ".md")), None)
if cand:
out.append(cand)
return out
def signal_indegree(edges, shortlist):
indeg = Counter(norm_ref(str(e.get("to", ""))) for e in edges)
scored = [(indeg[norm_ref(s)], s) for s in shortlist if indeg[norm_ref(s)] > 0]
return [s for _, s in sorted(scored, key=lambda x: (-x[0], x[1]))]
def signal_trust(pages, shortlist, today):
scored = []
for s in shortlist:
pg = pages.get(s)
if not pg:
continue
fm = pg["fm"]
try:
c, q = float(fm.get("confidence", 0.5)), float(fm.get("quality", 0.5))
except ValueError:
c, q = 0.5, 0.5
fresh = is_fresh(fm, today)
f = 1.0 if fresh in (True, None) else 0.5
scored.append((-(c * q * f), s))
return [s for _, s in sorted(scored)]
# ---------------------------------------------------------------- sections --
FTS_DB = Path("tmp") / "ckb-fts.sqlite"
FTS_SCHEMA = "3"
HEADING_RE = re.compile(r"^(#{1,4})[ \t]+(.+?)[ \t]*#*[ \t]*$", re.M)
ACCESS_LOG = Path("tmp") / "access.log"
def anchor(heading):
"""GitHub-style heading anchor."""
return re.sub(r"\s+", "-", re.sub(r"[^\w\s-]", "", heading.strip().lower()))
def chunks(title, tldr, body):
"""[(anchor, heading path, context, text)]: one chunk per heading section,
the text before the first `##` as the lead. The context column carries the
page title, heading path and tldr, so a section still says what it is about
alone; it counts for ranking pages, not for naming the matching section."""
out, stack, start, current = [], [], 0, ("", [])
marks = [m for m in HEADING_RE.finditer(body) if len(m.group(1)) >= 2]
for i, m in enumerate(marks + [None]):
end = m.start() if m else len(body)
text = body[start:end].strip()
if text:
path = current[1]
prefix = f"{title} › {' › '.join(path)} — {tldr}" if path else f"{title} — {tldr}"
out.append((current[0], " › ".join(path), prefix, text))
if m is None:
break
level, heading = len(m.group(1)), m.group(2).strip()
stack = [h for h in stack if h[0] < level] + [(level, heading)]
current, start = (anchor(heading), [h for _, h in stack]), m.end()
return out
def open_index(root):
"""A connection with the FTS5 schema in place, or None when FTS5 is missing."""
path = root / FTS_DB if (root / "tmp").is_dir() else None
for attempt in (1, 2):
try:
db = sqlite3.connect(str(path) if path else ":memory:")
db.execute("CREATE TABLE IF NOT EXISTS meta(key TEXT PRIMARY KEY, value TEXT)")
row = db.execute("SELECT value FROM meta WHERE key='schema'").fetchone()
if row is None or row[0] != FTS_SCHEMA:
db.executescript("DROP TABLE IF EXISTS chunks; DROP TABLE IF EXISTS files;")
db.execute("INSERT OR REPLACE INTO meta VALUES('schema', ?)", (FTS_SCHEMA,))
db.execute("CREATE VIRTUAL TABLE IF NOT EXISTS chunks USING "
"fts5(path UNINDEXED, anchor UNINDEXED, heading UNINDEXED, context, text, tokenize='porter unicode61')")
db.execute("CREATE TABLE IF NOT EXISTS files(path TEXT PRIMARY KEY, mtime REAL, sha TEXT)")
return db
except sqlite3.OperationalError as e:
if "fts5" in str(e).lower():
return None
raise
except sqlite3.DatabaseError:
if path and attempt == 1 and path.exists():
path.unlink() # a corrupt cache is rebuilt, never trusted
continue
raise
return None
def refresh_index(db, root, pages):
"""Re-chunk only files whose mtime moved and whose bytes actually changed."""
known = {p: (m, h) for p, m, h in db.execute("SELECT path, mtime, sha FROM files")}
for rel, pg in pages.items():
f = root / rel
try:
mtime = f.stat().st_mtime
except OSError:
continue
old = known.pop(rel, None)
if old and old[0] == mtime:
continue
data = f.read_bytes()
sha = hashlib.sha256(data).hexdigest()
if old and old[1] == sha:
db.execute("UPDATE files SET mtime=? WHERE path=?", (mtime, rel))
continue
db.execute("DELETE FROM chunks WHERE path=?", (rel,))
body = parse_frontmatter(data.decode("utf-8", "replace"))[1]
db.executemany("INSERT INTO chunks(path, anchor, heading, context, text) VALUES(?,?,?,?,?)",
[(rel, a, h, c, t) for a, h, c, t in chunks(pg["title"], pg["fm"].get("tldr", ""), body)])
db.execute("INSERT OR REPLACE INTO files VALUES(?,?,?)", (rel, mtime, sha))
for gone in known:
db.execute("DELETE FROM chunks WHERE path=?", (gone,))
db.execute("DELETE FROM files WHERE path=?", (gone,))
db.commit()
def signal_bm25(root, pages, toks, enabled=True):
"""(pages ranked by their best section, {page: [(anchor, heading)]}, status)."""
if not enabled:
return [], {}, "off"
db = open_index(root)
if db is None:
return [], {}, "unavailable (no FTS5 in this Python's sqlite3)"
try:
refresh_index(db, root, pages)
terms = list(dict.fromkeys(t for t in toks if len(t) > 1))[:16]
if not terms:
return [], {}, "ok"
query = " OR ".join('"' + t.replace('"', '""') + '"' for t in terms)
ranked = []
for (path,) in db.execute("SELECT path FROM chunks WHERE chunks MATCH ? ORDER BY bm25(chunks) LIMIT 200", (query,)):
if path not in ranked:
ranked.append(path)
# Name a section only when its own text matches, not merely its page's context.
sections = defaultdict(list)
for path, anc, heading in db.execute(
"SELECT path, anchor, heading FROM chunks WHERE chunks MATCH ? ORDER BY bm25(chunks) LIMIT 200",
(f"text : ({query})",)):
if heading and heading not in ("Sources", "Notes") and len(sections[path]) < 3:
sections[path].append((anc, heading))
return ranked, dict(sections), "ok"
finally:
db.close()
def log_opened(root, paths, today):
"""Append pages a person or agent actually read to tmp/access.log (per machine, untracked)."""
if not (root / "tmp").is_dir():
sys.exit("error: no tmp/ folder here — the access log lives in tmp/access.log")
with (root / ACCESS_LOG).open("a", encoding="utf-8") as fh:
for p in paths:
fh.write(f"{today.isoformat()}\t{p.lstrip('/')}\n")
print(f"logged {len(paths)} page(s) in {ACCESS_LOG.as_posix()}")
def est_tokens(root, rel):
try:
return -(-(root / rel).stat().st_size // 4)
except OSError:
return None
# ---------------------------------------------------------------- fusion ----
def rrf(lists, k):
score, ranks = defaultdict(float), defaultdict(dict)
for name, (items, w) in lists.items():
for i, item in enumerate(items, 1):
score[item] += w / (k + i)
ranks[item][name] = i
return sorted(score.items(), key=lambda kv: (-kv[1], kv[0])), ranks
def main():
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument("query", nargs="?")
ap.add_argument("--root", default=".")
ap.add_argument("--k", type=int, default=10)
ap.add_argument("--top", type=int, default=12)
ap.add_argument("--scope", help="force a wiki/projects/<slug> scope instead of matching one")
ap.add_argument("--today", help="YYYY-MM-DD, for reproducible freshness")
ap.add_argument("--tag", action="append", default=[], help="keep only pages with this tag (repeatable)")
ap.add_argument("--no-bm25", action="store_true", help="skip the section index (benchmarks)")
ap.add_argument("--opened", nargs="+", metavar="PATH", help="log pages actually read, then exit")
ap.add_argument("--json", action="store_true")
a = ap.parse_args()
root = Path(a.root).resolve()
today = datetime.strptime(a.today, "%Y-%m-%d").date() if a.today else date.today()
if a.opened:
log_opened(root, a.opened, today)
return
if not a.query:
ap.error("a question is required (or --opened PATH ...)")
toks, lits = tokens(a.query), literal_tokens(a.query)
pages, edges = load_pages(root), load_edges(root)
exact, inbox_hits = signal_exact(root, toks, lits)
tldr = signal_tldr(pages, toks)
scope, scope_page = signal_scope(root, pages, toks, a.scope)
bm25, sections, bm25_status = signal_bm25(root, pages, toks, not a.no_bm25)
shortlist = list(dict.fromkeys(tldr[:a.top] + [e for e in exact[:a.top] if e in pages] + scope + bm25[:a.top]))
graph = signal_graph(edges, pages, shortlist)
indeg = signal_indegree(edges, shortlist)
trust = signal_trust(pages, shortlist, today)
lists = {
"exact": (exact, 2.0 if lits else 1.0),
"tldr": (tldr, 1.0),
"scope": (scope, 1.0),
"graph": (graph, 0.7),
"indegree": (indeg, 0.5),
"trust": (trust, 0.3),
"bm25": (bm25, 1.0),
}
fused, ranks = rrf({n: v for n, v in lists.items() if v[0]}, a.k)
# Fold a cited raw/archive or outputs/ hit into the wiki page that cites it:
# same claim, one candidate, corroboration recorded rather than double-counted.
# A source cited by N pages corroborates all N, not just the first one walked.
cited_by = defaultdict(list)
for rel, pg in sorted(pages.items()):
for src in pg["sources"]:
cited_by[src.lstrip("/")].append(rel)
results, absorbed = [], defaultdict(list)
ranked = dict(fused)
for item, sc in fused:
owners = [o for o in cited_by.get(item, []) if o in ranked]
if owners:
for o in owners:
absorbed[o].append(item)
continue
results.append((item, sc))
if a.tag:
want = {t.strip().lower() for t in a.tag}
tagged = lambda pg: {t.strip().lower() for t in pg["fm"].get("tags", "").strip("[]").split(",") if t.strip()}
results = [(i, sc) for i, sc in results if i in pages and want <= tagged(pages[i])]
results = results[:a.top]
out = {
"query": a.query, "k": a.k, "literal_tokens": lits, "scope_page": scope_page, "tags": a.tag,
"inbox_hits": sorted(inbox_hits), "empty_graph": not edges, "bm25": bm25_status,
"candidates": [],
}
for item, sc in results:
pg = pages.get(item)
fm = pg["fm"] if pg else {}
out["candidates"].append({
"path": item, "score": round(sc, 4), "ranks": ranks[item],
"title": pg["title"] if pg else None, "type": fm.get("type"), "tldr": fm.get("tldr"),
"last_updated": fm.get("last_updated"), "stale": (is_fresh(fm, today) is False) if pg else None,
"confidence": fm.get("confidence"), "quality": fm.get("quality"),
"superseded_by": fm.get("superseded_by"), "review": fm.get("review"),
"sources": pg["sources"] if pg else [], "corroborated_by": absorbed.get(item, []),
"from_inbox": item.startswith("raw/inbox/"),
"sections": [{"anchor": an, "heading": h} for an, h in sections.get(item, [])],
"tokens": est_tokens(root, item),
})
if a.json:
json.dump(out, sys.stdout, indent=2, ensure_ascii=False)
print()
return
print(f"query: {a.query}")
if lits:
print(f"literal tokens (exact list weighted 2.0): {', '.join(lits)}")
if scope_page:
print(f"scope: {scope_page}")
if a.tag:
print(f"tags: only pages tagged {', '.join(a.tag)}")
if out["empty_graph"]:
print("graph: empty — graph/indegree lists skipped")
if bm25_status not in ("ok", "off"):
print(f"bm25: {bm25_status} — section search skipped")
if inbox_hits:
print(f"raw/inbox/ hits ({len(inbox_hits)}): un-ingested material — an ingest is overdue")
print()
for i, c in enumerate(out["candidates"], 1):
flags = [f for f, on in [("STALE", c["stale"]), ("INBOX", c["from_inbox"]), ("SUPERSEDED", c["superseded_by"]),
("DRAFT", c["review"] == "draft"), ("NEEDS_UPDATE", c["review"] == "needs_update")] if on]
rk = " ".join(f"{n}:{r}" for n, r in sorted(c["ranks"].items()))
cost = f" ~{c['tokens']} tok" if c["tokens"] else ""
print(f"{i:2}. {c['score']:.4f} {c['path']} [{rk}]{cost}" + (f" {' '.join(flags)}" if flags else ""))
if c["tldr"]:
print(f" {c['tldr']}")
if c["sections"]:
print(" sections: " + ", ".join(f"#{s['anchor']} ({s['heading']})" for s in c["sections"]))
if c["corroborated_by"]:
print(f" corroborated by: {', '.join(c['corroborated_by'])}")
if not out["candidates"]:
print("no candidates — nothing in the cascade matches; consider a wiki/query-gaps.md entry")
if __name__ == "__main__":
main()