- F10 bm25 section search: stdlib SQLite FTS5 over page sections, with title/heading path/tldr in a separate context column; cached in tmp/ckb-fts.sqlite, refreshed by mtime then sha256; section anchors per candidate. New benchmark tests/bench_retrieval.py over tests/fixtures/bench-kb (20 questions): hit@1 14/20, hit@3 20/20 with and without bm25; weight kept at 1.0 (D-0015) - F12 token estimate per candidate; layered reading in ckb-retrieve - F16 --opened access log in tmp/access.log; lint_report.py --cold proposes demotion to an index `## Archive` section, never moves - F17 ckb-retrieve offers a table/chart/deck/HTML once when it fits Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
526 lines
22 KiB
Python
Executable file
526 lines
22 KiB
Python
Executable file
#!/usr/bin/env python3
|
||
"""Mechanical half of ckb-retrieve: gather ranked candidate lists from several
|
||
local signals and fuse them with reciprocal rank fusion.
|
||
|
||
python3 retrieve_candidates.py "question text" [--root DIR] [--k 10]
|
||
[--top 12] [--scope SLUG] [--tag TAG ...] [--no-bm25] [--json]
|
||
python3 retrieve_candidates.py --opened PATH [PATH ...] # access log: pages actually read
|
||
|
||
Signals, each its own ranked list (weight in brackets):
|
||
exact rg/grep hits on literal tokens, across wiki/, raw/archive/, raw/inbox/,
|
||
outputs/, linked/*/ and connector-backed libs/*/ [1.0, or 2.0
|
||
when the question carries literal-looking tokens]
|
||
tldr title + tldr token overlap for every frontmatter page [1.0]
|
||
scope pages listed by a matching wiki/projects/ scope page [1.0]
|
||
graph one-hop neighbours of the text-matched shortlist [0.7]
|
||
indegree how many edges point at a shortlisted page [0.5]
|
||
trust confidence x quality x freshness, over the shortlist [0.3]
|
||
bm25 SQLite FTS5 over page sections, each chunk prefixed with
|
||
"title › heading path — tldr" (contextual retrieval); a page
|
||
ranks by its best section, and the hit sections are returned [1.0]
|
||
|
||
The bm25 index lives in tmp/ckb-fts.sqlite (in memory when tmp/ is absent)
|
||
and refreshes incrementally by mtime, then sha256. Without FTS5 in this
|
||
Python's sqlite3 the list is skipped and the header says so.
|
||
|
||
--tag keeps only wiki pages carrying every named tag (schema 1.6 `tags:`).
|
||
|
||
Fused score = sum(weight / (k + rank)) over the lists a candidate appears in.
|
||
`k` defaults to 10: a local wiki yields a dozen candidates, not hundreds, and
|
||
k=60 flattens them into near-identical scores.
|
||
|
||
The script ranks; it does not judge. Dedupe-by-claim, rerank against the literal
|
||
question, and source verification stay with the skill. It never writes.
|
||
"""
|
||
import argparse
|
||
import hashlib
|
||
import json
|
||
import os
|
||
import re
|
||
import shutil
|
||
import sqlite3
|
||
import subprocess
|
||
import sys
|
||
from collections import Counter, defaultdict
|
||
from datetime import date, datetime
|
||
from pathlib import Path
|
||
|
||
STOP = set("""a an and are as at be by for from how in is it of on or that the this to
|
||
was we what when where which who why with do does did our your their about into
|
||
""".split())
|
||
MD_EXCLUDE = {"index.md", "log.md"}
|
||
|
||
|
||
# ---------------------------------------------------------------- parsing ---
|
||
def parse_frontmatter(text):
|
||
if not text.startswith("---\n"):
|
||
return {}, text
|
||
end = text.find("\n---", 4)
|
||
if end < 0:
|
||
return {}, text
|
||
fm = {}
|
||
for line in text[4:end].splitlines():
|
||
m = re.match(r"^([A-Za-z_][\w-]*):\s*(.*)$", line)
|
||
if m:
|
||
fm[m.group(1)] = m.group(2).strip().strip('"').strip("'")
|
||
return fm, text[end + 4:]
|
||
|
||
|
||
def title_of(body, fallback):
|
||
m = re.search(r"^#\s+(.+)$", body, re.M)
|
||
return m.group(1).strip() if m else fallback
|
||
|
||
|
||
def tokens(s):
|
||
return [t for t in re.findall(r"[a-z0-9][a-z0-9_./:-]{1,}", s.lower()) if t not in STOP and len(t) > 1]
|
||
|
||
|
||
def literal_tokens(q):
|
||
"""Things that should match exactly: quoted phrases, paths, flags, ids, error-ish strings."""
|
||
lits = re.findall(r'"([^"]{3,})"|\'([^\']{3,})\'|`([^`]{2,})`', q)
|
||
out = [next(x for x in t if x) for t in lits]
|
||
for t in re.findall(r"\S+", q):
|
||
if re.search(r"[/\\.\-_]\w|^--?\w|\d{2,}|[A-Z]{2,}-\d+|^[A-Z][A-Z0-9_]{3,}$", t) and len(t) > 2:
|
||
out.append(t.strip(",.;:?!()"))
|
||
return list(dict.fromkeys(out))
|
||
|
||
|
||
def is_fresh(fm, today):
|
||
lu, win = fm.get("last_updated"), fm.get("freshness_window_days")
|
||
if not lu or not win:
|
||
return None
|
||
try:
|
||
return (today - datetime.strptime(lu, "%Y-%m-%d").date()).days <= int(win)
|
||
except ValueError:
|
||
return None
|
||
|
||
|
||
# ---------------------------------------------------------------- corpus ----
|
||
def page_roots(root):
|
||
yield root / "wiki"
|
||
linked = root / "linked"
|
||
if linked.is_dir():
|
||
for d in sorted(linked.iterdir()):
|
||
if d.is_dir():
|
||
yield d
|
||
libs = root / "libs"
|
||
if libs.is_dir():
|
||
for d in sorted(libs.iterdir()):
|
||
if d.is_dir() and (d / "source.yaml").exists():
|
||
yield d # connector index only; a git-copy lib is not indexed here
|
||
|
||
|
||
def load_pages(root):
|
||
pages = {}
|
||
for base in page_roots(root):
|
||
for p in sorted(base.rglob("*.md")):
|
||
if p.name in MD_EXCLUDE or "/.git/" in p.as_posix():
|
||
continue
|
||
try:
|
||
text = p.read_text(encoding="utf-8", errors="replace")
|
||
except OSError:
|
||
continue
|
||
fm, body = parse_frontmatter(text)
|
||
if not fm.get("type") and base.name == "wiki":
|
||
continue
|
||
rel = p.relative_to(root).as_posix()
|
||
pages[rel] = {
|
||
"path": rel, "fm": fm, "title": title_of(body, p.stem),
|
||
"sources": re.findall(r"^\s*-\s+`([^`]+)`", body.split("## Sources", 1)[1].split("\n## ", 1)[0], re.M)
|
||
if "## Sources" in body else [],
|
||
}
|
||
return pages
|
||
|
||
|
||
def load_edges(root):
|
||
edges = []
|
||
for f in [root / "wiki/graph/edges.json", *root.glob("libs/*/graph/edges.json")]:
|
||
if f.exists():
|
||
try:
|
||
edges += json.loads(f.read_text(encoding="utf-8")).get("edges", [])
|
||
except (OSError, ValueError):
|
||
pass
|
||
return edges
|
||
|
||
|
||
def norm_ref(ref):
|
||
return ref.lstrip("/").removeprefix("wiki/").removesuffix(".md")
|
||
|
||
|
||
# ---------------------------------------------------------------- signals ---
|
||
def signal_exact(root, toks, lits):
|
||
"""Ranked by (distinct tokens matched, total hits)."""
|
||
targets = [d for d in ["wiki", "raw/archive", "raw/inbox", "outputs", "linked"] if (root / d).exists()]
|
||
targets += [d.relative_to(root).as_posix() for d in (root / "libs").glob("*") if (d / "source.yaml").exists()]
|
||
if not targets:
|
||
return [], set()
|
||
needles = list(dict.fromkeys(lits + [t for t in toks if len(t) >= 4]))[:12]
|
||
per_file = defaultdict(lambda: [set(), 0])
|
||
rg = shutil.which("rg")
|
||
for n in needles:
|
||
if rg:
|
||
cmd = [rg, "-c", "-i", "-F", "--follow", "--no-messages", "-g", "!*.json", "-g", "!.git", "--", n, *targets]
|
||
else:
|
||
cmd = ["grep", "-r", "-c", "-i", "-F", "--exclude=*.json", "--exclude-dir=.git", "--", n, *targets]
|
||
try:
|
||
out = subprocess.run(cmd, cwd=root, capture_output=True, text=True, timeout=30).stdout
|
||
except (OSError, subprocess.TimeoutExpired):
|
||
continue
|
||
for line in out.splitlines():
|
||
path, _, cnt = line.rpartition(":")
|
||
if not path or not cnt.isdigit() or int(cnt) == 0:
|
||
continue
|
||
per_file[path][0].add(n)
|
||
per_file[path][1] += int(cnt)
|
||
ranked = sorted(per_file.items(), key=lambda kv: (-len(kv[1][0]), -kv[1][1], kv[0]))
|
||
inbox = {p for p in per_file if p.startswith("raw/inbox/")}
|
||
return [p for p, _ in ranked], inbox
|
||
|
||
|
||
def signal_tldr(pages, toks):
|
||
qs = set(toks)
|
||
scored = []
|
||
for rel, pg in pages.items():
|
||
hay = set(tokens(pg["title"] + " " + pg["fm"].get("tldr", "")))
|
||
hit = len(qs & hay)
|
||
if hit:
|
||
scored.append((-(hit / max(len(qs), 1)), -hit, rel))
|
||
return [r for _, _, r in sorted(scored)]
|
||
|
||
|
||
def signal_scope(root, pages, toks, forced):
|
||
proj = root / "wiki/projects"
|
||
if not proj.is_dir():
|
||
return [], None
|
||
qs = set(toks)
|
||
best, best_hit = None, 0
|
||
for p in proj.glob("*.md"):
|
||
if p.name == "index.md":
|
||
continue
|
||
if forced and p.stem == forced:
|
||
best = p
|
||
break
|
||
pg = pages.get(p.relative_to(root).as_posix())
|
||
hit = len(qs & set(tokens((pg["title"] if pg else "") + " " + (pg["fm"].get("tldr", "") if pg else "") + " " + p.stem)))
|
||
if hit > best_hit:
|
||
best, best_hit = p, hit
|
||
if not best:
|
||
return [], None
|
||
body = best.read_text(encoding="utf-8", errors="replace")
|
||
refs = re.findall(r"\]\(([^)]+\.md)\)|\[\[([^\]]+)\]\]|`((?:wiki|raw|libs|outputs)/[^`]+)`", body)
|
||
listed = []
|
||
for a, b, c in refs:
|
||
r = norm_ref(a or b or c)
|
||
cand = next((rel for rel in pages if norm_ref(rel) == r or rel.endswith(r + ".md")), None)
|
||
if cand and cand not in listed:
|
||
listed.append(cand)
|
||
return listed, best.relative_to(root).as_posix()
|
||
|
||
|
||
def signal_graph(edges, pages, shortlist):
|
||
short = {norm_ref(s) for s in shortlist}
|
||
neigh = Counter()
|
||
for e in edges:
|
||
a, b = norm_ref(str(e.get("from", ""))), norm_ref(str(e.get("to", "")))
|
||
if a in short and b not in short:
|
||
neigh[b] += 1
|
||
if b in short and a not in short:
|
||
neigh[a] += 1
|
||
out = []
|
||
for n, _ in neigh.most_common():
|
||
cand = next((rel for rel in pages if norm_ref(rel) == n or rel.endswith("/" + n + ".md")), None)
|
||
if cand:
|
||
out.append(cand)
|
||
return out
|
||
|
||
|
||
def signal_indegree(edges, shortlist):
|
||
indeg = Counter(norm_ref(str(e.get("to", ""))) for e in edges)
|
||
scored = [(indeg[norm_ref(s)], s) for s in shortlist if indeg[norm_ref(s)] > 0]
|
||
return [s for _, s in sorted(scored, key=lambda x: (-x[0], x[1]))]
|
||
|
||
|
||
def signal_trust(pages, shortlist, today):
|
||
scored = []
|
||
for s in shortlist:
|
||
pg = pages.get(s)
|
||
if not pg:
|
||
continue
|
||
fm = pg["fm"]
|
||
try:
|
||
c, q = float(fm.get("confidence", 0.5)), float(fm.get("quality", 0.5))
|
||
except ValueError:
|
||
c, q = 0.5, 0.5
|
||
fresh = is_fresh(fm, today)
|
||
f = 1.0 if fresh in (True, None) else 0.5
|
||
scored.append((-(c * q * f), s))
|
||
return [s for _, s in sorted(scored)]
|
||
|
||
|
||
# ---------------------------------------------------------------- sections --
|
||
FTS_DB = Path("tmp") / "ckb-fts.sqlite"
|
||
FTS_SCHEMA = "3"
|
||
HEADING_RE = re.compile(r"^(#{1,4})[ \t]+(.+?)[ \t]*#*[ \t]*$", re.M)
|
||
ACCESS_LOG = Path("tmp") / "access.log"
|
||
|
||
|
||
def anchor(heading):
|
||
"""GitHub-style heading anchor."""
|
||
return re.sub(r"\s+", "-", re.sub(r"[^\w\s-]", "", heading.strip().lower()))
|
||
|
||
|
||
def chunks(title, tldr, body):
|
||
"""[(anchor, heading path, context, text)]: one chunk per heading section,
|
||
the text before the first `##` as the lead. The context column carries the
|
||
page title, heading path and tldr, so a section still says what it is about
|
||
alone; it counts for ranking pages, not for naming the matching section."""
|
||
out, stack, start, current = [], [], 0, ("", [])
|
||
marks = [m for m in HEADING_RE.finditer(body) if len(m.group(1)) >= 2]
|
||
for i, m in enumerate(marks + [None]):
|
||
end = m.start() if m else len(body)
|
||
text = body[start:end].strip()
|
||
if text:
|
||
path = current[1]
|
||
prefix = f"{title} › {' › '.join(path)} — {tldr}" if path else f"{title} — {tldr}"
|
||
out.append((current[0], " › ".join(path), prefix, text))
|
||
if m is None:
|
||
break
|
||
level, heading = len(m.group(1)), m.group(2).strip()
|
||
stack = [h for h in stack if h[0] < level] + [(level, heading)]
|
||
current, start = (anchor(heading), [h for _, h in stack]), m.end()
|
||
return out
|
||
|
||
|
||
def open_index(root):
|
||
"""A connection with the FTS5 schema in place, or None when FTS5 is missing."""
|
||
path = root / FTS_DB if (root / "tmp").is_dir() else None
|
||
for attempt in (1, 2):
|
||
try:
|
||
db = sqlite3.connect(str(path) if path else ":memory:")
|
||
db.execute("CREATE TABLE IF NOT EXISTS meta(key TEXT PRIMARY KEY, value TEXT)")
|
||
row = db.execute("SELECT value FROM meta WHERE key='schema'").fetchone()
|
||
if row is None or row[0] != FTS_SCHEMA:
|
||
db.executescript("DROP TABLE IF EXISTS chunks; DROP TABLE IF EXISTS files;")
|
||
db.execute("INSERT OR REPLACE INTO meta VALUES('schema', ?)", (FTS_SCHEMA,))
|
||
db.execute("CREATE VIRTUAL TABLE IF NOT EXISTS chunks USING "
|
||
"fts5(path UNINDEXED, anchor UNINDEXED, heading UNINDEXED, context, text, tokenize='porter unicode61')")
|
||
db.execute("CREATE TABLE IF NOT EXISTS files(path TEXT PRIMARY KEY, mtime REAL, sha TEXT)")
|
||
return db
|
||
except sqlite3.OperationalError as e:
|
||
if "fts5" in str(e).lower():
|
||
return None
|
||
raise
|
||
except sqlite3.DatabaseError:
|
||
if path and attempt == 1 and path.exists():
|
||
path.unlink() # a corrupt cache is rebuilt, never trusted
|
||
continue
|
||
raise
|
||
return None
|
||
|
||
|
||
def refresh_index(db, root, pages):
|
||
"""Re-chunk only files whose mtime moved and whose bytes actually changed."""
|
||
known = {p: (m, h) for p, m, h in db.execute("SELECT path, mtime, sha FROM files")}
|
||
for rel, pg in pages.items():
|
||
f = root / rel
|
||
try:
|
||
mtime = f.stat().st_mtime
|
||
except OSError:
|
||
continue
|
||
old = known.pop(rel, None)
|
||
if old and old[0] == mtime:
|
||
continue
|
||
data = f.read_bytes()
|
||
sha = hashlib.sha256(data).hexdigest()
|
||
if old and old[1] == sha:
|
||
db.execute("UPDATE files SET mtime=? WHERE path=?", (mtime, rel))
|
||
continue
|
||
db.execute("DELETE FROM chunks WHERE path=?", (rel,))
|
||
body = parse_frontmatter(data.decode("utf-8", "replace"))[1]
|
||
db.executemany("INSERT INTO chunks(path, anchor, heading, context, text) VALUES(?,?,?,?,?)",
|
||
[(rel, a, h, c, t) for a, h, c, t in chunks(pg["title"], pg["fm"].get("tldr", ""), body)])
|
||
db.execute("INSERT OR REPLACE INTO files VALUES(?,?,?)", (rel, mtime, sha))
|
||
for gone in known:
|
||
db.execute("DELETE FROM chunks WHERE path=?", (gone,))
|
||
db.execute("DELETE FROM files WHERE path=?", (gone,))
|
||
db.commit()
|
||
|
||
|
||
def signal_bm25(root, pages, toks, enabled=True):
|
||
"""(pages ranked by their best section, {page: [(anchor, heading)]}, status)."""
|
||
if not enabled:
|
||
return [], {}, "off"
|
||
db = open_index(root)
|
||
if db is None:
|
||
return [], {}, "unavailable (no FTS5 in this Python's sqlite3)"
|
||
try:
|
||
refresh_index(db, root, pages)
|
||
terms = list(dict.fromkeys(t for t in toks if len(t) > 1))[:16]
|
||
if not terms:
|
||
return [], {}, "ok"
|
||
query = " OR ".join('"' + t.replace('"', '""') + '"' for t in terms)
|
||
ranked = []
|
||
for (path,) in db.execute("SELECT path FROM chunks WHERE chunks MATCH ? ORDER BY bm25(chunks) LIMIT 200", (query,)):
|
||
if path not in ranked:
|
||
ranked.append(path)
|
||
# Name a section only when its own text matches, not merely its page's context.
|
||
sections = defaultdict(list)
|
||
for path, anc, heading in db.execute(
|
||
"SELECT path, anchor, heading FROM chunks WHERE chunks MATCH ? ORDER BY bm25(chunks) LIMIT 200",
|
||
(f"text : ({query})",)):
|
||
if heading and heading not in ("Sources", "Notes") and len(sections[path]) < 3:
|
||
sections[path].append((anc, heading))
|
||
return ranked, dict(sections), "ok"
|
||
finally:
|
||
db.close()
|
||
|
||
|
||
def log_opened(root, paths, today):
|
||
"""Append pages a person or agent actually read to tmp/access.log (per machine, untracked)."""
|
||
if not (root / "tmp").is_dir():
|
||
sys.exit("error: no tmp/ folder here — the access log lives in tmp/access.log")
|
||
with (root / ACCESS_LOG).open("a", encoding="utf-8") as fh:
|
||
for p in paths:
|
||
fh.write(f"{today.isoformat()}\t{p.lstrip('/')}\n")
|
||
print(f"logged {len(paths)} page(s) in {ACCESS_LOG.as_posix()}")
|
||
|
||
|
||
def est_tokens(root, rel):
|
||
try:
|
||
return -(-(root / rel).stat().st_size // 4)
|
||
except OSError:
|
||
return None
|
||
|
||
|
||
# ---------------------------------------------------------------- fusion ----
|
||
def rrf(lists, k):
|
||
score, ranks = defaultdict(float), defaultdict(dict)
|
||
for name, (items, w) in lists.items():
|
||
for i, item in enumerate(items, 1):
|
||
score[item] += w / (k + i)
|
||
ranks[item][name] = i
|
||
return sorted(score.items(), key=lambda kv: (-kv[1], kv[0])), ranks
|
||
|
||
|
||
def main():
|
||
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||
ap.add_argument("query", nargs="?")
|
||
ap.add_argument("--root", default=".")
|
||
ap.add_argument("--k", type=int, default=10)
|
||
ap.add_argument("--top", type=int, default=12)
|
||
ap.add_argument("--scope", help="force a wiki/projects/<slug> scope instead of matching one")
|
||
ap.add_argument("--today", help="YYYY-MM-DD, for reproducible freshness")
|
||
ap.add_argument("--tag", action="append", default=[], help="keep only pages with this tag (repeatable)")
|
||
ap.add_argument("--no-bm25", action="store_true", help="skip the section index (benchmarks)")
|
||
ap.add_argument("--opened", nargs="+", metavar="PATH", help="log pages actually read, then exit")
|
||
ap.add_argument("--json", action="store_true")
|
||
a = ap.parse_args()
|
||
|
||
root = Path(a.root).resolve()
|
||
today = datetime.strptime(a.today, "%Y-%m-%d").date() if a.today else date.today()
|
||
if a.opened:
|
||
log_opened(root, a.opened, today)
|
||
return
|
||
if not a.query:
|
||
ap.error("a question is required (or --opened PATH ...)")
|
||
toks, lits = tokens(a.query), literal_tokens(a.query)
|
||
pages, edges = load_pages(root), load_edges(root)
|
||
|
||
exact, inbox_hits = signal_exact(root, toks, lits)
|
||
tldr = signal_tldr(pages, toks)
|
||
scope, scope_page = signal_scope(root, pages, toks, a.scope)
|
||
bm25, sections, bm25_status = signal_bm25(root, pages, toks, not a.no_bm25)
|
||
shortlist = list(dict.fromkeys(tldr[:a.top] + [e for e in exact[:a.top] if e in pages] + scope + bm25[:a.top]))
|
||
graph = signal_graph(edges, pages, shortlist)
|
||
indeg = signal_indegree(edges, shortlist)
|
||
trust = signal_trust(pages, shortlist, today)
|
||
|
||
lists = {
|
||
"exact": (exact, 2.0 if lits else 1.0),
|
||
"tldr": (tldr, 1.0),
|
||
"scope": (scope, 1.0),
|
||
"graph": (graph, 0.7),
|
||
"indegree": (indeg, 0.5),
|
||
"trust": (trust, 0.3),
|
||
"bm25": (bm25, 1.0),
|
||
}
|
||
fused, ranks = rrf({n: v for n, v in lists.items() if v[0]}, a.k)
|
||
|
||
# Fold a cited raw/archive or outputs/ hit into the wiki page that cites it:
|
||
# same claim, one candidate, corroboration recorded rather than double-counted.
|
||
# A source cited by N pages corroborates all N, not just the first one walked.
|
||
cited_by = defaultdict(list)
|
||
for rel, pg in sorted(pages.items()):
|
||
for src in pg["sources"]:
|
||
cited_by[src.lstrip("/")].append(rel)
|
||
results, absorbed = [], defaultdict(list)
|
||
ranked = dict(fused)
|
||
for item, sc in fused:
|
||
owners = [o for o in cited_by.get(item, []) if o in ranked]
|
||
if owners:
|
||
for o in owners:
|
||
absorbed[o].append(item)
|
||
continue
|
||
results.append((item, sc))
|
||
if a.tag:
|
||
want = {t.strip().lower() for t in a.tag}
|
||
tagged = lambda pg: {t.strip().lower() for t in pg["fm"].get("tags", "").strip("[]").split(",") if t.strip()}
|
||
results = [(i, sc) for i, sc in results if i in pages and want <= tagged(pages[i])]
|
||
results = results[:a.top]
|
||
|
||
out = {
|
||
"query": a.query, "k": a.k, "literal_tokens": lits, "scope_page": scope_page, "tags": a.tag,
|
||
"inbox_hits": sorted(inbox_hits), "empty_graph": not edges, "bm25": bm25_status,
|
||
"candidates": [],
|
||
}
|
||
for item, sc in results:
|
||
pg = pages.get(item)
|
||
fm = pg["fm"] if pg else {}
|
||
out["candidates"].append({
|
||
"path": item, "score": round(sc, 4), "ranks": ranks[item],
|
||
"title": pg["title"] if pg else None, "type": fm.get("type"), "tldr": fm.get("tldr"),
|
||
"last_updated": fm.get("last_updated"), "stale": (is_fresh(fm, today) is False) if pg else None,
|
||
"confidence": fm.get("confidence"), "quality": fm.get("quality"),
|
||
"superseded_by": fm.get("superseded_by"), "review": fm.get("review"),
|
||
"sources": pg["sources"] if pg else [], "corroborated_by": absorbed.get(item, []),
|
||
"from_inbox": item.startswith("raw/inbox/"),
|
||
"sections": [{"anchor": an, "heading": h} for an, h in sections.get(item, [])],
|
||
"tokens": est_tokens(root, item),
|
||
})
|
||
|
||
if a.json:
|
||
json.dump(out, sys.stdout, indent=2, ensure_ascii=False)
|
||
print()
|
||
return
|
||
print(f"query: {a.query}")
|
||
if lits:
|
||
print(f"literal tokens (exact list weighted 2.0): {', '.join(lits)}")
|
||
if scope_page:
|
||
print(f"scope: {scope_page}")
|
||
if a.tag:
|
||
print(f"tags: only pages tagged {', '.join(a.tag)}")
|
||
if out["empty_graph"]:
|
||
print("graph: empty — graph/indegree lists skipped")
|
||
if bm25_status not in ("ok", "off"):
|
||
print(f"bm25: {bm25_status} — section search skipped")
|
||
if inbox_hits:
|
||
print(f"raw/inbox/ hits ({len(inbox_hits)}): un-ingested material — an ingest is overdue")
|
||
print()
|
||
for i, c in enumerate(out["candidates"], 1):
|
||
flags = [f for f, on in [("STALE", c["stale"]), ("INBOX", c["from_inbox"]), ("SUPERSEDED", c["superseded_by"]),
|
||
("DRAFT", c["review"] == "draft"), ("NEEDS_UPDATE", c["review"] == "needs_update")] if on]
|
||
rk = " ".join(f"{n}:{r}" for n, r in sorted(c["ranks"].items()))
|
||
cost = f" ~{c['tokens']} tok" if c["tokens"] else ""
|
||
print(f"{i:2}. {c['score']:.4f} {c['path']} [{rk}]{cost}" + (f" {' '.join(flags)}" if flags else ""))
|
||
if c["tldr"]:
|
||
print(f" {c['tldr']}")
|
||
if c["sections"]:
|
||
print(" sections: " + ", ".join(f"#{s['anchor']} ({s['heading']})" for s in c["sections"]))
|
||
if c["corroborated_by"]:
|
||
print(f" corroborated by: {', '.join(c['corroborated_by'])}")
|
||
if not out["candidates"]:
|
||
print("no candidates — nothing in the cascade matches; consider a wiki/query-gaps.md entry")
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|