ckb/.agents/skills/ckb-retrieve/scripts/retrieve_candidates.py
Michał Kopeć 7b812e545b Template 1.11.1: fix five defects reported downstream
Four deterministic-script defects plus a contract gap, all reported by a
downstream KB running 1.11.0. None of them crashed; each produced confident,
wrong output, which is why they went unnoticed.

retrieve_candidates.py:
- load_pages() walked rglob unsorted, so tests/expected/retrieve.json encoded
  one machine's directory order and the smoke test failed on a clean clone.
- cited_by used setdefault, crediting a shared source to whichever page was
  walked first and dropping the source from the results entirely, so the other
  citing pages reported corroborated_by: []. Now every citing page is credited.

lint_report.py:
- check 12 reported a cited directory as "does not exist", because is_file()
  is false for a directory. Citing a subtree is now explicitly legal and
  carries no fingerprint; same fix in lint_quick.sh and lint_quick.ps1.
- check 8 keyed log entries on timestamp + file with the action type dropped,
  so a Rule B POINTER was indistinguishable from a duplicate of the entry it
  points at. POINTER is now exempt; genuine duplicates still fire.
- check 13 required a backticked path, so "— Chat conversation, Name, date"
  counted as unattributed although AGENTS.md lists it as a first-class source.
  Attribution and verifiability are now separate questions.

AGENTS.md documents both resolved contract gaps: directory citations, and the
two legal attribution shapes. The fixture KB gained coverage for all three lint
defects — each finding fires under the old script and is silent under the new.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-10-09 21:47:53 +02:00

355 lines
14 KiB
Python
Executable file

#!/usr/bin/env python3
"""Mechanical half of ckb-retrieve: gather ranked candidate lists from several
local signals and fuse them with reciprocal rank fusion.
python3 retrieve_candidates.py "question text" [--root DIR] [--k 10]
[--top 12] [--scope SLUG] [--json]
Signals, each its own ranked list (weight in brackets):
exact rg/grep hits on literal tokens, across wiki/, raw/archive/, raw/inbox/,
outputs/, linked/*/ and connector-backed libs/*/ [1.0, or 2.0
when the question carries literal-looking tokens]
tldr title + tldr token overlap for every frontmatter page [1.0]
scope pages listed by a matching wiki/projects/ scope page [1.0]
graph one-hop neighbours of the text-matched shortlist [0.7]
indegree how many edges point at a shortlisted page [0.5]
trust confidence x quality x freshness, over the shortlist [0.3]
Fused score = sum(weight / (k + rank)) over the lists a candidate appears in.
`k` defaults to 10: a local wiki yields a dozen candidates, not hundreds, and
k=60 flattens them into near-identical scores.
The script ranks; it does not judge. Dedupe-by-claim, rerank against the literal
question, and source verification stay with the skill. It never writes.
"""
import argparse
import json
import os
import re
import shutil
import subprocess
import sys
from collections import Counter, defaultdict
from datetime import date, datetime
from pathlib import Path
STOP = set("""a an and are as at be by for from how in is it of on or that the this to
was we what when where which who why with do does did our your their about into
""".split())
MD_EXCLUDE = {"index.md", "log.md"}
# ---------------------------------------------------------------- parsing ---
def parse_frontmatter(text):
if not text.startswith("---\n"):
return {}, text
end = text.find("\n---", 4)
if end < 0:
return {}, text
fm = {}
for line in text[4:end].splitlines():
m = re.match(r"^([A-Za-z_][\w-]*):\s*(.*)$", line)
if m:
fm[m.group(1)] = m.group(2).strip().strip('"').strip("'")
return fm, text[end + 4:]
def title_of(body, fallback):
m = re.search(r"^#\s+(.+)$", body, re.M)
return m.group(1).strip() if m else fallback
def tokens(s):
return [t for t in re.findall(r"[a-z0-9][a-z0-9_./:-]{1,}", s.lower()) if t not in STOP and len(t) > 1]
def literal_tokens(q):
"""Things that should match exactly: quoted phrases, paths, flags, ids, error-ish strings."""
lits = re.findall(r'"([^"]{3,})"|\'([^\']{3,})\'|`([^`]{2,})`', q)
out = [next(x for x in t if x) for t in lits]
for t in re.findall(r"\S+", q):
if re.search(r"[/\\.\-_]\w|^--?\w|\d{2,}|[A-Z]{2,}-\d+|^[A-Z][A-Z0-9_]{3,}$", t) and len(t) > 2:
out.append(t.strip(",.;:?!()"))
return list(dict.fromkeys(out))
def is_fresh(fm, today):
lu, win = fm.get("last_updated"), fm.get("freshness_window_days")
if not lu or not win:
return None
try:
return (today - datetime.strptime(lu, "%Y-%m-%d").date()).days <= int(win)
except ValueError:
return None
# ---------------------------------------------------------------- corpus ----
def page_roots(root):
yield root / "wiki"
linked = root / "linked"
if linked.is_dir():
for d in sorted(linked.iterdir()):
if d.is_dir():
yield d
libs = root / "libs"
if libs.is_dir():
for d in sorted(libs.iterdir()):
if d.is_dir() and (d / "source.yaml").exists():
yield d # connector index only; a git-copy lib is not indexed here
def load_pages(root):
pages = {}
for base in page_roots(root):
for p in sorted(base.rglob("*.md")):
if p.name in MD_EXCLUDE or "/.git/" in p.as_posix():
continue
try:
text = p.read_text(encoding="utf-8", errors="replace")
except OSError:
continue
fm, body = parse_frontmatter(text)
if not fm.get("type") and base.name == "wiki":
continue
rel = p.relative_to(root).as_posix()
pages[rel] = {
"path": rel, "fm": fm, "title": title_of(body, p.stem),
"sources": re.findall(r"^\s*-\s+`([^`]+)`", body.split("## Sources", 1)[1].split("\n## ", 1)[0], re.M)
if "## Sources" in body else [],
}
return pages
def load_edges(root):
edges = []
for f in [root / "wiki/graph/edges.json", *root.glob("libs/*/graph/edges.json")]:
if f.exists():
try:
edges += json.loads(f.read_text(encoding="utf-8")).get("edges", [])
except (OSError, ValueError):
pass
return edges
def norm_ref(ref):
return ref.lstrip("/").removeprefix("wiki/").removesuffix(".md")
# ---------------------------------------------------------------- signals ---
def signal_exact(root, toks, lits):
"""Ranked by (distinct tokens matched, total hits)."""
targets = [d for d in ["wiki", "raw/archive", "raw/inbox", "outputs", "linked"] if (root / d).exists()]
targets += [d.relative_to(root).as_posix() for d in (root / "libs").glob("*") if (d / "source.yaml").exists()]
if not targets:
return [], set()
needles = list(dict.fromkeys(lits + [t for t in toks if len(t) >= 4]))[:12]
per_file = defaultdict(lambda: [set(), 0])
rg = shutil.which("rg")
for n in needles:
if rg:
cmd = [rg, "-c", "-i", "-F", "--follow", "--no-messages", "-g", "!*.json", "-g", "!.git", "--", n, *targets]
else:
cmd = ["grep", "-r", "-c", "-i", "-F", "--exclude=*.json", "--exclude-dir=.git", "--", n, *targets]
try:
out = subprocess.run(cmd, cwd=root, capture_output=True, text=True, timeout=30).stdout
except (OSError, subprocess.TimeoutExpired):
continue
for line in out.splitlines():
path, _, cnt = line.rpartition(":")
if not path or not cnt.isdigit() or int(cnt) == 0:
continue
per_file[path][0].add(n)
per_file[path][1] += int(cnt)
ranked = sorted(per_file.items(), key=lambda kv: (-len(kv[1][0]), -kv[1][1], kv[0]))
inbox = {p for p in per_file if p.startswith("raw/inbox/")}
return [p for p, _ in ranked], inbox
def signal_tldr(pages, toks):
qs = set(toks)
scored = []
for rel, pg in pages.items():
hay = set(tokens(pg["title"] + " " + pg["fm"].get("tldr", "")))
hit = len(qs & hay)
if hit:
scored.append((-(hit / max(len(qs), 1)), -hit, rel))
return [r for _, _, r in sorted(scored)]
def signal_scope(root, pages, toks, forced):
proj = root / "wiki/projects"
if not proj.is_dir():
return [], None
qs = set(toks)
best, best_hit = None, 0
for p in proj.glob("*.md"):
if p.name == "index.md":
continue
if forced and p.stem == forced:
best = p
break
pg = pages.get(p.relative_to(root).as_posix())
hit = len(qs & set(tokens((pg["title"] if pg else "") + " " + (pg["fm"].get("tldr", "") if pg else "") + " " + p.stem)))
if hit > best_hit:
best, best_hit = p, hit
if not best:
return [], None
body = best.read_text(encoding="utf-8", errors="replace")
refs = re.findall(r"\]\(([^)]+\.md)\)|\[\[([^\]]+)\]\]|`((?:wiki|raw|libs|outputs)/[^`]+)`", body)
listed = []
for a, b, c in refs:
r = norm_ref(a or b or c)
cand = next((rel for rel in pages if norm_ref(rel) == r or rel.endswith(r + ".md")), None)
if cand and cand not in listed:
listed.append(cand)
return listed, best.relative_to(root).as_posix()
def signal_graph(edges, pages, shortlist):
short = {norm_ref(s) for s in shortlist}
neigh = Counter()
for e in edges:
a, b = norm_ref(str(e.get("from", ""))), norm_ref(str(e.get("to", "")))
if a in short and b not in short:
neigh[b] += 1
if b in short and a not in short:
neigh[a] += 1
out = []
for n, _ in neigh.most_common():
cand = next((rel for rel in pages if norm_ref(rel) == n or rel.endswith("/" + n + ".md")), None)
if cand:
out.append(cand)
return out
def signal_indegree(edges, shortlist):
indeg = Counter(norm_ref(str(e.get("to", ""))) for e in edges)
scored = [(indeg[norm_ref(s)], s) for s in shortlist if indeg[norm_ref(s)] > 0]
return [s for _, s in sorted(scored, key=lambda x: (-x[0], x[1]))]
def signal_trust(pages, shortlist, today):
scored = []
for s in shortlist:
pg = pages.get(s)
if not pg:
continue
fm = pg["fm"]
try:
c, q = float(fm.get("confidence", 0.5)), float(fm.get("quality", 0.5))
except ValueError:
c, q = 0.5, 0.5
fresh = is_fresh(fm, today)
f = 1.0 if fresh in (True, None) else 0.5
scored.append((-(c * q * f), s))
return [s for _, s in sorted(scored)]
# ---------------------------------------------------------------- fusion ----
def rrf(lists, k):
score, ranks = defaultdict(float), defaultdict(dict)
for name, (items, w) in lists.items():
for i, item in enumerate(items, 1):
score[item] += w / (k + i)
ranks[item][name] = i
return sorted(score.items(), key=lambda kv: (-kv[1], kv[0])), ranks
def main():
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument("query")
ap.add_argument("--root", default=".")
ap.add_argument("--k", type=int, default=10)
ap.add_argument("--top", type=int, default=12)
ap.add_argument("--scope", help="force a wiki/projects/<slug> scope instead of matching one")
ap.add_argument("--today", help="YYYY-MM-DD, for reproducible freshness")
ap.add_argument("--json", action="store_true")
a = ap.parse_args()
root = Path(a.root).resolve()
today = datetime.strptime(a.today, "%Y-%m-%d").date() if a.today else date.today()
toks, lits = tokens(a.query), literal_tokens(a.query)
pages, edges = load_pages(root), load_edges(root)
exact, inbox_hits = signal_exact(root, toks, lits)
tldr = signal_tldr(pages, toks)
scope, scope_page = signal_scope(root, pages, toks, a.scope)
shortlist = list(dict.fromkeys(tldr[:a.top] + [e for e in exact[:a.top] if e in pages] + scope))
graph = signal_graph(edges, pages, shortlist)
indeg = signal_indegree(edges, shortlist)
trust = signal_trust(pages, shortlist, today)
lists = {
"exact": (exact, 2.0 if lits else 1.0),
"tldr": (tldr, 1.0),
"scope": (scope, 1.0),
"graph": (graph, 0.7),
"indegree": (indeg, 0.5),
"trust": (trust, 0.3),
}
fused, ranks = rrf({n: v for n, v in lists.items() if v[0]}, a.k)
# Fold a cited raw/archive or outputs/ hit into the wiki page that cites it:
# same claim, one candidate, corroboration recorded rather than double-counted.
# A source cited by N pages corroborates all N, not just the first one walked.
cited_by = defaultdict(list)
for rel, pg in sorted(pages.items()):
for src in pg["sources"]:
cited_by[src.lstrip("/")].append(rel)
results, absorbed = [], defaultdict(list)
ranked = dict(fused)
for item, sc in fused:
owners = [o for o in cited_by.get(item, []) if o in ranked]
if owners:
for o in owners:
absorbed[o].append(item)
continue
results.append((item, sc))
results = results[:a.top]
out = {
"query": a.query, "k": a.k, "literal_tokens": lits, "scope_page": scope_page,
"inbox_hits": sorted(inbox_hits), "empty_graph": not edges,
"candidates": [],
}
for item, sc in results:
pg = pages.get(item)
fm = pg["fm"] if pg else {}
out["candidates"].append({
"path": item, "score": round(sc, 4), "ranks": ranks[item],
"title": pg["title"] if pg else None, "type": fm.get("type"), "tldr": fm.get("tldr"),
"last_updated": fm.get("last_updated"), "stale": (is_fresh(fm, today) is False) if pg else None,
"confidence": fm.get("confidence"), "quality": fm.get("quality"),
"superseded_by": fm.get("superseded_by"),
"sources": pg["sources"] if pg else [], "corroborated_by": absorbed.get(item, []),
"from_inbox": item.startswith("raw/inbox/"),
})
if a.json:
json.dump(out, sys.stdout, indent=2, ensure_ascii=False)
print()
return
print(f"query: {a.query}")
if lits:
print(f"literal tokens (exact list weighted 2.0): {', '.join(lits)}")
if scope_page:
print(f"scope: {scope_page}")
if out["empty_graph"]:
print("graph: empty — graph/indegree lists skipped")
if inbox_hits:
print(f"raw/inbox/ hits ({len(inbox_hits)}): un-ingested material — an ingest is overdue")
print()
for i, c in enumerate(out["candidates"], 1):
flags = [f for f, on in [("STALE", c["stale"]), ("INBOX", c["from_inbox"]), ("SUPERSEDED", c["superseded_by"])] if on]
rk = " ".join(f"{n}:{r}" for n, r in sorted(c["ranks"].items()))
print(f"{i:2}. {c['score']:.4f} {c['path']} [{rk}]" + (f" {' '.join(flags)}" if flags else ""))
if c["tldr"]:
print(f" {c['tldr']}")
if c["corroborated_by"]:
print(f" corroborated by: {', '.join(c['corroborated_by'])}")
if not out["candidates"]:
print("no candidates — nothing in the cascade matches; consider a wiki/query-gaps.md entry")
if __name__ == "__main__":
main()