Four deterministic-script defects plus a contract gap, all reported by a downstream KB running 1.11.0. None of them crashed; each produced confident, wrong output, which is why they went unnoticed. retrieve_candidates.py: - load_pages() walked rglob unsorted, so tests/expected/retrieve.json encoded one machine's directory order and the smoke test failed on a clean clone. - cited_by used setdefault, crediting a shared source to whichever page was walked first and dropping the source from the results entirely, so the other citing pages reported corroborated_by: []. Now every citing page is credited. lint_report.py: - check 12 reported a cited directory as "does not exist", because is_file() is false for a directory. Citing a subtree is now explicitly legal and carries no fingerprint; same fix in lint_quick.sh and lint_quick.ps1. - check 8 keyed log entries on timestamp + file with the action type dropped, so a Rule B POINTER was indistinguishable from a duplicate of the entry it points at. POINTER is now exempt; genuine duplicates still fire. - check 13 required a backticked path, so "— Chat conversation, Name, date" counted as unattributed although AGENTS.md lists it as a first-class source. Attribution and verifiability are now separate questions. AGENTS.md documents both resolved contract gaps: directory citations, and the two legal attribution shapes. The fixture KB gained coverage for all three lint defects — each finding fires under the old script and is silent under the new. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
355 lines
14 KiB
Python
Executable file
355 lines
14 KiB
Python
Executable file
#!/usr/bin/env python3
|
|
"""Mechanical half of ckb-retrieve: gather ranked candidate lists from several
|
|
local signals and fuse them with reciprocal rank fusion.
|
|
|
|
python3 retrieve_candidates.py "question text" [--root DIR] [--k 10]
|
|
[--top 12] [--scope SLUG] [--json]
|
|
|
|
Signals, each its own ranked list (weight in brackets):
|
|
exact rg/grep hits on literal tokens, across wiki/, raw/archive/, raw/inbox/,
|
|
outputs/, linked/*/ and connector-backed libs/*/ [1.0, or 2.0
|
|
when the question carries literal-looking tokens]
|
|
tldr title + tldr token overlap for every frontmatter page [1.0]
|
|
scope pages listed by a matching wiki/projects/ scope page [1.0]
|
|
graph one-hop neighbours of the text-matched shortlist [0.7]
|
|
indegree how many edges point at a shortlisted page [0.5]
|
|
trust confidence x quality x freshness, over the shortlist [0.3]
|
|
|
|
Fused score = sum(weight / (k + rank)) over the lists a candidate appears in.
|
|
`k` defaults to 10: a local wiki yields a dozen candidates, not hundreds, and
|
|
k=60 flattens them into near-identical scores.
|
|
|
|
The script ranks; it does not judge. Dedupe-by-claim, rerank against the literal
|
|
question, and source verification stay with the skill. It never writes.
|
|
"""
|
|
import argparse
|
|
import json
|
|
import os
|
|
import re
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
from collections import Counter, defaultdict
|
|
from datetime import date, datetime
|
|
from pathlib import Path
|
|
|
|
STOP = set("""a an and are as at be by for from how in is it of on or that the this to
|
|
was we what when where which who why with do does did our your their about into
|
|
""".split())
|
|
MD_EXCLUDE = {"index.md", "log.md"}
|
|
|
|
|
|
# ---------------------------------------------------------------- parsing ---
|
|
def parse_frontmatter(text):
|
|
if not text.startswith("---\n"):
|
|
return {}, text
|
|
end = text.find("\n---", 4)
|
|
if end < 0:
|
|
return {}, text
|
|
fm = {}
|
|
for line in text[4:end].splitlines():
|
|
m = re.match(r"^([A-Za-z_][\w-]*):\s*(.*)$", line)
|
|
if m:
|
|
fm[m.group(1)] = m.group(2).strip().strip('"').strip("'")
|
|
return fm, text[end + 4:]
|
|
|
|
|
|
def title_of(body, fallback):
|
|
m = re.search(r"^#\s+(.+)$", body, re.M)
|
|
return m.group(1).strip() if m else fallback
|
|
|
|
|
|
def tokens(s):
|
|
return [t for t in re.findall(r"[a-z0-9][a-z0-9_./:-]{1,}", s.lower()) if t not in STOP and len(t) > 1]
|
|
|
|
|
|
def literal_tokens(q):
|
|
"""Things that should match exactly: quoted phrases, paths, flags, ids, error-ish strings."""
|
|
lits = re.findall(r'"([^"]{3,})"|\'([^\']{3,})\'|`([^`]{2,})`', q)
|
|
out = [next(x for x in t if x) for t in lits]
|
|
for t in re.findall(r"\S+", q):
|
|
if re.search(r"[/\\.\-_]\w|^--?\w|\d{2,}|[A-Z]{2,}-\d+|^[A-Z][A-Z0-9_]{3,}$", t) and len(t) > 2:
|
|
out.append(t.strip(",.;:?!()"))
|
|
return list(dict.fromkeys(out))
|
|
|
|
|
|
def is_fresh(fm, today):
|
|
lu, win = fm.get("last_updated"), fm.get("freshness_window_days")
|
|
if not lu or not win:
|
|
return None
|
|
try:
|
|
return (today - datetime.strptime(lu, "%Y-%m-%d").date()).days <= int(win)
|
|
except ValueError:
|
|
return None
|
|
|
|
|
|
# ---------------------------------------------------------------- corpus ----
|
|
def page_roots(root):
|
|
yield root / "wiki"
|
|
linked = root / "linked"
|
|
if linked.is_dir():
|
|
for d in sorted(linked.iterdir()):
|
|
if d.is_dir():
|
|
yield d
|
|
libs = root / "libs"
|
|
if libs.is_dir():
|
|
for d in sorted(libs.iterdir()):
|
|
if d.is_dir() and (d / "source.yaml").exists():
|
|
yield d # connector index only; a git-copy lib is not indexed here
|
|
|
|
|
|
def load_pages(root):
|
|
pages = {}
|
|
for base in page_roots(root):
|
|
for p in sorted(base.rglob("*.md")):
|
|
if p.name in MD_EXCLUDE or "/.git/" in p.as_posix():
|
|
continue
|
|
try:
|
|
text = p.read_text(encoding="utf-8", errors="replace")
|
|
except OSError:
|
|
continue
|
|
fm, body = parse_frontmatter(text)
|
|
if not fm.get("type") and base.name == "wiki":
|
|
continue
|
|
rel = p.relative_to(root).as_posix()
|
|
pages[rel] = {
|
|
"path": rel, "fm": fm, "title": title_of(body, p.stem),
|
|
"sources": re.findall(r"^\s*-\s+`([^`]+)`", body.split("## Sources", 1)[1].split("\n## ", 1)[0], re.M)
|
|
if "## Sources" in body else [],
|
|
}
|
|
return pages
|
|
|
|
|
|
def load_edges(root):
|
|
edges = []
|
|
for f in [root / "wiki/graph/edges.json", *root.glob("libs/*/graph/edges.json")]:
|
|
if f.exists():
|
|
try:
|
|
edges += json.loads(f.read_text(encoding="utf-8")).get("edges", [])
|
|
except (OSError, ValueError):
|
|
pass
|
|
return edges
|
|
|
|
|
|
def norm_ref(ref):
|
|
return ref.lstrip("/").removeprefix("wiki/").removesuffix(".md")
|
|
|
|
|
|
# ---------------------------------------------------------------- signals ---
|
|
def signal_exact(root, toks, lits):
|
|
"""Ranked by (distinct tokens matched, total hits)."""
|
|
targets = [d for d in ["wiki", "raw/archive", "raw/inbox", "outputs", "linked"] if (root / d).exists()]
|
|
targets += [d.relative_to(root).as_posix() for d in (root / "libs").glob("*") if (d / "source.yaml").exists()]
|
|
if not targets:
|
|
return [], set()
|
|
needles = list(dict.fromkeys(lits + [t for t in toks if len(t) >= 4]))[:12]
|
|
per_file = defaultdict(lambda: [set(), 0])
|
|
rg = shutil.which("rg")
|
|
for n in needles:
|
|
if rg:
|
|
cmd = [rg, "-c", "-i", "-F", "--follow", "--no-messages", "-g", "!*.json", "-g", "!.git", "--", n, *targets]
|
|
else:
|
|
cmd = ["grep", "-r", "-c", "-i", "-F", "--exclude=*.json", "--exclude-dir=.git", "--", n, *targets]
|
|
try:
|
|
out = subprocess.run(cmd, cwd=root, capture_output=True, text=True, timeout=30).stdout
|
|
except (OSError, subprocess.TimeoutExpired):
|
|
continue
|
|
for line in out.splitlines():
|
|
path, _, cnt = line.rpartition(":")
|
|
if not path or not cnt.isdigit() or int(cnt) == 0:
|
|
continue
|
|
per_file[path][0].add(n)
|
|
per_file[path][1] += int(cnt)
|
|
ranked = sorted(per_file.items(), key=lambda kv: (-len(kv[1][0]), -kv[1][1], kv[0]))
|
|
inbox = {p for p in per_file if p.startswith("raw/inbox/")}
|
|
return [p for p, _ in ranked], inbox
|
|
|
|
|
|
def signal_tldr(pages, toks):
|
|
qs = set(toks)
|
|
scored = []
|
|
for rel, pg in pages.items():
|
|
hay = set(tokens(pg["title"] + " " + pg["fm"].get("tldr", "")))
|
|
hit = len(qs & hay)
|
|
if hit:
|
|
scored.append((-(hit / max(len(qs), 1)), -hit, rel))
|
|
return [r for _, _, r in sorted(scored)]
|
|
|
|
|
|
def signal_scope(root, pages, toks, forced):
|
|
proj = root / "wiki/projects"
|
|
if not proj.is_dir():
|
|
return [], None
|
|
qs = set(toks)
|
|
best, best_hit = None, 0
|
|
for p in proj.glob("*.md"):
|
|
if p.name == "index.md":
|
|
continue
|
|
if forced and p.stem == forced:
|
|
best = p
|
|
break
|
|
pg = pages.get(p.relative_to(root).as_posix())
|
|
hit = len(qs & set(tokens((pg["title"] if pg else "") + " " + (pg["fm"].get("tldr", "") if pg else "") + " " + p.stem)))
|
|
if hit > best_hit:
|
|
best, best_hit = p, hit
|
|
if not best:
|
|
return [], None
|
|
body = best.read_text(encoding="utf-8", errors="replace")
|
|
refs = re.findall(r"\]\(([^)]+\.md)\)|\[\[([^\]]+)\]\]|`((?:wiki|raw|libs|outputs)/[^`]+)`", body)
|
|
listed = []
|
|
for a, b, c in refs:
|
|
r = norm_ref(a or b or c)
|
|
cand = next((rel for rel in pages if norm_ref(rel) == r or rel.endswith(r + ".md")), None)
|
|
if cand and cand not in listed:
|
|
listed.append(cand)
|
|
return listed, best.relative_to(root).as_posix()
|
|
|
|
|
|
def signal_graph(edges, pages, shortlist):
|
|
short = {norm_ref(s) for s in shortlist}
|
|
neigh = Counter()
|
|
for e in edges:
|
|
a, b = norm_ref(str(e.get("from", ""))), norm_ref(str(e.get("to", "")))
|
|
if a in short and b not in short:
|
|
neigh[b] += 1
|
|
if b in short and a not in short:
|
|
neigh[a] += 1
|
|
out = []
|
|
for n, _ in neigh.most_common():
|
|
cand = next((rel for rel in pages if norm_ref(rel) == n or rel.endswith("/" + n + ".md")), None)
|
|
if cand:
|
|
out.append(cand)
|
|
return out
|
|
|
|
|
|
def signal_indegree(edges, shortlist):
|
|
indeg = Counter(norm_ref(str(e.get("to", ""))) for e in edges)
|
|
scored = [(indeg[norm_ref(s)], s) for s in shortlist if indeg[norm_ref(s)] > 0]
|
|
return [s for _, s in sorted(scored, key=lambda x: (-x[0], x[1]))]
|
|
|
|
|
|
def signal_trust(pages, shortlist, today):
|
|
scored = []
|
|
for s in shortlist:
|
|
pg = pages.get(s)
|
|
if not pg:
|
|
continue
|
|
fm = pg["fm"]
|
|
try:
|
|
c, q = float(fm.get("confidence", 0.5)), float(fm.get("quality", 0.5))
|
|
except ValueError:
|
|
c, q = 0.5, 0.5
|
|
fresh = is_fresh(fm, today)
|
|
f = 1.0 if fresh in (True, None) else 0.5
|
|
scored.append((-(c * q * f), s))
|
|
return [s for _, s in sorted(scored)]
|
|
|
|
|
|
# ---------------------------------------------------------------- fusion ----
|
|
def rrf(lists, k):
|
|
score, ranks = defaultdict(float), defaultdict(dict)
|
|
for name, (items, w) in lists.items():
|
|
for i, item in enumerate(items, 1):
|
|
score[item] += w / (k + i)
|
|
ranks[item][name] = i
|
|
return sorted(score.items(), key=lambda kv: (-kv[1], kv[0])), ranks
|
|
|
|
|
|
def main():
|
|
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
ap.add_argument("query")
|
|
ap.add_argument("--root", default=".")
|
|
ap.add_argument("--k", type=int, default=10)
|
|
ap.add_argument("--top", type=int, default=12)
|
|
ap.add_argument("--scope", help="force a wiki/projects/<slug> scope instead of matching one")
|
|
ap.add_argument("--today", help="YYYY-MM-DD, for reproducible freshness")
|
|
ap.add_argument("--json", action="store_true")
|
|
a = ap.parse_args()
|
|
|
|
root = Path(a.root).resolve()
|
|
today = datetime.strptime(a.today, "%Y-%m-%d").date() if a.today else date.today()
|
|
toks, lits = tokens(a.query), literal_tokens(a.query)
|
|
pages, edges = load_pages(root), load_edges(root)
|
|
|
|
exact, inbox_hits = signal_exact(root, toks, lits)
|
|
tldr = signal_tldr(pages, toks)
|
|
scope, scope_page = signal_scope(root, pages, toks, a.scope)
|
|
shortlist = list(dict.fromkeys(tldr[:a.top] + [e for e in exact[:a.top] if e in pages] + scope))
|
|
graph = signal_graph(edges, pages, shortlist)
|
|
indeg = signal_indegree(edges, shortlist)
|
|
trust = signal_trust(pages, shortlist, today)
|
|
|
|
lists = {
|
|
"exact": (exact, 2.0 if lits else 1.0),
|
|
"tldr": (tldr, 1.0),
|
|
"scope": (scope, 1.0),
|
|
"graph": (graph, 0.7),
|
|
"indegree": (indeg, 0.5),
|
|
"trust": (trust, 0.3),
|
|
}
|
|
fused, ranks = rrf({n: v for n, v in lists.items() if v[0]}, a.k)
|
|
|
|
# Fold a cited raw/archive or outputs/ hit into the wiki page that cites it:
|
|
# same claim, one candidate, corroboration recorded rather than double-counted.
|
|
# A source cited by N pages corroborates all N, not just the first one walked.
|
|
cited_by = defaultdict(list)
|
|
for rel, pg in sorted(pages.items()):
|
|
for src in pg["sources"]:
|
|
cited_by[src.lstrip("/")].append(rel)
|
|
results, absorbed = [], defaultdict(list)
|
|
ranked = dict(fused)
|
|
for item, sc in fused:
|
|
owners = [o for o in cited_by.get(item, []) if o in ranked]
|
|
if owners:
|
|
for o in owners:
|
|
absorbed[o].append(item)
|
|
continue
|
|
results.append((item, sc))
|
|
results = results[:a.top]
|
|
|
|
out = {
|
|
"query": a.query, "k": a.k, "literal_tokens": lits, "scope_page": scope_page,
|
|
"inbox_hits": sorted(inbox_hits), "empty_graph": not edges,
|
|
"candidates": [],
|
|
}
|
|
for item, sc in results:
|
|
pg = pages.get(item)
|
|
fm = pg["fm"] if pg else {}
|
|
out["candidates"].append({
|
|
"path": item, "score": round(sc, 4), "ranks": ranks[item],
|
|
"title": pg["title"] if pg else None, "type": fm.get("type"), "tldr": fm.get("tldr"),
|
|
"last_updated": fm.get("last_updated"), "stale": (is_fresh(fm, today) is False) if pg else None,
|
|
"confidence": fm.get("confidence"), "quality": fm.get("quality"),
|
|
"superseded_by": fm.get("superseded_by"),
|
|
"sources": pg["sources"] if pg else [], "corroborated_by": absorbed.get(item, []),
|
|
"from_inbox": item.startswith("raw/inbox/"),
|
|
})
|
|
|
|
if a.json:
|
|
json.dump(out, sys.stdout, indent=2, ensure_ascii=False)
|
|
print()
|
|
return
|
|
print(f"query: {a.query}")
|
|
if lits:
|
|
print(f"literal tokens (exact list weighted 2.0): {', '.join(lits)}")
|
|
if scope_page:
|
|
print(f"scope: {scope_page}")
|
|
if out["empty_graph"]:
|
|
print("graph: empty — graph/indegree lists skipped")
|
|
if inbox_hits:
|
|
print(f"raw/inbox/ hits ({len(inbox_hits)}): un-ingested material — an ingest is overdue")
|
|
print()
|
|
for i, c in enumerate(out["candidates"], 1):
|
|
flags = [f for f, on in [("STALE", c["stale"]), ("INBOX", c["from_inbox"]), ("SUPERSEDED", c["superseded_by"])] if on]
|
|
rk = " ".join(f"{n}:{r}" for n, r in sorted(c["ranks"].items()))
|
|
print(f"{i:2}. {c['score']:.4f} {c['path']} [{rk}]" + (f" {' '.join(flags)}" if flags else ""))
|
|
if c["tldr"]:
|
|
print(f" {c['tldr']}")
|
|
if c["corroborated_by"]:
|
|
print(f" corroborated by: {', '.join(c['corroborated_by'])}")
|
|
if not out["candidates"]:
|
|
print("no candidates — nothing in the cascade matches; consider a wiki/query-gaps.md entry")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|