#!/usr/bin/env python3 """Mechanical half of ckb-retrieve: gather ranked candidate lists from several local signals and fuse them with reciprocal rank fusion. python3 retrieve_candidates.py "question text" [--root DIR] [--k 10] [--top 12] [--scope SLUG] [--json] Signals, each its own ranked list (weight in brackets): exact rg/grep hits on literal tokens, across wiki/, raw/archive/, raw/inbox/, outputs/, linked/*/ and connector-backed libs/*/ [1.0, or 2.0 when the question carries literal-looking tokens] tldr title + tldr token overlap for every frontmatter page [1.0] scope pages listed by a matching wiki/projects/ scope page [1.0] graph one-hop neighbours of the text-matched shortlist [0.7] indegree how many edges point at a shortlisted page [0.5] trust confidence x quality x freshness, over the shortlist [0.3] Fused score = sum(weight / (k + rank)) over the lists a candidate appears in. `k` defaults to 10: a local wiki yields a dozen candidates, not hundreds, and k=60 flattens them into near-identical scores. The script ranks; it does not judge. Dedupe-by-claim, rerank against the literal question, and source verification stay with the skill. It never writes. """ import argparse import json import os import re import shutil import subprocess import sys from collections import Counter, defaultdict from datetime import date, datetime from pathlib import Path STOP = set("""a an and are as at be by for from how in is it of on or that the this to was we what when where which who why with do does did our your their about into """.split()) MD_EXCLUDE = {"index.md", "log.md"} # ---------------------------------------------------------------- parsing --- def parse_frontmatter(text): if not text.startswith("---\n"): return {}, text end = text.find("\n---", 4) if end < 0: return {}, text fm = {} for line in text[4:end].splitlines(): m = re.match(r"^([A-Za-z_][\w-]*):\s*(.*)$", line) if m: fm[m.group(1)] = m.group(2).strip().strip('"').strip("'") return fm, text[end + 4:] def title_of(body, fallback): m = re.search(r"^#\s+(.+)$", body, re.M) return m.group(1).strip() if m else fallback def tokens(s): return [t for t in re.findall(r"[a-z0-9][a-z0-9_./:-]{1,}", s.lower()) if t not in STOP and len(t) > 1] def literal_tokens(q): """Things that should match exactly: quoted phrases, paths, flags, ids, error-ish strings.""" lits = re.findall(r'"([^"]{3,})"|\'([^\']{3,})\'|`([^`]{2,})`', q) out = [next(x for x in t if x) for t in lits] for t in re.findall(r"\S+", q): if re.search(r"[/\\.\-_]\w|^--?\w|\d{2,}|[A-Z]{2,}-\d+|^[A-Z][A-Z0-9_]{3,}$", t) and len(t) > 2: out.append(t.strip(",.;:?!()")) return list(dict.fromkeys(out)) def is_fresh(fm, today): lu, win = fm.get("last_updated"), fm.get("freshness_window_days") if not lu or not win: return None try: return (today - datetime.strptime(lu, "%Y-%m-%d").date()).days <= int(win) except ValueError: return None # ---------------------------------------------------------------- corpus ---- def page_roots(root): yield root / "wiki" linked = root / "linked" if linked.is_dir(): for d in sorted(linked.iterdir()): if d.is_dir(): yield d libs = root / "libs" if libs.is_dir(): for d in sorted(libs.iterdir()): if d.is_dir() and (d / "source.yaml").exists(): yield d # connector index only; a git-copy lib is not indexed here def load_pages(root): pages = {} for base in page_roots(root): for p in sorted(base.rglob("*.md")): if p.name in MD_EXCLUDE or "/.git/" in p.as_posix(): continue try: text = p.read_text(encoding="utf-8", errors="replace") except OSError: continue fm, body = parse_frontmatter(text) if not fm.get("type") and base.name == "wiki": continue rel = p.relative_to(root).as_posix() pages[rel] = { "path": rel, "fm": fm, "title": title_of(body, p.stem), "sources": re.findall(r"^\s*-\s+`([^`]+)`", body.split("## Sources", 1)[1].split("\n## ", 1)[0], re.M) if "## Sources" in body else [], } return pages def load_edges(root): edges = [] for f in [root / "wiki/graph/edges.json", *root.glob("libs/*/graph/edges.json")]: if f.exists(): try: edges += json.loads(f.read_text(encoding="utf-8")).get("edges", []) except (OSError, ValueError): pass return edges def norm_ref(ref): return ref.lstrip("/").removeprefix("wiki/").removesuffix(".md") # ---------------------------------------------------------------- signals --- def signal_exact(root, toks, lits): """Ranked by (distinct tokens matched, total hits).""" targets = [d for d in ["wiki", "raw/archive", "raw/inbox", "outputs", "linked"] if (root / d).exists()] targets += [d.relative_to(root).as_posix() for d in (root / "libs").glob("*") if (d / "source.yaml").exists()] if not targets: return [], set() needles = list(dict.fromkeys(lits + [t for t in toks if len(t) >= 4]))[:12] per_file = defaultdict(lambda: [set(), 0]) rg = shutil.which("rg") for n in needles: if rg: cmd = [rg, "-c", "-i", "-F", "--follow", "--no-messages", "-g", "!*.json", "-g", "!.git", "--", n, *targets] else: cmd = ["grep", "-r", "-c", "-i", "-F", "--exclude=*.json", "--exclude-dir=.git", "--", n, *targets] try: out = subprocess.run(cmd, cwd=root, capture_output=True, text=True, timeout=30).stdout except (OSError, subprocess.TimeoutExpired): continue for line in out.splitlines(): path, _, cnt = line.rpartition(":") if not path or not cnt.isdigit() or int(cnt) == 0: continue per_file[path][0].add(n) per_file[path][1] += int(cnt) ranked = sorted(per_file.items(), key=lambda kv: (-len(kv[1][0]), -kv[1][1], kv[0])) inbox = {p for p in per_file if p.startswith("raw/inbox/")} return [p for p, _ in ranked], inbox def signal_tldr(pages, toks): qs = set(toks) scored = [] for rel, pg in pages.items(): hay = set(tokens(pg["title"] + " " + pg["fm"].get("tldr", ""))) hit = len(qs & hay) if hit: scored.append((-(hit / max(len(qs), 1)), -hit, rel)) return [r for _, _, r in sorted(scored)] def signal_scope(root, pages, toks, forced): proj = root / "wiki/projects" if not proj.is_dir(): return [], None qs = set(toks) best, best_hit = None, 0 for p in proj.glob("*.md"): if p.name == "index.md": continue if forced and p.stem == forced: best = p break pg = pages.get(p.relative_to(root).as_posix()) hit = len(qs & set(tokens((pg["title"] if pg else "") + " " + (pg["fm"].get("tldr", "") if pg else "") + " " + p.stem))) if hit > best_hit: best, best_hit = p, hit if not best: return [], None body = best.read_text(encoding="utf-8", errors="replace") refs = re.findall(r"\]\(([^)]+\.md)\)|\[\[([^\]]+)\]\]|`((?:wiki|raw|libs|outputs)/[^`]+)`", body) listed = [] for a, b, c in refs: r = norm_ref(a or b or c) cand = next((rel for rel in pages if norm_ref(rel) == r or rel.endswith(r + ".md")), None) if cand and cand not in listed: listed.append(cand) return listed, best.relative_to(root).as_posix() def signal_graph(edges, pages, shortlist): short = {norm_ref(s) for s in shortlist} neigh = Counter() for e in edges: a, b = norm_ref(str(e.get("from", ""))), norm_ref(str(e.get("to", ""))) if a in short and b not in short: neigh[b] += 1 if b in short and a not in short: neigh[a] += 1 out = [] for n, _ in neigh.most_common(): cand = next((rel for rel in pages if norm_ref(rel) == n or rel.endswith("/" + n + ".md")), None) if cand: out.append(cand) return out def signal_indegree(edges, shortlist): indeg = Counter(norm_ref(str(e.get("to", ""))) for e in edges) scored = [(indeg[norm_ref(s)], s) for s in shortlist if indeg[norm_ref(s)] > 0] return [s for _, s in sorted(scored, key=lambda x: (-x[0], x[1]))] def signal_trust(pages, shortlist, today): scored = [] for s in shortlist: pg = pages.get(s) if not pg: continue fm = pg["fm"] try: c, q = float(fm.get("confidence", 0.5)), float(fm.get("quality", 0.5)) except ValueError: c, q = 0.5, 0.5 fresh = is_fresh(fm, today) f = 1.0 if fresh in (True, None) else 0.5 scored.append((-(c * q * f), s)) return [s for _, s in sorted(scored)] # ---------------------------------------------------------------- fusion ---- def rrf(lists, k): score, ranks = defaultdict(float), defaultdict(dict) for name, (items, w) in lists.items(): for i, item in enumerate(items, 1): score[item] += w / (k + i) ranks[item][name] = i return sorted(score.items(), key=lambda kv: (-kv[1], kv[0])), ranks def main(): ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("query") ap.add_argument("--root", default=".") ap.add_argument("--k", type=int, default=10) ap.add_argument("--top", type=int, default=12) ap.add_argument("--scope", help="force a wiki/projects/ scope instead of matching one") ap.add_argument("--today", help="YYYY-MM-DD, for reproducible freshness") ap.add_argument("--json", action="store_true") a = ap.parse_args() root = Path(a.root).resolve() today = datetime.strptime(a.today, "%Y-%m-%d").date() if a.today else date.today() toks, lits = tokens(a.query), literal_tokens(a.query) pages, edges = load_pages(root), load_edges(root) exact, inbox_hits = signal_exact(root, toks, lits) tldr = signal_tldr(pages, toks) scope, scope_page = signal_scope(root, pages, toks, a.scope) shortlist = list(dict.fromkeys(tldr[:a.top] + [e for e in exact[:a.top] if e in pages] + scope)) graph = signal_graph(edges, pages, shortlist) indeg = signal_indegree(edges, shortlist) trust = signal_trust(pages, shortlist, today) lists = { "exact": (exact, 2.0 if lits else 1.0), "tldr": (tldr, 1.0), "scope": (scope, 1.0), "graph": (graph, 0.7), "indegree": (indeg, 0.5), "trust": (trust, 0.3), } fused, ranks = rrf({n: v for n, v in lists.items() if v[0]}, a.k) # Fold a cited raw/archive or outputs/ hit into the wiki page that cites it: # same claim, one candidate, corroboration recorded rather than double-counted. # A source cited by N pages corroborates all N, not just the first one walked. cited_by = defaultdict(list) for rel, pg in sorted(pages.items()): for src in pg["sources"]: cited_by[src.lstrip("/")].append(rel) results, absorbed = [], defaultdict(list) ranked = dict(fused) for item, sc in fused: owners = [o for o in cited_by.get(item, []) if o in ranked] if owners: for o in owners: absorbed[o].append(item) continue results.append((item, sc)) results = results[:a.top] out = { "query": a.query, "k": a.k, "literal_tokens": lits, "scope_page": scope_page, "inbox_hits": sorted(inbox_hits), "empty_graph": not edges, "candidates": [], } for item, sc in results: pg = pages.get(item) fm = pg["fm"] if pg else {} out["candidates"].append({ "path": item, "score": round(sc, 4), "ranks": ranks[item], "title": pg["title"] if pg else None, "type": fm.get("type"), "tldr": fm.get("tldr"), "last_updated": fm.get("last_updated"), "stale": (is_fresh(fm, today) is False) if pg else None, "confidence": fm.get("confidence"), "quality": fm.get("quality"), "superseded_by": fm.get("superseded_by"), "sources": pg["sources"] if pg else [], "corroborated_by": absorbed.get(item, []), "from_inbox": item.startswith("raw/inbox/"), }) if a.json: json.dump(out, sys.stdout, indent=2, ensure_ascii=False) print() return print(f"query: {a.query}") if lits: print(f"literal tokens (exact list weighted 2.0): {', '.join(lits)}") if scope_page: print(f"scope: {scope_page}") if out["empty_graph"]: print("graph: empty — graph/indegree lists skipped") if inbox_hits: print(f"raw/inbox/ hits ({len(inbox_hits)}): un-ingested material — an ingest is overdue") print() for i, c in enumerate(out["candidates"], 1): flags = [f for f, on in [("STALE", c["stale"]), ("INBOX", c["from_inbox"]), ("SUPERSEDED", c["superseded_by"])] if on] rk = " ".join(f"{n}:{r}" for n, r in sorted(c["ranks"].items())) print(f"{i:2}. {c['score']:.4f} {c['path']} [{rk}]" + (f" {' '.join(flags)}" if flags else "")) if c["tldr"]: print(f" {c['tldr']}") if c["corroborated_by"]: print(f" corroborated by: {', '.join(c['corroborated_by'])}") if not out["candidates"]: print("no candidates — nothing in the cascade matches; consider a wiki/query-gaps.md entry") if __name__ == "__main__": main()