ckb/.agents/skills/ckb-consolidate/scripts/consolidate.py
Michał Kopeć 221a6a29b1 Template 1.25.0: release 3 batch 2, consolidation and topics, schema 1.7
- F13 new skill ckb-consolidate with consolidate.py: a proposals-only
  report in outputs/reports/ (topic clusters and their state, merge
  candidates, missing links, stale pages, old log entries by month);
  never edits a page, never rewrites the log
- F15 topic pages (type: topic) in the new wiki/topics/, built only on
  approval, with member-page digests as ## Sources so lint flags drift;
  ckb-retrieve routes "overall" questions to wiki/topics/index.md first
- Schema 1.7 (D-0016): ckb-upgrade b4, ckb-init and ckb-reset know
  wiki/topics/; ckb-routine suggests a fortnightly Consolidate --confirm

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-02 20:30:26 +01:00

263 lines
11 KiB
Python

#!/usr/bin/env python3
"""Mechanical half of ckb-consolidate: find what the wiki could tidy, propose it.
consolidate.py [--root DIR] [--today YYYY-MM-DD] [--json] [--write]
Read-only towards wiki/. It reports five things:
topics clusters of related pages (a project scope's members, pages that
share a tag, then linked groups of three or more) and, for each,
whether wiki/topics/<name>.md exists and is current
merges page pairs of one type whose title and tldr say nearly the same
links page pairs that share two sources (or a source and a scope)
yet do not link to each other
stale pages past their freshness window or whose source changed
(lint_report.py findings, repeated here so one report has it all)
log wiki/log.md entries older than 90 days, by month — a digest
for the report, never a rewrite of the log
--write saves the report to outputs/reports/YYYY-MM-DD_consolidation.md (and
never edits a page). Python 3.8+, standard library only.
"""
import argparse
import hashlib
import json
import re
import subprocess
import sys
from collections import defaultdict
from datetime import date, datetime
from itertools import combinations
from pathlib import Path
HERE = Path(__file__).resolve().parent
LINT = HERE.parent.parent / "ckb-lint" / "scripts" / "lint_report.py"
RESERVED = {"index.md", "log.md"}
INFRA = {"overview.md", "error-book.md", "query-gaps.md", "tags.md", "lessons.md"}
STOP = set("a an and are as at be by for from how in is it of on or that the this to was with its".split())
LINK_RE = re.compile(r"\]\((/?wiki/[^)#\s]+\.md|[^)#\s:]+\.md)\)")
WIKILINK_RE = re.compile(r"\[\[([^\]|#]+)")
SOURCE_RE = re.compile(r"^\s*[-*]\s+`([^`]+)`", re.M)
LOG_HEADER_RE = re.compile(r"^##\s*\[(\d{4}-\d{2}-\d{2})[^\]]*\]\s*-\s*(.+?)\s*$", re.M)
MIN_CLUSTER = 3
MIN_SCOPE_CLUSTER = 2
OLD_LOG_DAYS = 90
def parse(text):
if not text.startswith("---\n"):
return {}, text
end = text.find("\n---\n", 4)
if end == -1:
return {}, text
fm = {}
for line in text[4:end].split("\n"):
k, sep, v = line.partition(":")
if sep and not line[:1].isspace():
fm[k.strip()] = v.strip().strip("\"'")
return fm, text[end + 5:]
def words(s):
return {w for w in re.findall(r"[a-z0-9]+", s.lower()) if w not in STOP and len(w) > 2}
def load(root):
pages = {}
wiki = root / "wiki"
for p in sorted(wiki.rglob("*.md")):
rel = p.relative_to(wiki).as_posix()
if p.name in RESERVED or rel in INFRA or any(part.startswith(".") for part in p.parts):
continue
text = p.read_text(encoding="utf-8", errors="replace")
fm, body = parse(text)
title = next((l[2:].strip() for l in body.splitlines() if l.startswith("# ")), p.stem)
links = set()
for t in LINK_RE.findall(body):
t = t.lstrip("/")
t = t[len("wiki/"):] if t.startswith("wiki/") else (p.parent.relative_to(wiki) / t).as_posix()
links.add(re.sub(r"[^/]+/\.\./", "", t))
links |= {(w.strip() if w.strip().endswith(".md") else w.strip() + ".md") for w in WIKILINK_RE.findall(body)}
sec = body.split("## Sources", 1)
sources = set(SOURCE_RE.findall(sec[1].split("\n## ", 1)[0])) if len(sec) > 1 else set()
pages[rel] = {"fm": fm, "title": title, "links": links, "sources": {s.lstrip("/") for s in sources},
"tags": {t.strip().lower() for t in fm.get("tags", "").split(",") if t.strip()},
"digest": hashlib.sha256(p.read_bytes()).hexdigest()[:8]}
return pages
def load_edges(root):
f = root / "wiki" / "graph" / "edges.json"
try:
data = json.loads(f.read_text(encoding="utf-8")) if f.is_file() else {}
except ValueError:
return []
edges = data.get("edges", []) if isinstance(data, dict) else data
norm = lambda v: (lambda s: s if s.endswith(".md") else s + ".md")(str(v).lstrip("/").removeprefix("wiki/"))
return [(norm(e.get("from", "")), norm(e.get("to", ""))) for e in edges if isinstance(e, dict)]
def clusters(pages, edges):
"""[(name, kind, sorted members)]: scopes first, then tags, then linked groups."""
out, members = [], [r for r in pages if not r.startswith(("topics/", "projects/"))]
for rel, pg in sorted(pages.items()):
if rel.startswith("projects/"):
m = sorted(x for x in pg["links"] if x in pages and x in members)
if len(m) >= MIN_SCOPE_CLUSTER:
out.append((Path(rel).stem, "scope", m))
by_tag = defaultdict(list)
for rel in members:
for t in pages[rel]["tags"]:
by_tag[t].append(rel)
out += [(t, "tag", sorted(m)) for t, m in sorted(by_tag.items()) if len(m) >= MIN_CLUSTER]
claimed = {m for _, _, ms in out for m in ms}
adj = defaultdict(set)
for a, b in edges + [(r, l) for r in members for l in pages[r]["links"]]:
if a in members and b in members and a != b:
adj[a].add(b)
adj[b].add(a)
seen = set(claimed)
for start in sorted(members):
if start in seen or start not in adj:
continue
comp, stack = set(), [start]
while stack:
n = stack.pop()
if n in comp:
continue
comp.add(n)
stack.extend(x for x in adj[n] if x not in claimed)
seen |= comp
if len(comp) >= MIN_CLUSTER:
hub = max(sorted(comp), key=lambda n: len(adj[n]))
out.append((Path(hub).stem, "linked", sorted(comp)))
return out
def topic_state(pages, name, members):
"""'missing', 'current', or what changed since the topic page was written."""
topic = pages.get(f"topics/{name}.md")
if topic is None:
return "missing", []
text = (Path(ROOT) / "wiki" / "topics" / f"{name}.md").read_text(encoding="utf-8", errors="replace")
recorded = dict(re.findall(r"^\s*[-*]\s+`/?(?:wiki/)?([^`]+)`\s+—\s+sha256:([0-9a-f]+)", text, re.M))
notes = [f"{m} changed" for m in members if m in recorded and not pages[m]["digest"].startswith(recorded[m])]
notes += [f"{m} is new to the cluster" for m in members if m not in recorded]
notes += [f"{m} left the cluster" for m in recorded if m not in members]
return ("current" if not notes else "outdated"), notes
def merges(pages):
out = []
items = [(r, p) for r, p in sorted(pages.items()) if not r.startswith("topics/")]
for (a, pa), (b, pb) in combinations(items, 2):
if pa["fm"].get("type") != pb["fm"].get("type") or pa["fm"].get("type") == "decision":
continue
wa, wb = words(pa["title"] + " " + pa["fm"].get("tldr", "")), words(pb["title"] + " " + pb["fm"].get("tldr", ""))
if wa and wb and len(wa & wb) / len(wa | wb) >= 0.5:
out.append((a, b, round(len(wa & wb) / len(wa | wb), 2)))
return out
def missing_links(pages, scope_members):
out = []
for (a, pa), (b, pb) in combinations(sorted(pages.items()), 2):
if a.startswith(("topics/", "projects/")) or b.startswith(("topics/", "projects/")):
continue
if b in pa["links"] or a in pb["links"]:
continue
shared = pa["sources"] & pb["sources"]
same_scope = any(a in m and b in m for m in scope_members)
if len(shared) >= 2 or (shared and same_scope):
out.append((a, b, sorted(shared)))
return out
def stale(root, today):
try:
out = subprocess.run([sys.executable, "-I", str(LINT), "--json", "--scope", "wiki", "--today", today.isoformat()],
cwd=str(root), capture_output=True, text=True, timeout=120).stdout
tree = json.loads(out)["trees"][0]
except (OSError, ValueError, KeyError, IndexError, subprocess.SubprocessError):
return ["(lint_report.py unavailable — run \"lint\" for stale pages)"]
return tree.get("freshness", []) + [f for f in tree.get("evidence", []) if "CHANGED" in f or "does not exist" in f]
def log_digest(root, today):
log = root / "wiki" / "log.md"
if not log.is_file():
return {}
months = defaultdict(int)
for d, _ in LOG_HEADER_RE.findall(log.read_text(encoding="utf-8", errors="replace")):
try:
when = datetime.strptime(d, "%Y-%m-%d").date()
except ValueError:
continue
if (today - when).days > OLD_LOG_DAYS:
months[d[:7]] += 1
return dict(sorted(months.items()))
def report(data, today):
lines = [f"---\ntype: report\ntitle: Consolidation proposals {today.isoformat()}\ndate: {today.isoformat()}\n---\n",
f"# Consolidation proposals — {today.isoformat()}\n",
"Proposals only. Nothing in `wiki/` was changed; approved items run through "
"`ckb-consolidate` (topics), `ckb-ingest`/`ckb-lint` (edits) or `ckb-decide`.\n",
"## Topics\n"]
for t in data["topics"]:
lines.append(f"- **{t['name']}** ({t['kind']}, {len(t['members'])} pages) — {t['state']}"
+ (": " + "; ".join(t["notes"]) if t["notes"] else "") + "\n " + ", ".join(f"`{m}`" for m in t["members"]))
if not data["topics"]:
lines.append("- none")
lines.append("\n## Merge candidates\n")
lines += [f"- `{a}` and `{b}` — title/tldr overlap {s}" for a, b, s in data["merges"]] or ["- none"]
lines.append("\n## Missing links\n")
lines += [f"- `{a}` ↔ `{b}` — both cite {', '.join(f'`{x}`' for x in shared)}" for a, b, shared in data["links"]] or ["- none"]
lines.append("\n## Stale or drifted\n")
lines += [f"- {s}" for s in data["stale"]] or ["- none"]
lines.append(f"\n## Log older than {OLD_LOG_DAYS} days\n")
lines += [f"- {m}: {n} entries" for m, n in data["log"].items()] or ["- none"]
return "\n".join(lines).rstrip() + "\n"
ROOT = Path(".")
def main(argv=None):
global ROOT
ap = argparse.ArgumentParser(description="Consolidation proposals for wiki/ (read-only).")
ap.add_argument("--root", default=".")
ap.add_argument("--today")
ap.add_argument("--json", action="store_true")
ap.add_argument("--write", action="store_true", help="save the report to outputs/reports/")
a = ap.parse_args(argv)
ROOT = Path(a.root).resolve()
if not (ROOT / "wiki").is_dir():
sys.exit(f"error: {ROOT} has no wiki/ — not a Cascade KB root")
today = datetime.strptime(a.today, "%Y-%m-%d").date() if a.today else date.today()
pages, edges = load(ROOT), load_edges(ROOT)
cl = clusters(pages, edges)
topics = []
for name, kind, members in cl:
state, notes = topic_state(pages, name, members)
topics.append({"name": name, "kind": kind, "members": members, "state": state, "notes": notes,
"digests": {m: pages[m]["digest"] for m in members}})
scope_members = [set(t["members"]) for t in topics if t["kind"] == "scope"]
data = {"today": today.isoformat(), "topics": topics, "merges": merges(pages),
"links": missing_links(pages, scope_members), "stale": stale(ROOT, today), "log": log_digest(ROOT, today)}
if a.json:
print(json.dumps(data, indent=2))
return 0
text = report(data, today)
if a.write:
out = ROOT / "outputs" / "reports" / f"{today.isoformat()}_consolidation.md"
out.parent.mkdir(parents=True, exist_ok=True)
out.write_text(text, encoding="utf-8")
print(f"wrote {out.relative_to(ROOT).as_posix()}")
else:
print(text, end="")
return 0
if __name__ == "__main__":
sys.exit(main())