- F13 new skill ckb-consolidate with consolidate.py: a proposals-only report in outputs/reports/ (topic clusters and their state, merge candidates, missing links, stale pages, old log entries by month); never edits a page, never rewrites the log - F15 topic pages (type: topic) in the new wiki/topics/, built only on approval, with member-page digests as ## Sources so lint flags drift; ckb-retrieve routes "overall" questions to wiki/topics/index.md first - Schema 1.7 (D-0016): ckb-upgrade b4, ckb-init and ckb-reset know wiki/topics/; ckb-routine suggests a fortnightly Consolidate --confirm Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
263 lines
11 KiB
Python
263 lines
11 KiB
Python
#!/usr/bin/env python3
|
|
"""Mechanical half of ckb-consolidate: find what the wiki could tidy, propose it.
|
|
|
|
consolidate.py [--root DIR] [--today YYYY-MM-DD] [--json] [--write]
|
|
|
|
Read-only towards wiki/. It reports five things:
|
|
|
|
topics clusters of related pages (a project scope's members, pages that
|
|
share a tag, then linked groups of three or more) and, for each,
|
|
whether wiki/topics/<name>.md exists and is current
|
|
merges page pairs of one type whose title and tldr say nearly the same
|
|
links page pairs that share two sources (or a source and a scope)
|
|
yet do not link to each other
|
|
stale pages past their freshness window or whose source changed
|
|
(lint_report.py findings, repeated here so one report has it all)
|
|
log wiki/log.md entries older than 90 days, by month — a digest
|
|
for the report, never a rewrite of the log
|
|
|
|
--write saves the report to outputs/reports/YYYY-MM-DD_consolidation.md (and
|
|
never edits a page). Python 3.8+, standard library only.
|
|
"""
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
import re
|
|
import subprocess
|
|
import sys
|
|
from collections import defaultdict
|
|
from datetime import date, datetime
|
|
from itertools import combinations
|
|
from pathlib import Path
|
|
|
|
HERE = Path(__file__).resolve().parent
|
|
LINT = HERE.parent.parent / "ckb-lint" / "scripts" / "lint_report.py"
|
|
RESERVED = {"index.md", "log.md"}
|
|
INFRA = {"overview.md", "error-book.md", "query-gaps.md", "tags.md", "lessons.md"}
|
|
STOP = set("a an and are as at be by for from how in is it of on or that the this to was with its".split())
|
|
LINK_RE = re.compile(r"\]\((/?wiki/[^)#\s]+\.md|[^)#\s:]+\.md)\)")
|
|
WIKILINK_RE = re.compile(r"\[\[([^\]|#]+)")
|
|
SOURCE_RE = re.compile(r"^\s*[-*]\s+`([^`]+)`", re.M)
|
|
LOG_HEADER_RE = re.compile(r"^##\s*\[(\d{4}-\d{2}-\d{2})[^\]]*\]\s*-\s*(.+?)\s*$", re.M)
|
|
MIN_CLUSTER = 3
|
|
MIN_SCOPE_CLUSTER = 2
|
|
OLD_LOG_DAYS = 90
|
|
|
|
|
|
def parse(text):
|
|
if not text.startswith("---\n"):
|
|
return {}, text
|
|
end = text.find("\n---\n", 4)
|
|
if end == -1:
|
|
return {}, text
|
|
fm = {}
|
|
for line in text[4:end].split("\n"):
|
|
k, sep, v = line.partition(":")
|
|
if sep and not line[:1].isspace():
|
|
fm[k.strip()] = v.strip().strip("\"'")
|
|
return fm, text[end + 5:]
|
|
|
|
|
|
def words(s):
|
|
return {w for w in re.findall(r"[a-z0-9]+", s.lower()) if w not in STOP and len(w) > 2}
|
|
|
|
|
|
def load(root):
|
|
pages = {}
|
|
wiki = root / "wiki"
|
|
for p in sorted(wiki.rglob("*.md")):
|
|
rel = p.relative_to(wiki).as_posix()
|
|
if p.name in RESERVED or rel in INFRA or any(part.startswith(".") for part in p.parts):
|
|
continue
|
|
text = p.read_text(encoding="utf-8", errors="replace")
|
|
fm, body = parse(text)
|
|
title = next((l[2:].strip() for l in body.splitlines() if l.startswith("# ")), p.stem)
|
|
links = set()
|
|
for t in LINK_RE.findall(body):
|
|
t = t.lstrip("/")
|
|
t = t[len("wiki/"):] if t.startswith("wiki/") else (p.parent.relative_to(wiki) / t).as_posix()
|
|
links.add(re.sub(r"[^/]+/\.\./", "", t))
|
|
links |= {(w.strip() if w.strip().endswith(".md") else w.strip() + ".md") for w in WIKILINK_RE.findall(body)}
|
|
sec = body.split("## Sources", 1)
|
|
sources = set(SOURCE_RE.findall(sec[1].split("\n## ", 1)[0])) if len(sec) > 1 else set()
|
|
pages[rel] = {"fm": fm, "title": title, "links": links, "sources": {s.lstrip("/") for s in sources},
|
|
"tags": {t.strip().lower() for t in fm.get("tags", "").split(",") if t.strip()},
|
|
"digest": hashlib.sha256(p.read_bytes()).hexdigest()[:8]}
|
|
return pages
|
|
|
|
|
|
def load_edges(root):
|
|
f = root / "wiki" / "graph" / "edges.json"
|
|
try:
|
|
data = json.loads(f.read_text(encoding="utf-8")) if f.is_file() else {}
|
|
except ValueError:
|
|
return []
|
|
edges = data.get("edges", []) if isinstance(data, dict) else data
|
|
norm = lambda v: (lambda s: s if s.endswith(".md") else s + ".md")(str(v).lstrip("/").removeprefix("wiki/"))
|
|
return [(norm(e.get("from", "")), norm(e.get("to", ""))) for e in edges if isinstance(e, dict)]
|
|
|
|
|
|
def clusters(pages, edges):
|
|
"""[(name, kind, sorted members)]: scopes first, then tags, then linked groups."""
|
|
out, members = [], [r for r in pages if not r.startswith(("topics/", "projects/"))]
|
|
for rel, pg in sorted(pages.items()):
|
|
if rel.startswith("projects/"):
|
|
m = sorted(x for x in pg["links"] if x in pages and x in members)
|
|
if len(m) >= MIN_SCOPE_CLUSTER:
|
|
out.append((Path(rel).stem, "scope", m))
|
|
by_tag = defaultdict(list)
|
|
for rel in members:
|
|
for t in pages[rel]["tags"]:
|
|
by_tag[t].append(rel)
|
|
out += [(t, "tag", sorted(m)) for t, m in sorted(by_tag.items()) if len(m) >= MIN_CLUSTER]
|
|
claimed = {m for _, _, ms in out for m in ms}
|
|
adj = defaultdict(set)
|
|
for a, b in edges + [(r, l) for r in members for l in pages[r]["links"]]:
|
|
if a in members and b in members and a != b:
|
|
adj[a].add(b)
|
|
adj[b].add(a)
|
|
seen = set(claimed)
|
|
for start in sorted(members):
|
|
if start in seen or start not in adj:
|
|
continue
|
|
comp, stack = set(), [start]
|
|
while stack:
|
|
n = stack.pop()
|
|
if n in comp:
|
|
continue
|
|
comp.add(n)
|
|
stack.extend(x for x in adj[n] if x not in claimed)
|
|
seen |= comp
|
|
if len(comp) >= MIN_CLUSTER:
|
|
hub = max(sorted(comp), key=lambda n: len(adj[n]))
|
|
out.append((Path(hub).stem, "linked", sorted(comp)))
|
|
return out
|
|
|
|
|
|
def topic_state(pages, name, members):
|
|
"""'missing', 'current', or what changed since the topic page was written."""
|
|
topic = pages.get(f"topics/{name}.md")
|
|
if topic is None:
|
|
return "missing", []
|
|
text = (Path(ROOT) / "wiki" / "topics" / f"{name}.md").read_text(encoding="utf-8", errors="replace")
|
|
recorded = dict(re.findall(r"^\s*[-*]\s+`/?(?:wiki/)?([^`]+)`\s+—\s+sha256:([0-9a-f]+)", text, re.M))
|
|
notes = [f"{m} changed" for m in members if m in recorded and not pages[m]["digest"].startswith(recorded[m])]
|
|
notes += [f"{m} is new to the cluster" for m in members if m not in recorded]
|
|
notes += [f"{m} left the cluster" for m in recorded if m not in members]
|
|
return ("current" if not notes else "outdated"), notes
|
|
|
|
|
|
def merges(pages):
|
|
out = []
|
|
items = [(r, p) for r, p in sorted(pages.items()) if not r.startswith("topics/")]
|
|
for (a, pa), (b, pb) in combinations(items, 2):
|
|
if pa["fm"].get("type") != pb["fm"].get("type") or pa["fm"].get("type") == "decision":
|
|
continue
|
|
wa, wb = words(pa["title"] + " " + pa["fm"].get("tldr", "")), words(pb["title"] + " " + pb["fm"].get("tldr", ""))
|
|
if wa and wb and len(wa & wb) / len(wa | wb) >= 0.5:
|
|
out.append((a, b, round(len(wa & wb) / len(wa | wb), 2)))
|
|
return out
|
|
|
|
|
|
def missing_links(pages, scope_members):
|
|
out = []
|
|
for (a, pa), (b, pb) in combinations(sorted(pages.items()), 2):
|
|
if a.startswith(("topics/", "projects/")) or b.startswith(("topics/", "projects/")):
|
|
continue
|
|
if b in pa["links"] or a in pb["links"]:
|
|
continue
|
|
shared = pa["sources"] & pb["sources"]
|
|
same_scope = any(a in m and b in m for m in scope_members)
|
|
if len(shared) >= 2 or (shared and same_scope):
|
|
out.append((a, b, sorted(shared)))
|
|
return out
|
|
|
|
|
|
def stale(root, today):
|
|
try:
|
|
out = subprocess.run([sys.executable, "-I", str(LINT), "--json", "--scope", "wiki", "--today", today.isoformat()],
|
|
cwd=str(root), capture_output=True, text=True, timeout=120).stdout
|
|
tree = json.loads(out)["trees"][0]
|
|
except (OSError, ValueError, KeyError, IndexError, subprocess.SubprocessError):
|
|
return ["(lint_report.py unavailable — run \"lint\" for stale pages)"]
|
|
return tree.get("freshness", []) + [f for f in tree.get("evidence", []) if "CHANGED" in f or "does not exist" in f]
|
|
|
|
|
|
def log_digest(root, today):
|
|
log = root / "wiki" / "log.md"
|
|
if not log.is_file():
|
|
return {}
|
|
months = defaultdict(int)
|
|
for d, _ in LOG_HEADER_RE.findall(log.read_text(encoding="utf-8", errors="replace")):
|
|
try:
|
|
when = datetime.strptime(d, "%Y-%m-%d").date()
|
|
except ValueError:
|
|
continue
|
|
if (today - when).days > OLD_LOG_DAYS:
|
|
months[d[:7]] += 1
|
|
return dict(sorted(months.items()))
|
|
|
|
|
|
def report(data, today):
|
|
lines = [f"---\ntype: report\ntitle: Consolidation proposals {today.isoformat()}\ndate: {today.isoformat()}\n---\n",
|
|
f"# Consolidation proposals — {today.isoformat()}\n",
|
|
"Proposals only. Nothing in `wiki/` was changed; approved items run through "
|
|
"`ckb-consolidate` (topics), `ckb-ingest`/`ckb-lint` (edits) or `ckb-decide`.\n",
|
|
"## Topics\n"]
|
|
for t in data["topics"]:
|
|
lines.append(f"- **{t['name']}** ({t['kind']}, {len(t['members'])} pages) — {t['state']}"
|
|
+ (": " + "; ".join(t["notes"]) if t["notes"] else "") + "\n " + ", ".join(f"`{m}`" for m in t["members"]))
|
|
if not data["topics"]:
|
|
lines.append("- none")
|
|
lines.append("\n## Merge candidates\n")
|
|
lines += [f"- `{a}` and `{b}` — title/tldr overlap {s}" for a, b, s in data["merges"]] or ["- none"]
|
|
lines.append("\n## Missing links\n")
|
|
lines += [f"- `{a}` ↔ `{b}` — both cite {', '.join(f'`{x}`' for x in shared)}" for a, b, shared in data["links"]] or ["- none"]
|
|
lines.append("\n## Stale or drifted\n")
|
|
lines += [f"- {s}" for s in data["stale"]] or ["- none"]
|
|
lines.append(f"\n## Log older than {OLD_LOG_DAYS} days\n")
|
|
lines += [f"- {m}: {n} entries" for m, n in data["log"].items()] or ["- none"]
|
|
return "\n".join(lines).rstrip() + "\n"
|
|
|
|
|
|
ROOT = Path(".")
|
|
|
|
|
|
def main(argv=None):
|
|
global ROOT
|
|
ap = argparse.ArgumentParser(description="Consolidation proposals for wiki/ (read-only).")
|
|
ap.add_argument("--root", default=".")
|
|
ap.add_argument("--today")
|
|
ap.add_argument("--json", action="store_true")
|
|
ap.add_argument("--write", action="store_true", help="save the report to outputs/reports/")
|
|
a = ap.parse_args(argv)
|
|
ROOT = Path(a.root).resolve()
|
|
if not (ROOT / "wiki").is_dir():
|
|
sys.exit(f"error: {ROOT} has no wiki/ — not a Cascade KB root")
|
|
today = datetime.strptime(a.today, "%Y-%m-%d").date() if a.today else date.today()
|
|
pages, edges = load(ROOT), load_edges(ROOT)
|
|
cl = clusters(pages, edges)
|
|
topics = []
|
|
for name, kind, members in cl:
|
|
state, notes = topic_state(pages, name, members)
|
|
topics.append({"name": name, "kind": kind, "members": members, "state": state, "notes": notes,
|
|
"digests": {m: pages[m]["digest"] for m in members}})
|
|
scope_members = [set(t["members"]) for t in topics if t["kind"] == "scope"]
|
|
data = {"today": today.isoformat(), "topics": topics, "merges": merges(pages),
|
|
"links": missing_links(pages, scope_members), "stale": stale(ROOT, today), "log": log_digest(ROOT, today)}
|
|
if a.json:
|
|
print(json.dumps(data, indent=2))
|
|
return 0
|
|
text = report(data, today)
|
|
if a.write:
|
|
out = ROOT / "outputs" / "reports" / f"{today.isoformat()}_consolidation.md"
|
|
out.parent.mkdir(parents=True, exist_ok=True)
|
|
out.write_text(text, encoding="utf-8")
|
|
print(f"wrote {out.relative_to(ROOT).as_posix()}")
|
|
else:
|
|
print(text, end="")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|