ckb/.agents/modules/mcp/server/kb.py
Michał Kopeć da2833abfe Sync: local changes as of 2026-11-01T20:31:36Z
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-01 21:31:36 +01:00

507 lines
25 KiB
Python

"""Knowledge-base access for the ckb MCP server: path policy, reads, search,
inbox writes and git sync.
No MCP dependency on purpose, so the policy that decides what a remote client
may read or write can be tested on its own (tests/mcp_policy.py).
"""
import json
import os
import re
import subprocess
import sys
import threading
import time
from dataclasses import dataclass
from datetime import datetime, timezone
from pathlib import Path
# What a remote client may read. Everything else - AGENTS.md aside, which is
# served as a resource - stays private: .git, .env, tmp/, workload/, and
# raw/saved-conversations/ (verbatim transcripts never leave the machine).
READ_PREFIXES = ("wiki/", "linked/", "libs/", "raw/inbox/", "raw/archive/", "raw/retractions/", "outputs/")
READ_ROOTS = tuple(p.rstrip("/") for p in READ_PREFIXES)
RETRIEVE = Path(".agents/skills/ckb-retrieve/scripts/retrieve_candidates.py")
MAX_LIST = 500
class PolicyError(ValueError):
"""A request the path or write policy refuses. The message is safe to show the client."""
class UnsavedWork(RuntimeError):
"""The sync-mode checkout holds work the server did not make, so it will not reset it."""
@dataclass
class Author:
name: str
email: str
def git(self):
return f"{self.name} <{self.email}>"
def parse_frontmatter(text):
"""Flat key: value frontmatter, the same deliberately simple reading as the rest of the template."""
if not text.startswith("---\n"):
return {}, text
end = text.find("\n---", 4)
if end < 0:
return {}, text
fm = {}
for line in text[4:end].splitlines():
m = re.match(r"^([A-Za-z_][\w-]*):\s*(.*)$", line)
if m:
fm[m.group(1)] = m.group(2).strip().strip('"').strip("'")
return fm, text[end + 4:].lstrip("\n")
def slugify(s, limit=60):
s = re.sub(r"[^a-z0-9]+", "-", s.lower()).strip("-")
return (s[:limit].rstrip("-")) or "note"
class KB:
def __init__(self, root, git_mode="off", pull_interval=60, max_inbox_bytes=200_000,
committer=Author("ckb-mcp", "ckb-mcp@localhost"), remote="origin", branch=None):
self.root = Path(root).resolve()
if not (self.root / "wiki" / "index.md").is_file():
raise SystemExit(f"{self.root} is not a Cascade KB root (no wiki/index.md)")
if git_mode not in ("off", "commit", "sync"):
raise SystemExit(f"CKB_MCP_GIT must be off, commit or sync, not {git_mode!r}")
self.git_mode = git_mode
self.pull_interval = pull_interval
self.max_inbox_bytes = max_inbox_bytes
self.committer = committer
self.remote = remote
self.branch = branch
self._last_pull = 0.0
# One lock for every git operation and inbox write. Reentrant: a write holds it
# while forcing a pull; a read that finds it busy skips its pull instead of waiting.
self._lock = threading.RLock()
# ------------------------------------------------------------- paths ---
def _allowed_dirs(self):
"""Resolved directories a read may land in: the root, plus each linked/<name> target,
since linked/ entries are symlinks to other KB roots by design."""
dirs = [self.root]
linked = self.root / "linked"
if linked.is_dir():
dirs += [p.resolve() for p in linked.iterdir() if p.is_dir()]
return dirs
def resolve_read(self, rel):
rel = (rel or "").strip().lstrip("/")
if not rel or "\x00" in rel:
raise PolicyError("empty path")
parts = Path(rel).parts
if ".." in parts:
raise PolicyError("'..' is not allowed in a path")
if any(p.startswith(".") for p in parts):
raise PolicyError("hidden files are not readable")
if parts[-1].endswith(".local.yaml"):
raise PolicyError("per-user local configuration is not readable")
if rel not in READ_ROOTS and not rel.startswith(READ_PREFIXES):
raise PolicyError(f"only {', '.join(READ_PREFIXES)} are readable")
# A linked/<name> is another KB's root: the same areas are readable inside it,
# not its workload/, tmp/ or root files.
if parts[0] == "linked" and len(parts) > 2:
inner = "/".join(parts[2:])
if (inner not in READ_ROOTS and not inner.startswith(READ_PREFIXES)
or parts[2] == "linked" or inner.startswith("raw/retractions")):
raise PolicyError(f"inside linked/<name>/ only {', '.join(p for p in READ_PREFIXES if p != 'linked/')} are readable")
# The checks above judge the path as written. A symlink would let an innocent
# name (wiki/x.md) land on .git/config, .env or workload/, so none is followed
# except linked/<name> itself, which is what linked/ exists for. With no other
# link on the way, the resolved path is the written one and the checks hold.
cur = self.root
for i, part in enumerate(parts):
cur = cur / part
if cur.is_symlink() and not (i == 1 and parts[0] == "linked"):
raise PolicyError("symbolic links are not served")
full = cur.resolve()
if not any(full == d or d in full.parents for d in self._allowed_dirs()):
raise PolicyError("path escapes the knowledge base")
return full
def _may_see(self, rel, viewer):
"""raw/retractions/ is private to each requester: a request's reason and page list
can say why material was confidential, or name another KB."""
if not rel.startswith("raw/retractions/"):
return True
fm, _ = parse_frontmatter((self.root / rel).read_text(encoding="utf-8", errors="replace"))
m = re.search(r"<([^>]+)>", fm.get("requested_by", ""))
return bool(viewer and m and m.group(1).strip().lower() == viewer.email.lower())
def rel(self, full):
try:
return full.relative_to(self.root).as_posix()
except ValueError:
return full.as_posix()
# ------------------------------------------------------------- reads ---
def read_page(self, rel, viewer=None):
self.maybe_pull()
full = self.resolve_read(rel)
if not full.is_file():
raise PolicyError(f"no such file: {rel}")
if not self._may_see(self.rel(full), viewer):
raise PolicyError("retraction requests are readable only by the person who made them")
if full.stat().st_size > 2_000_000:
raise PolicyError("file larger than 2 MB; not served")
return full.read_text(encoding="utf-8", errors="replace")
def list_pages(self, directory="wiki", viewer=None):
self.maybe_pull()
base = self.resolve_read(directory)
if not base.is_dir():
raise PolicyError(f"not a directory: {directory}")
# rglob does not descend into symlinked directories, and every linked/<name> is
# one, so linked/ itself is walked one KB at a time from each link.
starts = sorted(p for p in base.iterdir() if p.is_dir()) if base == self.root / "linked" else [base]
out = []
for f in sorted(f for s in starts for f in s.rglob("*.md")):
rel = f"{directory.strip('/')}/{f.relative_to(base).as_posix()}"
try:
self.resolve_read(rel) # same policy as read_page: skips links and hidden parts
except PolicyError:
continue
if not self._may_see(rel, viewer):
continue
fm, body = parse_frontmatter(f.read_text(encoding="utf-8", errors="replace"))
m = re.search(r"^#\s+(.+)$", body, re.M)
out.append({
"path": rel,
"title": m.group(1).strip() if m else f.stem,
"type": fm.get("type"), "tldr": fm.get("tldr"),
})
if len(out) >= MAX_LIST:
break
return out
def overview(self):
self.maybe_pull()
parts = []
for name in ("ABOUT.md", "wiki/index.md", "wiki/overview.md"):
p = self.root / name
if p.is_file():
parts.append(f"===== {name} =====\n{p.read_text(encoding='utf-8')}")
layers = []
for layer in ("linked", "libs"):
d = self.root / layer
names = sorted(p.name for p in d.iterdir() if p.is_dir() and not p.name.startswith(".")) if d.is_dir() else []
layers.append(f"{layer}/: {', '.join(names) if names else '(none)'}")
inbox = self.root / "raw" / "inbox"
pending = len([p for p in inbox.iterdir() if p.is_file() and not p.name.startswith(".")]) if inbox.is_dir() else 0
layers.append(f"raw/inbox/: {pending} item(s) awaiting ingest")
retr = self.root / "raw" / "retractions"
n = len(list(retr.glob("*.md"))) if retr.is_dir() else 0
layers.append(f"raw/retractions/: {n} retraction request(s) awaiting ckb-retract")
parts.append("===== cascade layers =====\n" + "\n".join(layers))
return "\n\n".join(parts)
def rules(self):
p = self.root / "AGENTS.md"
return p.read_text(encoding="utf-8") if p.is_file() else ""
def search(self, query, top=10, scope=None):
self.maybe_pull()
script = self.root / RETRIEVE
if not script.is_file():
raise PolicyError("retrieval script missing from this KB; run ckb-upgrade")
cmd = [sys.executable, str(script), query, "--root", str(self.root), "--json", "--top", str(int(top))]
if scope:
cmd += ["--scope", scope]
r = subprocess.run(cmd, capture_output=True, text=True, timeout=60)
if r.returncode != 0:
raise RuntimeError(f"retrieval failed: {r.stderr.strip()[-500:]}")
return json.loads(r.stdout)
# ------------------------------------------------------------ writes ---
def add_to_inbox(self, title, content, author, source=None, now=None):
title = (title or "").strip()
if not title:
raise PolicyError("a title is required")
if not (content or "").strip():
raise PolicyError("content is empty")
if len(content.encode("utf-8")) > self.max_inbox_bytes:
raise PolicyError(f"content larger than {self.max_inbox_bytes} bytes")
now = now or datetime.now(timezone.utc)
inbox = self.root / "raw" / "inbox"
def apply():
inbox.mkdir(parents=True, exist_ok=True)
stem = f"{now:%Y-%m-%d}_{slugify(title)}"
path, n = inbox / f"{stem}.md", 2
while path.exists():
path, n = inbox / f"{stem}-{n}.md", n + 1
# Provenance lives in frontmatter so ckb-ingest can attribute the item;
# raw/ files are inputs, not wiki pages, so this is not the page schema.
header = [
"---",
f"title: {json.dumps(title, ensure_ascii=False)}",
f"submitted_by: {json.dumps(author.git(), ensure_ascii=False)}",
f"submitted_at: {now.isoformat(timespec='seconds')}",
"via: mcp",
]
if source:
header.append(f"source: {json.dumps(source.strip(), ensure_ascii=False)}")
header.append("---")
path.write_text("\n".join(header) + f"\n\n# {title}\n\n{content.rstrip()}\n", encoding="utf-8")
rel = self.rel(path)
return [rel], {"path": rel}
return self._write(apply, f"Inbox via MCP: {title}", author)
# ------------------------------------------------- submissions, undo ---
@staticmethod
def _submitter_email(fm):
m = re.search(r"<([^>]+)>", fm.get("submitted_by", ""))
return m.group(1).strip().lower() if m else None
def _dependents(self, rel):
"""Wiki pages and graph edges that cite a source path - what an ingest built from it."""
pages, edges = [], 0
wiki = self.root / "wiki"
for f in sorted(wiki.rglob("*.md")):
if f.is_symlink():
continue
if rel in f.read_text(encoding="utf-8", errors="replace"):
pages.append(self.rel(f))
ej = wiki / "graph" / "edges.json"
if ej.is_file():
try:
edges = sum(1 for e in json.loads(ej.read_text(encoding="utf-8")).get("edges", [])
if rel in json.dumps(e))
except ValueError:
pass
return {"pages": pages, "edges": edges}
def submissions(self, author):
"""The caller's own MCP submissions: pending in raw/inbox/, or ingested into
raw/archive/ together with what was built from them."""
self.maybe_pull()
email = author.email.lower()
out = []
for area, status in (("raw/inbox", "pending"), ("raw/archive", "ingested")):
base = self.root / area
if not base.is_dir():
continue
for f in sorted(base.rglob("*.md")):
if f.is_symlink():
continue
fm, _ = parse_frontmatter(f.read_text(encoding="utf-8", errors="replace"))
if fm.get("via") != "mcp" or self._submitter_email(fm) != email:
continue
item = {"path": self.rel(f), "title": fm.get("title"), "submitted_at": fm.get("submitted_at"),
"status": status}
if status == "ingested":
item.update(self._dependents(item["path"]))
item["retraction_requested"] = self._pending_retraction(item["path"]) is not None
out.append(item)
return out
def _pending_retraction(self, target):
d = self.root / "raw" / "retractions"
for f in sorted(d.glob("*.md")) if d.is_dir() else []:
fm, _ = parse_frontmatter(f.read_text(encoding="utf-8", errors="replace"))
if fm.get("target") == target:
return self.rel(f)
return None
def _inbox_item(self, rel, prefix):
full = self.resolve_read(rel)
if not self.rel(full).startswith(prefix) or not full.is_file():
raise PolicyError(f"not a file under {prefix}: {rel}")
fm, _ = parse_frontmatter(full.read_text(encoding="utf-8", errors="replace"))
return full, fm
def withdraw(self, rel, author):
"""Delete a not-yet-ingested MCP submission from raw/inbox/. Only its submitter may."""
def apply():
# Runs on a checkout freshly reset to the remote, so "gone" means gone upstream.
try:
full, fm = self._inbox_item(rel, "raw/inbox/")
except PolicyError:
name = Path(rel).name
if rel.lstrip("/").startswith("raw/inbox/") and any((self.root / "raw" / "archive").glob(f"*/{name}")):
raise PolicyError("already ingested: it is in raw/archive/ now, so use request_retraction")
if rel.lstrip("/").startswith("raw/inbox/"):
raise PolicyError("no longer in raw/inbox/: already withdrawn, or moved by someone else")
raise
if fm.get("via") != "mcp" or self._submitter_email(fm) != author.email.lower():
raise PolicyError("only the person who submitted this item can withdraw it")
full.unlink()
path = self.rel(full)
apply.title = fm.get("title", path)
return [path], {"withdrawn": path}
return self._write(apply, lambda: f"Withdraw via MCP: {apply.title}", author)
def request_retraction(self, rel, author, reason, now=None):
"""Queue the removal of an ingested source and everything built from it. Only the
person who submitted the source may ask. The request is a file in raw/retractions/;
ckb-retract carries it out with a person, because unwinding wiki pages takes
judgement a remote client must not exercise."""
reason = (reason or "").strip()
if not reason:
raise PolicyError("a reason is required (e.g. 'submitted to the wrong KB; belongs in project-beta')")
now = now or datetime.now(timezone.utc)
def apply():
full, fm = self._inbox_item(rel, "raw/archive/")
target = self.rel(full)
owner = self._submitter_email(fm)
if owner is None:
raise PolicyError("this source was not submitted through MCP, so it has no submitter who could "
"remove it; ask the KB's maintainers")
if owner != author.email.lower():
raise PolicyError("only the person who submitted this source can ask for its removal")
existing = self._pending_retraction(target)
if existing:
return [], {"request": existing, "already_requested": True}
deps = self._dependents(target)
d = self.root / "raw" / "retractions"
d.mkdir(parents=True, exist_ok=True)
stem = f"{now:%Y-%m-%d}_{slugify(Path(target).stem)}"
path, n = d / f"{stem}.md", 2
while path.exists():
path, n = d / f"{stem}-{n}.md", n + 1
lines = [
"---",
"type: retraction-request",
f"target: {target}",
f"requested_by: {json.dumps(author.git(), ensure_ascii=False)}",
f"requested_at: {now.isoformat(timespec='seconds')}",
f"reason: {json.dumps(reason, ensure_ascii=False)}",
"via: mcp",
"---", "",
f"# Retract `{target}`", "",
reason, "",
"Cited at request time by:", "",
*([f"- `{p}`" for p in deps["pages"]] or ["- (no wiki page)"]),
f"- {deps['edges']} graph edge(s)", "",
]
path.write_text("\n".join(lines), encoding="utf-8")
apply.target = target
return [self.rel(path)], {"request": self.rel(path), "target": target, **deps}
return self._write(apply, lambda: f"Retraction request via MCP: {apply.target}", author)
# --------------------------------------------------------------- git ---
def _git(self, *args, check=True):
# Every call carries the committer identity: a container has no global git
# identity to fall back on.
ident = ["-c", f"user.name={self.committer.name}", "-c", f"user.email={self.committer.email}"]
try:
r = subprocess.run(["git", "-C", str(self.root), *ident, *args], capture_output=True, text=True, timeout=120)
except subprocess.TimeoutExpired:
raise RuntimeError(f"git {args[0]} timed out after 120 s") from None
if check and r.returncode != 0:
raise RuntimeError(f"git {args[0]} failed: {(r.stderr or r.stdout).strip()[-500:]}")
return r
def maybe_pull(self, force=False):
"""Bring the checkout up to the remote at most once per pull_interval. A failure is
logged and the stale checkout keeps serving: reads must not go down with the remote."""
if self.git_mode != "sync":
return
if not force and time.monotonic() - self._last_pull < self.pull_interval:
return
if not self._lock.acquire(blocking=force):
return # a write or another pull is running; serve the current checkout
try:
self._reset_to_remote()
except RuntimeError as e:
print(f"ckb-mcp: {e}; serving the current checkout", file=sys.stderr)
finally:
self._lock.release()
def _tracking(self):
branch = self.branch or self._git("rev-parse", "--abbrev-ref", "HEAD").stdout.strip()
return branch, f"{self.remote}/{branch}"
def _unsaved_work(self, ref):
"""What a reset to ref would destroy that the server did not put there: local
changes or unpushed commits. None when the checkout holds nothing of the kind."""
if self._git("status", "--porcelain").stdout.strip():
return "local changes"
ahead = self._git("rev-list", "--count", f"{ref}..HEAD").stdout.strip()
return None if ahead == "0" else f"{ahead} unpushed commit(s)"
def _reset_to_remote(self, own=False):
"""Make the checkout exactly the remote branch. In sync mode the server's clone holds
nothing but writes it has already pushed, so this loses nothing - and it means a
rejected or conflicting write can never leave the clone diverged. That holds only
while nobody else works in the clone, so unless the leftovers are the server's own
(own=True: a write it just failed), any local work stops the reset instead."""
self._last_pull = time.monotonic()
branch, ref = self._tracking()
self._git("fetch", "--quiet", self.remote, branch)
if not own:
found = self._unsaved_work(ref)
if found:
raise UnsavedWork(f"{self.root} has {found} that the server did not make; not resetting it "
f"to {ref}. Give the server its own clone, or push / discard those changes")
self._git("reset", "--hard", "--quiet", ref)
self._git("clean", "-fdq", "--", "raw") # files a failed write may have left; never ignored ones
def _discard_own(self):
try:
self._reset_to_remote(own=True)
except RuntimeError:
pass
def check_dedicated_clone(self):
"""sync mode resets this checkout to the remote, so refuse a checkout holding
anything that a reset would destroy. The server must have its own clone. Runs at
startup, so every failure is one line naming the cause, never a traceback."""
if self.git_mode != "sync":
return
try:
branch, ref = self._tracking()
self._git("fetch", "--quiet", self.remote, branch)
found = self._unsaved_work(ref)
except RuntimeError as e:
raise SystemExit(f"CKB_MCP_GIT=sync could not reach {self.remote} from {self.root}: {e}") from None
if found:
raise SystemExit(f"CKB_MCP_GIT=sync resets {self.root} to {ref} before every write, and this "
f"checkout has {found}. "
"Give the server its own clone (or push / discard those changes first).")
def _write(self, apply, message, author):
"""One write: in sync mode reset to the remote, apply, commit, push; a rejected push
is retried once from a fresh reset, and a second rejection resets again and reports
that nothing was saved. A checkout holding someone else's work is never reset: the
write is refused instead, and reads keep serving it as it is. apply() returns (paths, result) and raises PolicyError to
refuse; it re-runs on a retry, so its checks always see the current remote state."""
with self._lock:
for attempt in (1, 2):
if self.git_mode == "sync":
try:
# A retry starts from the server's own rejected commit, so only the
# first reset checks for someone else's work.
self._reset_to_remote(own=attempt == 2)
except UnsavedWork as e:
print(f"ckb-mcp: {e}; refusing the write", file=sys.stderr)
raise PolicyError("the server's checkout holds changes it did not make, so it is not "
"accepting writes until its operator clears them; nothing was saved") from None
try:
paths, result = apply()
if not paths or self.git_mode == "off":
return {**result, "committed": False, "pushed": False}
msg = message() if callable(message) else message
self._git("add", "-A", "--", *paths) # -A: a withdrawal stages a deletion
self._git("commit", "--quiet", f"--author={author.git()}", "-m", msg, "--", *paths)
commit = self._git("rev-parse", "--short", "HEAD").stdout.strip()
if self.git_mode == "commit":
return {**result, "committed": True, "commit": commit, "pushed": False}
branch, _ = self._tracking()
if self._git("push", "--quiet", self.remote, f"HEAD:{branch}", check=False).returncode == 0:
return {**result, "committed": True, "commit": commit, "pushed": True}
except Exception:
if self.git_mode == "sync":
self._discard_own() # whatever this write left is the server's own
raise
self._discard_own()
raise PolicyError("the remote kept changing while saving; nothing was saved - please try again")