"""Knowledge-base access for the ckb MCP server: path policy, reads, search, inbox writes and git sync. No MCP dependency on purpose, so the policy that decides what a remote client may read or write can be tested on its own (tests/mcp_policy.py). """ import json import os import re import subprocess import sys import threading import time from dataclasses import dataclass from datetime import datetime, timezone from pathlib import Path # What a remote client may read. Everything else - AGENTS.md aside, which is # served as a resource - stays private: .git, .env, tmp/, workload/, and # raw/saved-conversations/ (verbatim transcripts never leave the machine). READ_PREFIXES = ("wiki/", "linked/", "libs/", "raw/inbox/", "raw/archive/", "raw/retractions/", "outputs/") READ_ROOTS = tuple(p.rstrip("/") for p in READ_PREFIXES) RETRIEVE = Path(".agents/skills/ckb-retrieve/scripts/retrieve_candidates.py") MAX_LIST = 500 class PolicyError(ValueError): """A request the path or write policy refuses. The message is safe to show the client.""" class UnsavedWork(RuntimeError): """The sync-mode checkout holds work the server did not make, so it will not reset it.""" @dataclass class Author: name: str email: str def git(self): return f"{self.name} <{self.email}>" def parse_frontmatter(text): """Flat key: value frontmatter, the same deliberately simple reading as the rest of the template.""" if not text.startswith("---\n"): return {}, text end = text.find("\n---", 4) if end < 0: return {}, text fm = {} for line in text[4:end].splitlines(): m = re.match(r"^([A-Za-z_][\w-]*):\s*(.*)$", line) if m: fm[m.group(1)] = m.group(2).strip().strip('"').strip("'") return fm, text[end + 4:].lstrip("\n") def slugify(s, limit=60): s = re.sub(r"[^a-z0-9]+", "-", s.lower()).strip("-") return (s[:limit].rstrip("-")) or "note" class KB: def __init__(self, root, git_mode="off", pull_interval=60, max_inbox_bytes=200_000, committer=Author("ckb-mcp", "ckb-mcp@localhost"), remote="origin", branch=None): self.root = Path(root).resolve() if not (self.root / "wiki" / "index.md").is_file(): raise SystemExit(f"{self.root} is not a Cascade KB root (no wiki/index.md)") if git_mode not in ("off", "commit", "sync"): raise SystemExit(f"CKB_MCP_GIT must be off, commit or sync, not {git_mode!r}") self.git_mode = git_mode self.pull_interval = pull_interval self.max_inbox_bytes = max_inbox_bytes self.committer = committer self.remote = remote self.branch = branch self._last_pull = 0.0 # One lock for every git operation and inbox write. Reentrant: a write holds it # while forcing a pull; a read that finds it busy skips its pull instead of waiting. self._lock = threading.RLock() # ------------------------------------------------------------- paths --- def _allowed_dirs(self): """Resolved directories a read may land in: the root, plus each linked/ target, since linked/ entries are symlinks to other KB roots by design.""" dirs = [self.root] linked = self.root / "linked" if linked.is_dir(): dirs += [p.resolve() for p in linked.iterdir() if p.is_dir()] return dirs def resolve_read(self, rel): rel = (rel or "").strip().lstrip("/") if not rel or "\x00" in rel: raise PolicyError("empty path") parts = Path(rel).parts if ".." in parts: raise PolicyError("'..' is not allowed in a path") if any(p.startswith(".") for p in parts): raise PolicyError("hidden files are not readable") if parts[-1].endswith(".local.yaml"): raise PolicyError("per-user local configuration is not readable") if rel not in READ_ROOTS and not rel.startswith(READ_PREFIXES): raise PolicyError(f"only {', '.join(READ_PREFIXES)} are readable") # A linked/ is another KB's root: the same areas are readable inside it, # not its workload/, tmp/ or root files. if parts[0] == "linked" and len(parts) > 2: inner = "/".join(parts[2:]) if (inner not in READ_ROOTS and not inner.startswith(READ_PREFIXES) or parts[2] == "linked" or inner.startswith("raw/retractions")): raise PolicyError(f"inside linked// only {', '.join(p for p in READ_PREFIXES if p != 'linked/')} are readable") # The checks above judge the path as written. A symlink would let an innocent # name (wiki/x.md) land on .git/config, .env or workload/, so none is followed # except linked/ itself, which is what linked/ exists for. With no other # link on the way, the resolved path is the written one and the checks hold. cur = self.root for i, part in enumerate(parts): cur = cur / part if cur.is_symlink() and not (i == 1 and parts[0] == "linked"): raise PolicyError("symbolic links are not served") full = cur.resolve() if not any(full == d or d in full.parents for d in self._allowed_dirs()): raise PolicyError("path escapes the knowledge base") return full def _may_see(self, rel, viewer): """raw/retractions/ is private to each requester: a request's reason and page list can say why material was confidential, or name another KB.""" if not rel.startswith("raw/retractions/"): return True fm, _ = parse_frontmatter((self.root / rel).read_text(encoding="utf-8", errors="replace")) m = re.search(r"<([^>]+)>", fm.get("requested_by", "")) return bool(viewer and m and m.group(1).strip().lower() == viewer.email.lower()) def rel(self, full): try: return full.relative_to(self.root).as_posix() except ValueError: return full.as_posix() # ------------------------------------------------------------- reads --- def read_page(self, rel, viewer=None): self.maybe_pull() full = self.resolve_read(rel) if not full.is_file(): raise PolicyError(f"no such file: {rel}") if not self._may_see(self.rel(full), viewer): raise PolicyError("retraction requests are readable only by the person who made them") if full.stat().st_size > 2_000_000: raise PolicyError("file larger than 2 MB; not served") return full.read_text(encoding="utf-8", errors="replace") def list_pages(self, directory="wiki", viewer=None): self.maybe_pull() base = self.resolve_read(directory) if not base.is_dir(): raise PolicyError(f"not a directory: {directory}") # rglob does not descend into symlinked directories, and every linked/ is # one, so linked/ itself is walked one KB at a time from each link. starts = sorted(p for p in base.iterdir() if p.is_dir()) if base == self.root / "linked" else [base] out = [] for f in sorted(f for s in starts for f in s.rglob("*.md")): rel = f"{directory.strip('/')}/{f.relative_to(base).as_posix()}" try: self.resolve_read(rel) # same policy as read_page: skips links and hidden parts except PolicyError: continue if not self._may_see(rel, viewer): continue fm, body = parse_frontmatter(f.read_text(encoding="utf-8", errors="replace")) m = re.search(r"^#\s+(.+)$", body, re.M) out.append({ "path": rel, "title": m.group(1).strip() if m else f.stem, "type": fm.get("type"), "tldr": fm.get("tldr"), }) if len(out) >= MAX_LIST: break return out def overview(self): self.maybe_pull() parts = [] for name in ("ABOUT.md", "wiki/index.md", "wiki/overview.md"): p = self.root / name if p.is_file(): parts.append(f"===== {name} =====\n{p.read_text(encoding='utf-8')}") layers = [] for layer in ("linked", "libs"): d = self.root / layer names = sorted(p.name for p in d.iterdir() if p.is_dir() and not p.name.startswith(".")) if d.is_dir() else [] layers.append(f"{layer}/: {', '.join(names) if names else '(none)'}") inbox = self.root / "raw" / "inbox" pending = len([p for p in inbox.iterdir() if p.is_file() and not p.name.startswith(".")]) if inbox.is_dir() else 0 layers.append(f"raw/inbox/: {pending} item(s) awaiting ingest") retr = self.root / "raw" / "retractions" n = len(list(retr.glob("*.md"))) if retr.is_dir() else 0 layers.append(f"raw/retractions/: {n} retraction request(s) awaiting ckb-retract") parts.append("===== cascade layers =====\n" + "\n".join(layers)) return "\n\n".join(parts) def rules(self): p = self.root / "AGENTS.md" return p.read_text(encoding="utf-8") if p.is_file() else "" def search(self, query, top=10, scope=None): self.maybe_pull() script = self.root / RETRIEVE if not script.is_file(): raise PolicyError("retrieval script missing from this KB; run ckb-upgrade") cmd = [sys.executable, str(script), query, "--root", str(self.root), "--json", "--top", str(int(top))] if scope: cmd += ["--scope", scope] r = subprocess.run(cmd, capture_output=True, text=True, timeout=60) if r.returncode != 0: raise RuntimeError(f"retrieval failed: {r.stderr.strip()[-500:]}") return json.loads(r.stdout) # ------------------------------------------------------------ writes --- def add_to_inbox(self, title, content, author, source=None, now=None): title = (title or "").strip() if not title: raise PolicyError("a title is required") if not (content or "").strip(): raise PolicyError("content is empty") if len(content.encode("utf-8")) > self.max_inbox_bytes: raise PolicyError(f"content larger than {self.max_inbox_bytes} bytes") now = now or datetime.now(timezone.utc) inbox = self.root / "raw" / "inbox" def apply(): inbox.mkdir(parents=True, exist_ok=True) stem = f"{now:%Y-%m-%d}_{slugify(title)}" path, n = inbox / f"{stem}.md", 2 while path.exists(): path, n = inbox / f"{stem}-{n}.md", n + 1 # Provenance lives in frontmatter so ckb-ingest can attribute the item; # raw/ files are inputs, not wiki pages, so this is not the page schema. header = [ "---", f"title: {json.dumps(title, ensure_ascii=False)}", f"submitted_by: {json.dumps(author.git(), ensure_ascii=False)}", f"submitted_at: {now.isoformat(timespec='seconds')}", "via: mcp", ] if source: header.append(f"source: {json.dumps(source.strip(), ensure_ascii=False)}") header.append("---") path.write_text("\n".join(header) + f"\n\n# {title}\n\n{content.rstrip()}\n", encoding="utf-8") rel = self.rel(path) return [rel], {"path": rel} return self._write(apply, f"Inbox via MCP: {title}", author) # ------------------------------------------------- submissions, undo --- @staticmethod def _submitter_email(fm): m = re.search(r"<([^>]+)>", fm.get("submitted_by", "")) return m.group(1).strip().lower() if m else None def _dependents(self, rel): """Wiki pages and graph edges that cite a source path - what an ingest built from it.""" pages, edges = [], 0 wiki = self.root / "wiki" for f in sorted(wiki.rglob("*.md")): if f.is_symlink(): continue if rel in f.read_text(encoding="utf-8", errors="replace"): pages.append(self.rel(f)) ej = wiki / "graph" / "edges.json" if ej.is_file(): try: edges = sum(1 for e in json.loads(ej.read_text(encoding="utf-8")).get("edges", []) if rel in json.dumps(e)) except ValueError: pass return {"pages": pages, "edges": edges} def submissions(self, author): """The caller's own MCP submissions: pending in raw/inbox/, or ingested into raw/archive/ together with what was built from them.""" self.maybe_pull() email = author.email.lower() out = [] for area, status in (("raw/inbox", "pending"), ("raw/archive", "ingested")): base = self.root / area if not base.is_dir(): continue for f in sorted(base.rglob("*.md")): if f.is_symlink(): continue fm, _ = parse_frontmatter(f.read_text(encoding="utf-8", errors="replace")) if fm.get("via") != "mcp" or self._submitter_email(fm) != email: continue item = {"path": self.rel(f), "title": fm.get("title"), "submitted_at": fm.get("submitted_at"), "status": status} if status == "ingested": item.update(self._dependents(item["path"])) item["retraction_requested"] = self._pending_retraction(item["path"]) is not None out.append(item) return out def _pending_retraction(self, target): d = self.root / "raw" / "retractions" for f in sorted(d.glob("*.md")) if d.is_dir() else []: fm, _ = parse_frontmatter(f.read_text(encoding="utf-8", errors="replace")) if fm.get("target") == target: return self.rel(f) return None def _inbox_item(self, rel, prefix): full = self.resolve_read(rel) if not self.rel(full).startswith(prefix) or not full.is_file(): raise PolicyError(f"not a file under {prefix}: {rel}") fm, _ = parse_frontmatter(full.read_text(encoding="utf-8", errors="replace")) return full, fm def withdraw(self, rel, author): """Delete a not-yet-ingested MCP submission from raw/inbox/. Only its submitter may.""" def apply(): # Runs on a checkout freshly reset to the remote, so "gone" means gone upstream. try: full, fm = self._inbox_item(rel, "raw/inbox/") except PolicyError: name = Path(rel).name if rel.lstrip("/").startswith("raw/inbox/") and any((self.root / "raw" / "archive").glob(f"*/{name}")): raise PolicyError("already ingested: it is in raw/archive/ now, so use request_retraction") if rel.lstrip("/").startswith("raw/inbox/"): raise PolicyError("no longer in raw/inbox/: already withdrawn, or moved by someone else") raise if fm.get("via") != "mcp" or self._submitter_email(fm) != author.email.lower(): raise PolicyError("only the person who submitted this item can withdraw it") full.unlink() path = self.rel(full) apply.title = fm.get("title", path) return [path], {"withdrawn": path} return self._write(apply, lambda: f"Withdraw via MCP: {apply.title}", author) def request_retraction(self, rel, author, reason, now=None): """Queue the removal of an ingested source and everything built from it. Only the person who submitted the source may ask. The request is a file in raw/retractions/; ckb-retract carries it out with a person, because unwinding wiki pages takes judgement a remote client must not exercise.""" reason = (reason or "").strip() if not reason: raise PolicyError("a reason is required (e.g. 'submitted to the wrong KB; belongs in project-beta')") now = now or datetime.now(timezone.utc) def apply(): full, fm = self._inbox_item(rel, "raw/archive/") target = self.rel(full) owner = self._submitter_email(fm) if owner is None: raise PolicyError("this source was not submitted through MCP, so it has no submitter who could " "remove it; ask the KB's maintainers") if owner != author.email.lower(): raise PolicyError("only the person who submitted this source can ask for its removal") existing = self._pending_retraction(target) if existing: return [], {"request": existing, "already_requested": True} deps = self._dependents(target) d = self.root / "raw" / "retractions" d.mkdir(parents=True, exist_ok=True) stem = f"{now:%Y-%m-%d}_{slugify(Path(target).stem)}" path, n = d / f"{stem}.md", 2 while path.exists(): path, n = d / f"{stem}-{n}.md", n + 1 lines = [ "---", "type: retraction-request", f"target: {target}", f"requested_by: {json.dumps(author.git(), ensure_ascii=False)}", f"requested_at: {now.isoformat(timespec='seconds')}", f"reason: {json.dumps(reason, ensure_ascii=False)}", "via: mcp", "---", "", f"# Retract `{target}`", "", reason, "", "Cited at request time by:", "", *([f"- `{p}`" for p in deps["pages"]] or ["- (no wiki page)"]), f"- {deps['edges']} graph edge(s)", "", ] path.write_text("\n".join(lines), encoding="utf-8") apply.target = target return [self.rel(path)], {"request": self.rel(path), "target": target, **deps} return self._write(apply, lambda: f"Retraction request via MCP: {apply.target}", author) # --------------------------------------------------------------- git --- def _git(self, *args, check=True): # Every call carries the committer identity: a container has no global git # identity to fall back on. ident = ["-c", f"user.name={self.committer.name}", "-c", f"user.email={self.committer.email}"] try: r = subprocess.run(["git", "-C", str(self.root), *ident, *args], capture_output=True, text=True, timeout=120) except subprocess.TimeoutExpired: raise RuntimeError(f"git {args[0]} timed out after 120 s") from None if check and r.returncode != 0: raise RuntimeError(f"git {args[0]} failed: {(r.stderr or r.stdout).strip()[-500:]}") return r def maybe_pull(self, force=False): """Bring the checkout up to the remote at most once per pull_interval. A failure is logged and the stale checkout keeps serving: reads must not go down with the remote.""" if self.git_mode != "sync": return if not force and time.monotonic() - self._last_pull < self.pull_interval: return if not self._lock.acquire(blocking=force): return # a write or another pull is running; serve the current checkout try: self._reset_to_remote() except RuntimeError as e: print(f"ckb-mcp: {e}; serving the current checkout", file=sys.stderr) finally: self._lock.release() def _tracking(self): branch = self.branch or self._git("rev-parse", "--abbrev-ref", "HEAD").stdout.strip() return branch, f"{self.remote}/{branch}" def _unsaved_work(self, ref): """What a reset to ref would destroy that the server did not put there: local changes or unpushed commits. None when the checkout holds nothing of the kind.""" if self._git("status", "--porcelain").stdout.strip(): return "local changes" ahead = self._git("rev-list", "--count", f"{ref}..HEAD").stdout.strip() return None if ahead == "0" else f"{ahead} unpushed commit(s)" def _reset_to_remote(self, own=False): """Make the checkout exactly the remote branch. In sync mode the server's clone holds nothing but writes it has already pushed, so this loses nothing - and it means a rejected or conflicting write can never leave the clone diverged. That holds only while nobody else works in the clone, so unless the leftovers are the server's own (own=True: a write it just failed), any local work stops the reset instead.""" self._last_pull = time.monotonic() branch, ref = self._tracking() self._git("fetch", "--quiet", self.remote, branch) if not own: found = self._unsaved_work(ref) if found: raise UnsavedWork(f"{self.root} has {found} that the server did not make; not resetting it " f"to {ref}. Give the server its own clone, or push / discard those changes") self._git("reset", "--hard", "--quiet", ref) self._git("clean", "-fdq", "--", "raw") # files a failed write may have left; never ignored ones def _discard_own(self): try: self._reset_to_remote(own=True) except RuntimeError: pass def check_dedicated_clone(self): """sync mode resets this checkout to the remote, so refuse a checkout holding anything that a reset would destroy. The server must have its own clone. Runs at startup, so every failure is one line naming the cause, never a traceback.""" if self.git_mode != "sync": return try: branch, ref = self._tracking() self._git("fetch", "--quiet", self.remote, branch) found = self._unsaved_work(ref) except RuntimeError as e: raise SystemExit(f"CKB_MCP_GIT=sync could not reach {self.remote} from {self.root}: {e}") from None if found: raise SystemExit(f"CKB_MCP_GIT=sync resets {self.root} to {ref} before every write, and this " f"checkout has {found}. " "Give the server its own clone (or push / discard those changes first).") def _write(self, apply, message, author): """One write: in sync mode reset to the remote, apply, commit, push; a rejected push is retried once from a fresh reset, and a second rejection resets again and reports that nothing was saved. A checkout holding someone else's work is never reset: the write is refused instead, and reads keep serving it as it is. apply() returns (paths, result) and raises PolicyError to refuse; it re-runs on a retry, so its checks always see the current remote state.""" with self._lock: for attempt in (1, 2): if self.git_mode == "sync": try: # A retry starts from the server's own rejected commit, so only the # first reset checks for someone else's work. self._reset_to_remote(own=attempt == 2) except UnsavedWork as e: print(f"ckb-mcp: {e}; refusing the write", file=sys.stderr) raise PolicyError("the server's checkout holds changes it did not make, so it is not " "accepting writes until its operator clears them; nothing was saved") from None try: paths, result = apply() if not paths or self.git_mode == "off": return {**result, "committed": False, "pushed": False} msg = message() if callable(message) else message self._git("add", "-A", "--", *paths) # -A: a withdrawal stages a deletion self._git("commit", "--quiet", f"--author={author.git()}", "-m", msg, "--", *paths) commit = self._git("rev-parse", "--short", "HEAD").stdout.strip() if self.git_mode == "commit": return {**result, "committed": True, "commit": commit, "pushed": False} branch, _ = self._tracking() if self._git("push", "--quiet", self.remote, f"HEAD:{branch}", check=False).returncode == 0: return {**result, "committed": True, "commit": commit, "pushed": True} except Exception: if self.git_mode == "sync": self._discard_own() # whatever this write left is the server's own raise self._discard_own() raise PolicyError("the remote kept changing while saving; nothing was saved - please try again")