ckb/.agents/modules/share/skills/ckb-share/scripts/share.py
Michał Kopeć f6aa0f5b1d Template 2.0.6: frontmatter contract across eight readers, dirty-KB lint test
All eight frontmatter readers close at a `---` line, including one at
end of file (consolidate, dedup, OKF and share missed it; retrieve,
Starlight and mcp stopped at any line starting with `---`).
tests/unit/test_frontmatter_contract.py (T9) and
tests/unit/test_lint_dirty.py (T2: one planted fault per lint check).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-11 17:38:59 +01:00

932 lines
43 KiB
Python

#!/usr/bin/env python3
"""ckb-share: send documents between Cascade KBs through a shared git repo (a *channel*).
share.py join --channel NAME --url URL [--branch main] [--kb-name NAME]
share.py send SOURCE --to beta[,gamma|*] [--channel NAME] [--slug S] [--title T]
[--keep-notes] [--raw-ok] [--dry-run]
share.py unshare SHARE_ID [--version N|all] [--reason TEXT] [--channel NAME] [--dry-run]
share.py list [--offline] [--json] # incoming messages not yet handled
share.py receive --take all|ID[,ID] [--decline ID[,ID]] [--accept-unverified ID[,ID]] [--offline]
share.py status [--offline] # what this KB sent, what waits
Common: --root DIR (default .), --today YYYY-MM-DD, --author "Name <email>".
Channel layout: CHANNEL.md, members.yaml (kb name -> owners), mail/<kb>/ — one
append-only mailbox per KB, written only by that KB. Clones live in
tmp/share/<channel>/. The `share:` block of ckb.yaml is owned by this script:
channels, this KB's mailbox name, and the ledger of incoming messages handled.
Received documents land in raw/inbox/ (never wiki/); retraction messages become
requests in raw/retractions/ for ckb-retract. Every received file is marked
`untrusted: true`. A message whose git author is not an owner of the mailbox it
sits in is reported as unverified and is taken only with --accept-unverified.
Exit codes: 0 ok; 1 nothing to do / refused; 2 usage or config error;
3 the packed document contains a possible secret (never sent).
Python 3.8+, standard library and git only.
"""
import argparse
import importlib.util
import json
import os
import re
import subprocess
import sys
from datetime import date, datetime, timezone
from pathlib import Path
NAME_RE = re.compile(r"^[a-z0-9][a-z0-9-]{0,62}$")
SLUG_RE = re.compile(r"[^a-z0-9]+")
LINK_RE = re.compile(r"(!?)\[([^\]]*)\]\(([^)\s]+)(?:\s+\"[^\"]*\")?\)")
WIKILINK_RE = re.compile(r"\[\[([^\]|]+)(?:\|([^\]]+))?\]\]")
PATH_LINE_RE = re.compile(r"^(\s*(?:[-*]|—|--)\s+)`([^`]+)`")
PRIVATE_RE = re.compile(r"<private>.*?</private>", re.S | re.I)
PRIVATE_OPEN_RE = re.compile(r"<private>.*\Z", re.S | re.I) # unclosed: drop to the end
# Names that reach file paths, git arguments or the ckb.yaml ledger. Anything
# else in a channel is ignored: a channel member controls these strings.
FILE_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,200}\.md$")
BRANCH_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._/-]{0,100}$")
LEDGER_TOKEN_RE = re.compile(r"^[^\s\x00-\x1f\x7f]+$")
SEND_ROOTS = ("outputs/documents/", "outputs/answers/", "wiki/")
RAW_ROOTS = ("raw/archive/",)
class Fail(Exception):
def __init__(self, msg, code=2):
super().__init__(msg)
self.code = code
# ------------------------------------------------------------------ helpers
def slugify(text):
return SLUG_RE.sub("-", text.lower()).strip("-")[:60] or "document"
def git(args, cwd, check=True, ident=None):
cmd = ["git"]
if ident:
cmd += ["-c", f"user.name={ident[0]}", "-c", f"user.email={ident[1]}"]
r = subprocess.run(cmd + list(args), cwd=str(cwd), capture_output=True, text=True, encoding="utf-8", errors="replace")
if check and r.returncode != 0:
raise Fail(f"git {' '.join(args)} failed in {cwd}: {(r.stderr or r.stdout).strip()}")
return r
def identity(root, author):
if author:
m = re.match(r"^\s*(.*?)\s*<([^>]+)>\s*$", author)
if not m:
raise Fail('--author must look like "Name <email>"')
return m.group(1), m.group(2)
name = git(["config", "user.name"], root, check=False).stdout.strip()
email = git(["config", "user.email"], root, check=False).stdout.strip()
user = os.environ.get("USER") or os.environ.get("USERNAME") or "unknown"
return name or user, email or f"{user}@localhost"
FM_CLOSE_RE = re.compile(r"^---[ \t]*$", re.M) # the closing line; may end the file
def split_frontmatter(text):
if text.startswith("---\n"):
close = FM_CLOSE_RE.search(text, 4)
if close is not None:
return text[4:close.start()].rstrip("\n"), text[close.end() + 1:]
return "", text
def parse_value(v):
v = v.strip()
if v.startswith("[") and v.endswith("]"):
return [x.strip().strip("\"'") for x in v[1:-1].split(",") if x.strip()]
if len(v) >= 2 and v[0] == '"' and v[-1] == '"':
try:
return json.loads(v)
except ValueError:
return v[1:-1]
if len(v) >= 2 and v[0] == "'" and v[-1] == "'":
return v[1:-1]
return v
def parse_fm(text):
fm, _ = split_frontmatter(text)
out = {}
for line in fm.split("\n"):
m = re.match(r"^([A-Za-z_][A-Za-z0-9_]*):\s*(.*)$", line)
if m:
out[m.group(1)] = parse_value(m.group(2))
return out
def fm_line(key, value):
if isinstance(value, list):
return f"{key}: [{', '.join(value)}]"
if isinstance(value, bool):
return f"{key}: {'true' if value else 'false'}"
s = str(value)
if s == "" or re.search(r"[:#\[\]{}\"'`]|^\s|\s$", s) or s.lower() in ("true", "false", "null", "yes", "no"):
return f"{key}: {json.dumps(s, ensure_ascii=False)}"
return f"{key}: {s}"
def render(fields, body):
return "---\n" + "\n".join(fm_line(k, v) for k, v in fields.items() if v is not None) + "\n---\n\n" + body.lstrip("\n")
# ------------------------------------------------------------- ckb.yaml block
class Config:
"""The `share:` block of ckb.yaml. Script-owned: rewritten whole on save."""
def __init__(self, root):
self.path = root / "ckb.yaml"
self.lines = self.path.read_text(encoding="utf-8-sig").splitlines() if self.path.exists() else []
self.kb_name, self.channels, self.received = "", [], []
self.span = None
start = next((i for i, l in enumerate(self.lines) if re.match(r"^share:\s*(#.*)?$", l)), None)
if start is None:
return
end = len(self.lines)
for j in range(start + 1, len(self.lines)):
l = self.lines[j]
if l.strip() and not l.startswith((" ", "\t", "#")):
end = j
break
self.span = (start, end)
section, cur = None, None
for l in self.lines[start + 1:end]:
text = l.split(" #", 1)[0].rstrip() if not l.lstrip().startswith("- ") else l.rstrip()
if not text.strip() or text.lstrip().startswith("#"):
continue
m = re.match(r"^ ([a-z_]+):\s*(.*)$", text)
if m:
section = m.group(1)
if section == "kb_name":
self.kb_name = parse_value(m.group(2).split(" #", 1)[0])
continue
if section == "channels":
m = re.match(r"^ -\s+name:\s*(.*)$", text)
if m:
cur = {"name": parse_value(m.group(1)), "url": "", "branch": "main"}
self.channels.append(cur)
continue
m = re.match(r"^ (url|branch):\s*(.*)$", text)
if m and cur is not None:
cur[m.group(1)] = parse_value(m.group(2))
elif section == "received":
m = re.match(r"^ -\s+(.*)$", text)
if m:
parts = m.group(1).split()
if len(parts) >= 4:
self.received.append({"channel": parts[0], "path": parts[1], "action": parts[2], "date": parts[3]})
def channel(self, name=None):
if not self.channels:
raise Fail("no share channel configured — join one first: share.py join --channel NAME --url URL")
if name is None:
if len(self.channels) > 1:
raise Fail("several channels configured; name one with --channel: "
+ ", ".join(c["name"] for c in self.channels))
return self.channels[0]
for c in self.channels:
if c["name"] == name:
return c
raise Fail(f"no channel named {name!r} in ckb.yaml")
def handled(self, channel, path):
return next((r for r in self.received if r["channel"] == channel and r["path"] == path), None)
def save(self):
for c in self.channels:
for key in ("name", "url", "branch"):
if not LEDGER_TOKEN_RE.match(str(c.get(key) or "")):
raise Fail(f"channel {key} {c.get(key)!r} contains whitespace or control characters")
for r in self.received:
if not all(LEDGER_TOKEN_RE.match(str(r[k])) for k in ("channel", "path", "action", "date")):
raise Fail(f"refusing to record ledger entry {r!r}: whitespace or control characters")
block = ["share: # ckb-share: channels, this KB's mailbox name, incoming ledger (script-owned)",
f" kb_name: {self.kb_name}",
" channels:" + ("" if self.channels else " []")]
for c in self.channels:
block += [f" - name: {c['name']}", f" url: {c['url']}", f" branch: {c['branch']}"]
block.append(" received:" + ("" if self.received else " []")
+ " # <channel> <message path> <taken|declined|filed|seen> <date>")
block += [f" - {r['channel']} {r['path']} {r['action']} {r['date']}" for r in self.received]
lines = list(self.lines)
if self.span:
lines[self.span[0]:self.span[1]] = block + ([""] if self.span[1] < len(lines) else [])
else:
if lines and lines[-1].strip():
lines.append("")
lines += block
while lines and not lines[-1].strip():
lines.pop()
self.path.write_text("\n".join(lines) + "\n", encoding="utf-8")
# ------------------------------------------------------------------ channel
class Members(dict):
"""members.yaml as {mailbox: [owner emails]}, plus `.duplicates`."""
duplicates = frozenset()
class Channel:
def __init__(self, root, cfg, ident, today):
self.cfg, self.ident, self.today = cfg, ident, today
self.name, self.url, self.branch = cfg["name"], cfg["url"], cfg.get("branch") or "main"
# These come from committed ckb.yaml and reach git's argv: a value
# starting with `-` would be read as an option (e.g. --upload-pack).
if not NAME_RE.match(self.name):
raise Fail(f"channel name {self.name!r} is not a plain name")
if not BRANCH_RE.match(self.branch) or ".." in self.branch:
raise Fail(f"channel {self.name}: branch {self.branch!r} is not a plain branch name")
if not self.url or self.url.startswith("-"):
raise Fail(f"channel {self.name}: url {self.url!r} is not usable")
self.dir = root / "tmp" / "share" / self.name
def ensure_clone(self):
if (self.dir / ".git").is_dir():
return
self.dir.parent.mkdir(parents=True, exist_ok=True)
git(["clone", "--quiet", "--", self.url, str(self.dir)], self.dir.parent)
# The remote's HEAD may name another branch (a bare repo defaults to master):
# track origin/<branch> when it exists, else start it here.
if git(["rev-parse", "--verify", "--quiet", f"refs/remotes/origin/{self.branch}"], self.dir, check=False).returncode == 0:
git(["checkout", "--quiet", "-B", self.branch, f"origin/{self.branch}"], self.dir)
else:
if git(["switch", "--quiet", "--orphan", self.branch], self.dir, check=False).returncode != 0:
git(["checkout", "--quiet", "-B", self.branch], self.dir, check=False)
def has_commits(self):
return git(["rev-parse", "--verify", "HEAD"], self.dir, check=False).returncode == 0
def pull(self, offline=False):
self.ensure_clone()
if offline or not self.has_commits():
return True
r = git(["pull", "--quiet", "--rebase", "origin", self.branch], self.dir, check=False)
if r.returncode != 0:
print(f"warning: could not pull channel {self.name} ({(r.stderr or r.stdout).strip()[:200]}); "
"using the local copy", file=sys.stderr)
return False
return True
def commit_push(self, paths, message):
git(["add", "--"] + paths, self.dir)
git(["commit", "--quiet", "-m", message], self.dir, ident=self.ident)
r = git(["push", "--quiet", "-u", "origin", self.branch], self.dir, check=False)
if r.returncode != 0:
git(["pull", "--quiet", "--rebase", "origin", self.branch], self.dir)
git(["push", "--quiet", "-u", "origin", self.branch], self.dir)
return git(["rev-parse", "--short", "HEAD"], self.dir).stdout.strip()
def members(self):
path = self.dir / "members.yaml"
out, cur = {}, None
self.duplicates = set()
if not path.exists():
return out
for line in path.read_text(encoding="utf-8-sig").splitlines():
text = line.split("#", 1)[0].rstrip()
m = re.match(r"^ ([a-z0-9][a-z0-9-]*):\s*$", text)
if m:
# A second entry for the same mailbox would quietly add owners;
# treat the mailbox as unowned instead (verify() says why).
if m.group(1) in out:
self.duplicates.add(m.group(1))
cur = out.setdefault(m.group(1), [])
continue
m = re.match(r"^ owners:\s*(.*)$", text)
if m and cur is not None:
val = parse_value(m.group(1))
cur.extend(x.lower() for x in (val if isinstance(val, list) else [val]) if x)
for name in self.duplicates:
out[name] = []
out = Members(out)
out.duplicates = set(self.duplicates)
return out
def messages(self, mailbox=None):
base = self.dir / "mail"
if not base.is_dir():
return []
out = []
boxes = [base / mailbox] if mailbox else sorted(p for p in base.iterdir() if p.is_dir())
for box in boxes:
if not box.is_dir() or not NAME_RE.match(box.name):
continue
for f in sorted(box.glob("*.md")):
if not FILE_RE.match(f.name) or f.is_symlink():
print(f"warning: ignoring {f.relative_to(self.dir).as_posix()!r} in channel {self.name}: "
"not a plain file name", file=sys.stderr)
continue
text = f.read_text(encoding="utf-8-sig", errors="replace")
fm = parse_fm(text)
if fm.get("type") != "share":
continue
bad = malformed(fm, box.name)
if bad:
print(f"warning: ignoring {f.relative_to(self.dir).as_posix()} in channel {self.name}: {bad}",
file=sys.stderr)
continue
out.append({"file": f, "path": f.relative_to(self.dir).as_posix(), "box": box.name,
"fm": fm, "text": text})
return out
def authors(self, rel):
"""Every commit author that touched `rel`, oldest first ("Name <email>")."""
r = git(["log", "--format=%an <%ae>", "--", rel], self.dir, check=False)
return list(reversed([l for l in r.stdout.strip().split("\n") if l]))
def malformed(fm, box):
"""Why a share message's identifiers can't be trusted as path parts, or ''."""
if fm.get("from") != box:
return f"`from: {fm.get('from')}` sits in mailbox {box}"
sid = str(fm.get("share_id") or "")
owner, _, slug = sid.partition("/")
if owner != box or not NAME_RE.match(slug or "-"):
return f"share_id {sid!r} does not belong to mailbox {box}"
if fm.get("kind") == "document":
if not re.match(r"^[1-9][0-9]{0,5}$", str(fm.get("version") or "")):
return f"version {fm.get('version')!r} is not a positive number"
elif fm.get("kind") == "retraction":
if not re.match(r"^(all|[1-9][0-9]{0,5})$", str(fm.get("retracts_version") or "all")):
return f"retracts_version {fm.get('retracts_version')!r} is not a version"
return ""
def verify(msg, members):
"""(ok, reason, author) — every git author who touched the file must own its mailbox."""
fm, authors = msg["fm"], msg.get("authors", [])
author = authors[0] if authors else ""
if fm.get("from") != msg["box"]:
return False, f"`from: {fm.get('from')}` sits in mailbox {msg['box']}", author
owners = members.get(msg["box"], [])
if msg["box"] in getattr(members, "duplicates", ()):
return False, f"mailbox {msg['box']} is listed more than once in members.yaml", author
if not owners:
return False, f"mailbox {msg['box']} has no owners in members.yaml", author
if not authors:
return False, "the file has no commit history in the channel", author
for a in authors:
m = re.search(r"<([^>]+)>", a)
if not m or m.group(1).lower() not in owners:
verb = "added" if a == authors[0] else "changed"
return False, f"{verb} by {a}, who is not an owner of mailbox {msg['box']}", author
return True, "", author
# -------------------------------------------------------------------- pack
def secret_hits(root, text):
lint = root / ".agents" / "skills" / "ckb-lint" / "scripts" / "lint_report.py"
if not lint.exists():
lint = Path(__file__).resolve().parents[2] / "ckb-lint" / "scripts" / "lint_report.py"
if not lint.exists():
raise Fail("ckb-lint's lint_report.py not found: nothing is sent without a secret scan", 3)
spec = importlib.util.spec_from_file_location("ckb_lint_report", str(lint))
mod = importlib.util.module_from_spec(spec)
spec.loader.exec_module(mod)
hits = []
lineno = lambda pos: text.count("\n", 0, pos) + 1
for label, pattern, word in mod.SECRET_PATTERNS:
for m in pattern.finditer(text):
if word and m.start() and (text[m.start() - 1].isalnum() or text[m.start() - 1] == "_"):
continue
hits.append(f"line {lineno(m.start())}: possible {label} ({m.group(0)[:4]}…)")
for m in mod.ASSIGNED_VALUE_RE.finditer(text):
if mod.ASSIGNED_KEY_RE.search(text, max(0, m.start() - 24), m.start()):
hits.append(f"line {lineno(m.start())}: possible assigned secret")
return hits
def pack_body(body, kb, keep_notes):
"""Self-contained copy: private spans and ## Notes dropped, local links as text."""
dropped = []
body, found = strip_private(body)
if found:
dropped.append("<private> spans")
out, skipping = [], False
for line in body.split("\n"):
if re.match(r"^#{1,2}\s", line):
skipping = (not keep_notes) and line.strip().lower() == "## notes"
if skipping:
if "## Notes" not in dropped:
dropped.append("## Notes")
continue
if skipping:
continue
out.append(line)
text = "\n".join(out)
def link(m):
bang, label, target = m.groups()
if re.match(r"^(?:[a-z][a-z0-9+.-]*:|#)", target, re.I):
return m.group(0)
path = target.lstrip("/").split("#", 1)[0]
if bang:
return f"[image not shared: {label or path}]"
return f"{label} ({kb}: {path})" if label else f"({kb}: {path})"
text = LINK_RE.sub(link, text)
text = WIKILINK_RE.sub(lambda m: m.group(2) or m.group(1).rstrip("/").split("/")[-1], text)
text = "\n".join(PATH_LINE_RE.sub(lambda m: f"{m.group(1)}{kb}: `{m.group(2)}`", l) if "://" not in l else l
for l in text.split("\n"))
return text, dropped
def strip_private(text):
"""(text without <private> spans, whether any were found). An unclosed tag drops the rest."""
out = PRIVATE_RE.sub("", text)
out = PRIVATE_OPEN_RE.sub("", out)
return out, out != text
def strip_title(body):
lines = body.lstrip("\n").split("\n")
if lines and lines[0].startswith("# "):
return lines[0][2:].strip(), "\n".join(lines[1:]).lstrip("\n")
return "", body
# ----------------------------------------------------------------- commands
def cmd_join(a, root, ident, today):
cfg = Config(root)
kb = a.kb_name or cfg.kb_name or slugify(root.name)
if not NAME_RE.match(kb):
raise Fail(f"mailbox name {kb!r} must be lower-case letters, digits and hyphens")
if cfg.kb_name and a.kb_name and a.kb_name != cfg.kb_name:
raise Fail(f"this KB already uses mailbox name {cfg.kb_name!r} in ckb.yaml")
if not NAME_RE.match(a.channel):
raise Fail("channel name must be lower-case letters, digits and hyphens")
existing = next((c for c in cfg.channels if c["name"] == a.channel), None)
if existing and existing["url"] != a.url:
raise Fail(f"channel {a.channel!r} is already configured with {existing['url']}")
ch_cfg = existing or {"name": a.channel, "url": a.url, "branch": a.branch}
ch = Channel(root, ch_cfg, ident, today)
ch.pull()
changed = []
if not (ch.dir / "CHANNEL.md").exists():
(ch.dir / "CHANNEL.md").write_text(
f"# Share channel `{a.channel}`\n\n"
"A mailbox repo for Cascade KBs (the `share` module). Each KB writes only into\n"
"`mail/<its-name>/`, and files there are never edited or deleted: a new version or a\n"
"retraction is a new file. Everyone with access to this repo can read every mailbox,\n"
"so a channel serves one trust group only.\n\n"
"Owners of each mailbox are listed in `members.yaml`. A receiving KB checks that the\n"
"git author of every message is an owner of the mailbox it sits in.\n\n"
"## Purpose\n\n<!-- What this channel is for, and who may join. -->\n", encoding="utf-8")
changed.append("CHANNEL.md")
members_path = ch.dir / "members.yaml"
if not members_path.exists():
members_path.write_text(f"# Mailbox owners: git emails allowed to write mail/<kb>/.\nchannel: {a.channel}\nmembers:\n",
encoding="utf-8")
changed.append("members.yaml")
members = ch.members()
email = ident[1].lower()
note = ""
if kb not in members:
with members_path.open("a", encoding="utf-8-sig") as fh:
fh.write(f" {kb}:\n owners: [{ident[1]}]\n")
if "members.yaml" not in changed:
changed.append("members.yaml")
note = f"mailbox {kb} registered with owner {ident[1]}"
elif email not in members[kb]:
note = (f"mailbox {kb} exists and {ident[1]} is not one of its owners — receiving works, "
f"but sending needs an owner to add you in members.yaml")
else:
note = f"mailbox {kb} already registered, you are an owner"
keep = ch.dir / "mail" / kb / ".gitkeep"
if not keep.exists():
keep.parent.mkdir(parents=True, exist_ok=True)
keep.write_text("", encoding="utf-8")
changed.append(f"mail/{kb}/.gitkeep")
if changed:
commit = ch.commit_push(changed, f"Join {a.channel} as {kb}")
note += f"; pushed {', '.join(changed)} ({commit})"
cfg.kb_name = kb
if not existing:
cfg.channels.append(ch_cfg)
cfg.save()
print(f"joined {a.channel} as {kb}: {note}")
print(f"clone: {ch.dir.relative_to(root).as_posix()}")
return 0
def own_docs(ch, kb, share_id=None):
return [m for m in ch.messages(kb) if m["fm"].get("kind") == "document"
and (share_id is None or m["fm"].get("share_id") == share_id)]
def cmd_send(a, root, ident, today):
cfg = Config(root)
ch = Channel(root, cfg.channel(a.channel), ident, today)
kb = cfg.kb_name
src = (root / a.source).resolve() if not Path(a.source).is_absolute() else Path(a.source).resolve()
try:
rel = src.relative_to(root).as_posix()
except ValueError:
raise Fail(f"{a.source} is outside this KB")
if not src.is_file() or src.suffix.lower() != ".md":
raise Fail(f"{rel} is not a Markdown file")
if not rel.startswith(SEND_ROOTS):
if rel.startswith(RAW_ROOTS) and a.raw_ok:
pass
elif rel.startswith(RAW_ROOTS):
raise Fail(f"{rel} is a raw source, possibly someone else's material: confirm with the user, then add --raw-ok", 1)
else:
raise Fail(f"only outputs/documents/, outputs/answers/ and wiki/ pages can be shared (got {rel})", 1)
ch.pull(a.offline)
members = ch.members()
if kb not in members:
raise Fail(f"mailbox {kb} is not in the channel's members.yaml — run join first")
if ident[1].lower() not in members[kb]:
raise Fail(f"{ident[1]} is not an owner of mailbox {kb}; an owner must add you to members.yaml", 1)
to = [t.strip() for t in a.to.split(",") if t.strip()]
if not to:
raise Fail("--to needs at least one recipient (or *)")
for t in to:
if t != "*" and t not in members:
raise Fail(f"{t} is not a member of channel {ch.name} (members: {', '.join(sorted(members))})", 1)
if t == kb:
raise Fail("a KB does not send to itself", 1)
text = src.read_text(encoding="utf-8-sig", errors="replace")
fm = parse_fm(text)
_, body = split_frontmatter(text)
body, had_private = strip_private(body) # before the H1 is split off
h1, body = strip_title(body)
title, t_private = strip_private(str(a.title or fm.get("title") or h1 or src.stem))
tldr, d_private = strip_private(str(fm.get("tldr") or ""))
title = " ".join(title.split()) or src.stem
slug = slugify(a.slug or re.sub(r"^\d{4}-\d{2}-\d{2}[_-]?", "", src.stem))
share_id = f"{kb}/{slug}"
prev = own_docs(ch, kb, share_id)
version = max([int(m["fm"].get("version") or 0) for m in prev] or [0]) + 1
packed, dropped = pack_body(body, kb, a.keep_notes)
if (had_private or t_private or d_private) and "<private> spans" not in dropped:
dropped.insert(0, "<private> spans")
name, email = ident
fields = {
"type": "share", "kind": "document", "share_id": share_id, "version": version,
"from": kb, "to": to, "sent_by": f"{name} <{email}>", "sent_on": today, "title": title,
"tldr": " ".join(tldr.split()) or None, "origin": rel, "origin_type": fm.get("type") or None,
"supersedes_version": version - 1 if version > 1 else None, "untrusted": True,
}
banner = (f"> Shared from the **{kb}** knowledge base by {name} on {today} (version {version}). "
f"Links into {kb}'s own files are shown as text.")
doc = render(fields, f"# {title}\n\n{banner}\n\n{packed.strip()}\n")
hits = secret_hits(root, doc)
target = f"mail/{kb}/{today}_{slug}_v{version}.md"
preview = root / "tmp" / "share" / "outgoing" / f"{today}_{slug}_v{version}.md"
preview.parent.mkdir(parents=True, exist_ok=True)
preview.write_text(doc, encoding="utf-8")
print(f"channel: {ch.name} ({ch.url})")
print(f"from: {kb} → to: {', '.join(to)}")
print(f"document: {title!r} share_id {share_id} version {version}"
+ (f" (supersedes v{version - 1})" if version > 1 else ""))
print(f"source: {rel}")
print(f"dropped: {', '.join(dropped) if dropped else 'nothing'}")
print(f"preview: {preview.relative_to(root).as_posix()}")
about = root / "ABOUT.md"
if about.exists():
m = re.search(r"^##\s+Sensitivity\s*\n(.*?)(?=^##\s|\Z)", about.read_text(encoding="utf-8-sig"), re.S | re.M)
sens = re.sub(r"<!--.*?-->", "", m.group(1), flags=re.S).strip() if m else ""
print("ABOUT.md sensitivity: " + (" ".join(sens.split())[:400] if sens else "(not filled in)"))
if hits:
preview.unlink()
print("BLOCKED — possible secrets in the packed document (nothing sent, preview removed):")
for h in hits:
print(f" {h}")
return 3
if a.dry_run:
print(f"dry run: would write {target} in channel {ch.name}")
return 0
dest = ch.dir / target
if dest.exists():
raise Fail(f"{target} already exists in the channel")
dest.parent.mkdir(parents=True, exist_ok=True)
dest.write_text(doc, encoding="utf-8")
commit = ch.commit_push([target], f"{kb}: share {share_id} v{version} to {', '.join(to)}")
print(f"sent: {target} ({commit})")
return 0
def cmd_unshare(a, root, ident, today):
cfg = Config(root)
ch = Channel(root, cfg.channel(a.channel), ident, today)
kb = cfg.kb_name
ch.pull(a.offline)
share_id = a.share_id if "/" in a.share_id else f"{kb}/{a.share_id}"
if not share_id.startswith(kb + "/"):
raise Fail(f"{share_id} was not sent by this KB ({kb}); only the sender can retract it", 1)
docs = own_docs(ch, kb, share_id)
if not docs:
raise Fail(f"nothing with share_id {share_id} in mail/{kb}/", 1)
versions = sorted(int(m["fm"].get("version") or 0) for m in docs)
if a.version != "all" and int(a.version) not in versions:
raise Fail(f"{share_id} has versions {versions}, not {a.version}", 1)
to = sorted({t for m in docs for t in (m["fm"].get("to") or [])})
slug = share_id.split("/", 1)[1]
target = f"mail/{kb}/{today}_{slug}_retract.md"
n = 2
while (ch.dir / target).exists():
target, n = f"mail/{kb}/{today}_{slug}_retract-{n}.md", n + 1
fields = {"type": "share", "kind": "retraction", "share_id": share_id, "retracts_version": a.version,
"from": kb, "to": to, "sent_by": f"{ident[0]} <{ident[1]}>", "sent_on": today,
"title": f"Retract {share_id}", "reason": a.reason or None, "untrusted": True}
body = (f"# Retract `{share_id}` ({'all versions' if a.version == 'all' else 'version ' + a.version})\n\n"
+ (a.reason or "No reason given.") + "\n")
print(f"retract: {share_id} {'all versions' if a.version == 'all' else 'v' + a.version} → {', '.join(to)}")
print("note: the original stays in the channel's git history; receivers unwind it at their next receive")
if a.dry_run:
print(f"dry run: would write {target}")
return 0
(ch.dir / target).parent.mkdir(parents=True, exist_ok=True)
(ch.dir / target).write_text(render(fields, body), encoding="utf-8")
commit = ch.commit_push([target], f"{kb}: retract {share_id} {a.version}")
print(f"sent: {target} ({commit})")
return 0
def local_received(root):
"""Received share files still present here: [(rel path, fm)] from raw/inbox and raw/archive."""
out = []
for top in ("raw/inbox", "raw/archive"):
base = root / top
if not base.is_dir():
continue
for f in sorted(base.rglob("*.md")):
try:
with f.open(encoding="utf-8-sig", errors="replace") as fh:
head = fh.read(4096)
except OSError:
continue
if "type: share" not in head:
continue
fm = parse_fm(head if head.count("\n---\n") else f.read_text(encoding="utf-8-sig", errors="replace"))
if fm.get("type") == "share" and fm.get("share_channel"):
out.append((f.relative_to(root).as_posix(), fm))
return out
def covers(retraction_fm, version):
rv = str(retraction_fm.get("retracts_version") or "all")
return rv == "all" or rv == str(version)
def pending(root, cfg, ident, today, offline):
kb = cfg.kb_name
local = local_received(root)
have = {(fm.get("share_channel"), str(fm.get("share_ref", "")).split("@", 1)[0]) for _, fm in local}
items = []
for c in cfg.channels:
ch = Channel(root, c, ident, today)
ch.pull(offline)
members = ch.members()
msgs = [m for m in ch.messages() if m["box"] != kb]
# Only a retraction from the document's own mailbox, by its owners,
# may hide a document that was never taken.
retractions = [m for m in msgs if m["fm"].get("kind") == "retraction"
and base_item(ch, m, members)["verified"]]
for m in msgs:
fm = m["fm"]
to = fm.get("to") or []
to = to if isinstance(to, list) else [to]
if kb not in to and "*" not in to:
continue
if cfg.handled(ch.name, m["path"]) or (ch.name, m["path"]) in have:
continue
kind = fm.get("kind")
if kind == "document" and any(r["fm"].get("share_id") == fm.get("share_id") and covers(r["fm"], fm.get("version"))
for r in retractions):
items.append({**base_item(ch, m, members), "state": "retracted-before-taken"})
continue
if kind not in ("document", "retraction"):
continue
items.append({**base_item(ch, m, members), "state": "pending"})
return items, local
def base_item(ch, m, members):
m["authors"] = ch.authors(m["path"])
ok, why, author = verify(m, members)
fm = m["fm"]
return {"id": f"{ch.name}:{Path(m['path']).name}", "channel": ch.name, "path": m["path"],
"file": str(m["file"]), "from": fm.get("from"), "kind": fm.get("kind"),
"share_id": fm.get("share_id"), "version": fm.get("version") or fm.get("retracts_version"),
"title": fm.get("title"), "sent_on": fm.get("sent_on"), "author": author,
"verified": ok, "why": why, "reason": fm.get("reason")}
def describe(it):
v = f"v{it['version']}" if it["kind"] == "document" else f"retracts {it['version']}"
flag = "" if it["verified"] else f" UNVERIFIED: {it['why']}"
return f"{it['id']} [{it['kind']}] {it['share_id']} {v} — {it['title']!s} (from {it['from']}, {it['sent_on']}){flag}"
def cmd_list(a, root, ident, today):
cfg = Config(root)
items, _ = pending(root, cfg, ident, today, a.offline)
if a.json:
print(json.dumps(items, indent=2, ensure_ascii=False))
return 0 if items else 1
live = [i for i in items if i["state"] == "pending"]
for it in live:
print(describe(it))
gone = [i for i in items if i["state"] != "pending"]
for it in gone:
print(f"{it['id']} [document] {it['share_id']} v{it['version']} — retracted by its sender before it was taken")
print(f"share: {len(live)} waiting" + (f", {len(gone)} retracted before taken" if gone else ""))
return 0 if items else 1
def pick(spec, items):
if not spec:
return set()
if spec.strip() == "all":
return {i["id"] for i in items}
want = {s.strip() for s in spec.split(",") if s.strip()}
known = {i["id"] for i in items}
unknown = want - known
if unknown:
raise Fail(f"not waiting: {', '.join(sorted(unknown))} (run list)")
return want
def cmd_receive(a, root, ident, today):
cfg = Config(root)
items, local = pending(root, cfg, ident, today, a.offline)
take, decline, unverified_ok = pick(a.take, items), pick(a.decline, items), pick(a.accept_unverified, items)
if take & decline:
raise Fail("an item cannot be both taken and declined")
inbox = root / "raw" / "inbox"
results, skipped = [], []
for it in items:
if it["state"] != "pending":
cfg.received.append({"channel": it["channel"], "path": it["path"], "action": "seen", "date": today})
results.append(f"seen {it['id']} — retracted before it was taken, nothing to do")
continue
if it["id"] in decline:
cfg.received.append({"channel": it["channel"], "path": it["path"], "action": "declined", "date": today})
results.append(f"declined {it['id']}")
continue
if it["id"] not in take:
continue
if not it["verified"] and it["id"] not in unverified_ok:
skipped.append(f"skipped {it['id']} — UNVERIFIED: {it['why']} (take it only with --accept-unverified)")
continue
text = Path(it["file"]).read_text(encoding="utf-8-sig", errors="replace")
commit = git(["log", "-1", "--format=%h", "--", it["path"]], root / "tmp" / "share" / it["channel"]).stdout.strip()
if it["kind"] == "document":
fm, body = split_frontmatter(text)
earlier = sorted(((p, f) for p, f in local if f.get("share_channel") == it["channel"]
and f.get("share_id") == it["share_id"]
and int(f.get("version") or 0) < int(it["version"] or 0)),
key=lambda pf: int(pf[1].get("version") or 0))
extra = [fm_line("share_channel", it["channel"]),
fm_line("share_ref", f"{it['path']}@{commit}"),
fm_line("received_on", today),
fm_line("submitted_by", it["author"] or parse_fm(text).get("sent_by", "")),
fm_line("submitted_at", today)]
if earlier:
extra.append(fm_line("supersedes_source", earlier[-1][0]))
if not it["verified"]:
extra.append(fm_line("share_unverified", it["why"]))
slug = it["share_id"].split("/", 1)[-1]
dest = inbox / f"{today}_{it['from']}_{slug}_v{it['version']}.md"
if dest.resolve().parent != inbox.resolve():
raise Fail(f"refusing to write {dest}: outside raw/inbox/")
# The sender's frontmatter is data: received material is always
# untrusted here, and its review state is not ours to inherit.
kept = [l for l in fm.rstrip("\n").split("\n")
if not re.match(r"^(untrusted|review|reviewed_by|reviewed_on):", l)]
extra.append(fm_line("untrusted", True))
inbox.mkdir(parents=True, exist_ok=True)
dest.write_text("---\n" + "\n".join(kept) + "\n" + "\n".join(extra) + "\n---\n" + body, encoding="utf-8")
cfg.received.append({"channel": it["channel"], "path": it["path"], "action": "taken", "date": today})
note = f" (newer version of {earlier[-1][0]})" if earlier else ""
results.append(f"taken {it['id']} → {dest.relative_to(root).as_posix()}{note}")
else:
targets = [p for p, f in local if f.get("share_channel") == it["channel"]
and f.get("share_id") == it["share_id"] and covers({"retracts_version": it["version"]}, f.get("version"))]
rdir = root / "raw" / "retractions"
for t in targets:
rdir.mkdir(parents=True, exist_ok=True)
stem = today + "_" + slugify(re.sub(r"^\d{4}-\d{2}-\d{2}[_-]?", "", Path(t).stem))
path, n = rdir / f"{stem}.md", 2
while path.exists():
path, n = rdir / f"{stem}-{n}.md", n + 1
now = datetime.now(timezone.utc).astimezone().isoformat(timespec="seconds")
fields = {"type": "retraction-request", "target": t,
"requested_by": it["author"] or parse_fm(text).get("sent_by", ""),
"requested_at": now, "reason": it.get("reason") or "Retracted by its sender",
"via": f"share:{it['channel']}", "share_ref": f"{it['path']}@{commit}"}
if not it["verified"]:
fields["share_unverified"] = it["why"]
body = (f"# Retract `{t}`\n\nThe sender ({it['from']}) retracted `{it['share_id']}` "
f"through share channel `{it['channel']}`. Their reason, quoted:\n\n> "
+ str(it.get("reason") or "none given") + "\n")
path.write_text(render(fields, body), encoding="utf-8")
results.append(f"filed {it['id']} → {path.relative_to(root).as_posix()} (target {t})")
if not targets:
results.append(f"seen {it['id']} — nothing from {it['share_id']} is held here")
cfg.received.append({"channel": it["channel"], "path": it["path"],
"action": "filed" if targets else "seen", "date": today})
cfg.save()
for r in results + skipped:
print(r)
left = [i for i in items if i["state"] == "pending" and i["id"] not in take | decline]
if left:
print(f"still waiting: {len(left)}")
return 0 if results else 1
def cmd_status(a, root, ident, today):
cfg = Config(root)
kb = cfg.kb_name
for c in cfg.channels:
ch = Channel(root, c, ident, today)
ch.pull(a.offline)
print(f"channel {ch.name} ({ch.url}) as {kb}")
sent = ch.messages(kb)
members = ch.members()
if not sent:
print(" sent: nothing")
for m in sent:
fm = m["fm"]
m["authors"] = ch.authors(m["path"])
ok, why, _ = verify(m, members)
what = f"v{fm.get('version')}" if fm.get("kind") == "document" else f"retraction of {fm.get('retracts_version')}"
flag = "" if ok else f" NOT FROM AN OWNER: {why}"
print(f" sent {fm.get('sent_on')} {fm.get('share_id')} {what} → {', '.join(fm.get('to') or [])}{flag}")
items, _ = pending(root, cfg, ident, today, True)
print(f"share: {sum(1 for i in items if i['state'] == 'pending')} waiting")
return 0
def main(argv=None):
for stream in (sys.stdout, sys.stderr): # UTF-8 output even on a Windows code page
if hasattr(stream, "reconfigure"):
stream.reconfigure(encoding="utf-8", errors="replace")
ap = argparse.ArgumentParser(description="Send and receive documents between Cascade KBs through a git channel.")
ap.add_argument("--root", default=".")
ap.add_argument("--today", default=None)
ap.add_argument("--author", default=None, help='"Name <email>"; default: git config of the KB')
sub = ap.add_subparsers(dest="cmd", required=True)
p = sub.add_parser("join")
p.add_argument("--channel", required=True)
p.add_argument("--url", required=True)
p.add_argument("--branch", default="main")
p.add_argument("--kb-name", default=None)
p = sub.add_parser("send")
p.add_argument("source")
p.add_argument("--to", required=True)
p.add_argument("--channel", default=None)
p.add_argument("--slug", default=None)
p.add_argument("--title", default=None)
p.add_argument("--keep-notes", action="store_true")
p.add_argument("--raw-ok", action="store_true")
p.add_argument("--dry-run", action="store_true")
p.add_argument("--offline", action="store_true")
p = sub.add_parser("unshare")
p.add_argument("share_id")
p.add_argument("--version", default="all")
p.add_argument("--reason", default="")
p.add_argument("--channel", default=None)
p.add_argument("--dry-run", action="store_true")
p.add_argument("--offline", action="store_true")
p = sub.add_parser("list")
p.add_argument("--offline", action="store_true")
p.add_argument("--json", action="store_true")
p = sub.add_parser("receive")
p.add_argument("--take", default="")
p.add_argument("--decline", default="")
p.add_argument("--accept-unverified", default="")
p.add_argument("--offline", action="store_true")
p = sub.add_parser("status")
p.add_argument("--offline", action="store_true")
a = ap.parse_args(argv)
root = Path(a.root).resolve()
if not (root / "wiki").is_dir():
print(f"error: {root} has no wiki/ — not a Cascade KB root", file=sys.stderr)
return 2
today = a.today or date.today().isoformat()
try:
ident = identity(root, a.author)
if a.cmd != "join" and not Config(root).kb_name:
raise Fail("no share channel configured — join one first: share.py join --channel NAME --url URL")
return {"join": cmd_join, "send": cmd_send, "unshare": cmd_unshare, "list": cmd_list,
"receive": cmd_receive, "status": cmd_status}[a.cmd](a, root, ident, today)
except Fail as e:
print(f"error: {e}", file=sys.stderr)
return e.code
if __name__ == "__main__":
sys.exit(main())