ckb/.agents/skills/ckb-ingest/scripts/dedup_inbox.py
Michał Kopeć dad2264a65 Template 1.26.0: release 3 batch 3, capture module, skill evals, multi-agent docs
- F11 new opt-in module `capture` (skill ckb-capture, capture_url.py):
  stdlib fetch with readable-text extraction, Wayback only over a failed
  or thinner live page, --from-file for the agent's own fetch, PDFs saved
  as files, --media via yt-dlp with consent, untrusted: true marking,
  dedup at capture time; dedup_inbox.py now compares source_url (D-0017)
- F22 tests/run_evals.py (list/prepare/check/validate) and five
  structural eval cases; reference run for ckb-decide/record
- F18 manual section with untested setups for Codex CLI, Cursor,
  OpenCode and Gemini CLI, plus the Claude Code-only feature table;
  AGENTS.md tells agents without skill loading to read SKILL.md

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-02 20:44:05 +01:00

234 lines
9.3 KiB
Python

#!/usr/bin/env python3
"""Step 1 of ckb-ingest: which inbox files has this KB already ingested?
dedup_inbox.py [--root DIR] [--json]
Read-only. For every file in raw/inbox/ it reports, at most once per file:
DUPLICATE byte-identical to a file in raw/archive/<date>/ (ingested on that date)
SAME TEXT same text as an archived file once whitespace and line endings are ignored
CITED its sha256 prefix is recorded in a wiki page's `## Sources` or `source_fingerprint`
SAME URL a link file (.url, .webloc, or a text file holding only a URL), or a
ckb-capture snapshot (`source_url:` frontmatter), whose normalised
URL a wiki page already cites or an archived capture already holds
INBOX TWIN byte-identical to another inbox file
and lists the wiki pages built from the match, so the agent can say "already
ingested on X → pages" and ask before skipping. Exact hashes miss
near-duplicates such as a re-exported PDF; that stays a judgment call.
Exit 0 when nothing in the inbox is a duplicate, 1 when something is.
Python 3.8+, standard library only.
"""
import argparse
import hashlib
import json
import re
import sys
from pathlib import Path
from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
TEXT_SUFFIXES = {".md", ".txt", ".eml", ".csv", ".json", ".yaml", ".yml", ".html", ".rst", ".log"}
URL_RE = re.compile(r"https?://[^\s`<>\"')\]]+")
TRACKING = re.compile(r"^(utm_.*|fbclid|gclid|mc_cid|mc_eid|ref|ref_src)$", re.I)
FP_RE = re.compile(r"\bsha256:([0-9a-fA-F]{6,64})")
SOURCE_BULLET_RE = re.compile(r"^\s*[-*]\s+`([^`]+)`(.*)$")
def sha256(path):
return hashlib.sha256(path.read_bytes()).hexdigest()
def text_digest(path):
"""Digest of the text with all whitespace collapsed, or None for non-text."""
if path.suffix.lower() not in TEXT_SUFFIXES:
return None
try:
text = path.read_text(encoding="utf-8", errors="replace")
except OSError:
return None
return hashlib.sha256(" ".join(text.split()).encode("utf-8")).hexdigest()
def normalise_url(url):
"""Lower-case scheme and host, drop the fragment, tracking parameters,
`www.` and a trailing slash, so the same page cited two ways compares equal."""
try:
parts = urlsplit(url.strip().rstrip(".,;"))
except ValueError:
return url
host = parts.netloc.lower()
host = host[4:] if host.startswith("www.") else host
query = urlencode([(k, v) for k, v in parse_qsl(parts.query, keep_blank_values=True) if not TRACKING.match(k)])
return urlunsplit((parts.scheme.lower(), host, parts.path.rstrip("/") or "/", query, ""))
def link_url(path):
"""The URL a link file points at, or None."""
try:
text = path.read_text(encoding="utf-8", errors="replace")
except OSError:
return None
suffix = path.suffix.lower()
captured = frontmatter_value(text, "source_url") if suffix == ".md" else ""
if captured:
return captured
if suffix == ".url":
m = re.search(r"(?im)^URL=(\S+)", text)
return m.group(1) if m else None
if suffix == ".webloc":
m = re.search(r"<string>(https?://[^<]+)</string>", text)
return m.group(1) if m else None
if suffix in TEXT_SUFFIXES and len(text) < 2048:
found = URL_RE.findall(text)
rest = URL_RE.sub("", text).strip()
if len(found) == 1 and not rest:
return found[0]
return None
def files_under(base):
if not base.is_dir():
return []
return [p for p in sorted(base.rglob("*"))
if p.is_file() and not any(part.startswith(".") for part in p.relative_to(base).parts)]
def frontmatter_value(text, key):
if not text.startswith("---\n"):
return ""
end = text.find("\n---\n", 4)
for line in text[4:end if end != -1 else 0].split("\n"):
if line.startswith(f"{key}:"):
return line.split(":", 1)[1].strip().strip("\"'")
return ""
def wiki_citations(root):
"""({sha256 prefix: [pages]}, {normalised url: [pages]}, {cited path: [pages]})."""
by_fp, by_url, by_path = {}, {}, {}
for page in sorted((root / "wiki").rglob("*.md")):
rel = page.relative_to(root).as_posix()
text = page.read_text(encoding="utf-8", errors="replace")
for key in ("source_fingerprint", "resource"):
value = frontmatter_value(text, key)
m = FP_RE.search(value)
if m:
by_fp.setdefault(m.group(1).lower(), []).append(rel)
if URL_RE.match(value):
by_url.setdefault(normalise_url(value), []).append(rel)
in_sources = False
for line in text.split("\n"):
if line.startswith("## "):
in_sources = line.strip() == "## Sources"
continue
if not in_sources:
continue
b = SOURCE_BULLET_RE.match(line)
if not b:
continue
cited, rest = b.group(1).strip(), b.group(2)
if URL_RE.match(cited):
by_url.setdefault(normalise_url(cited), []).append(rel)
else:
by_path.setdefault(cited.lstrip("/"), []).append(rel)
m = FP_RE.search(rest)
if m:
by_fp.setdefault(m.group(1).lower(), []).append(rel)
return by_fp, by_url, by_path
def fp_pages(by_fp, full):
pages = set()
for prefix, rels in by_fp.items():
if full.startswith(prefix):
pages.update(rels)
return sorted(pages)
def archive_date(rel):
m = re.match(r"raw/archive/(\d{4}-\d{2}-\d{2})/", rel)
return m.group(1) if m else "an unknown date"
def scan(root):
inbox = files_under(root / "raw" / "inbox")
archive = files_under(root / "raw" / "archive")
by_fp, by_url, by_path = wiki_citations(root)
arch_hash, arch_text, arch_url = {}, {}, {}
for f in archive:
rel = f.relative_to(root).as_posix()
arch_hash.setdefault(sha256(f), rel)
url = link_url(f) if f.suffix.lower() == ".md" else None
if url:
arch_url.setdefault(normalise_url(url), rel)
td = text_digest(f)
if td:
arch_text.setdefault(td, rel)
hits, fresh, seen, seen_url = [], [], {}, {}
for f in inbox:
rel = f.relative_to(root).as_posix()
full = sha256(f)
hit = None
if full in arch_hash:
match = arch_hash[full]
hit = {"kind": "DUPLICATE", "file": rel, "match": match,
"detail": f"identical to {match} (ingested {archive_date(match)})",
"pages": sorted(set(by_path.get(match, [])) | set(fp_pages(by_fp, full)))}
elif text_digest(f) and text_digest(f) in arch_text:
match = arch_text[text_digest(f)]
hit = {"kind": "SAME TEXT", "file": rel, "match": match,
"detail": f"same text as {match} (ingested {archive_date(match)}; only whitespace or line endings differ)",
"pages": sorted(set(by_path.get(match, [])))}
elif fp_pages(by_fp, full):
hit = {"kind": "CITED", "file": rel, "match": f"sha256:{full[:8]}",
"detail": f"sha256:{full[:8]} is already cited as a source", "pages": fp_pages(by_fp, full)}
else:
url = link_url(f)
nurl = normalise_url(url) if url else None
if nurl and nurl in by_url:
hit = {"kind": "SAME URL", "file": rel, "match": url,
"detail": f"{url} is already cited as a source", "pages": sorted(set(by_url[nurl]))}
elif nurl and nurl in arch_url:
hit = {"kind": "SAME URL", "file": rel, "match": url,
"detail": f"{url} was captured before as {arch_url[nurl]} (ingested {archive_date(arch_url[nurl])})",
"pages": sorted(set(by_path.get(arch_url[nurl], [])))}
elif nurl and nurl in seen_url:
hit = {"kind": "INBOX TWIN", "file": rel, "match": seen_url[nurl],
"detail": f"same URL as {seen_url[nurl]}, also in the inbox", "pages": []}
elif full in seen:
hit = {"kind": "INBOX TWIN", "file": rel, "match": seen[full],
"detail": f"identical to {seen[full]}, also in the inbox", "pages": []}
seen.setdefault(full, rel)
url = link_url(f)
if url:
seen_url.setdefault(normalise_url(url), rel)
if hit:
hits.append(hit)
else:
fresh.append(rel)
return hits, fresh
def main(argv=None):
ap = argparse.ArgumentParser(description="Report inbox files this KB has already ingested.")
ap.add_argument("--root", default=".", help="KB root (default: current directory)")
ap.add_argument("--json", action="store_true")
a = ap.parse_args(argv)
root = Path(a.root).resolve()
if not (root / "wiki").is_dir():
sys.exit(f"error: {root} has no wiki/ — not a Cascade KB root")
hits, fresh = scan(root)
if a.json:
print(json.dumps({"duplicates": hits, "new": fresh}, indent=2))
return 1 if hits else 0
for h in hits:
pages = f" → {', '.join(h['pages'])}" if h["pages"] else ""
print(f"{h['kind']:<10} {h['file']} — {h['detail']}{pages}")
print(f"inbox: {len(fresh)} new, {len(hits)} already ingested or duplicated")
return 1 if hits else 0
if __name__ == "__main__":
sys.exit(main())