- F11 new opt-in module `capture` (skill ckb-capture, capture_url.py): stdlib fetch with readable-text extraction, Wayback only over a failed or thinner live page, --from-file for the agent's own fetch, PDFs saved as files, --media via yt-dlp with consent, untrusted: true marking, dedup at capture time; dedup_inbox.py now compares source_url (D-0017) - F22 tests/run_evals.py (list/prepare/check/validate) and five structural eval cases; reference run for ckb-decide/record - F18 manual section with untested setups for Codex CLI, Cursor, OpenCode and Gemini CLI, plus the Claude Code-only feature table; AGENTS.md tells agents without skill loading to read SKILL.md Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
234 lines
9.3 KiB
Python
234 lines
9.3 KiB
Python
#!/usr/bin/env python3
|
|
"""Step 1 of ckb-ingest: which inbox files has this KB already ingested?
|
|
|
|
dedup_inbox.py [--root DIR] [--json]
|
|
|
|
Read-only. For every file in raw/inbox/ it reports, at most once per file:
|
|
|
|
DUPLICATE byte-identical to a file in raw/archive/<date>/ (ingested on that date)
|
|
SAME TEXT same text as an archived file once whitespace and line endings are ignored
|
|
CITED its sha256 prefix is recorded in a wiki page's `## Sources` or `source_fingerprint`
|
|
SAME URL a link file (.url, .webloc, or a text file holding only a URL), or a
|
|
ckb-capture snapshot (`source_url:` frontmatter), whose normalised
|
|
URL a wiki page already cites or an archived capture already holds
|
|
INBOX TWIN byte-identical to another inbox file
|
|
|
|
and lists the wiki pages built from the match, so the agent can say "already
|
|
ingested on X → pages" and ask before skipping. Exact hashes miss
|
|
near-duplicates such as a re-exported PDF; that stays a judgment call.
|
|
|
|
Exit 0 when nothing in the inbox is a duplicate, 1 when something is.
|
|
Python 3.8+, standard library only.
|
|
"""
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
import re
|
|
import sys
|
|
from pathlib import Path
|
|
from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
|
|
|
|
TEXT_SUFFIXES = {".md", ".txt", ".eml", ".csv", ".json", ".yaml", ".yml", ".html", ".rst", ".log"}
|
|
URL_RE = re.compile(r"https?://[^\s`<>\"')\]]+")
|
|
TRACKING = re.compile(r"^(utm_.*|fbclid|gclid|mc_cid|mc_eid|ref|ref_src)$", re.I)
|
|
FP_RE = re.compile(r"\bsha256:([0-9a-fA-F]{6,64})")
|
|
SOURCE_BULLET_RE = re.compile(r"^\s*[-*]\s+`([^`]+)`(.*)$")
|
|
|
|
|
|
def sha256(path):
|
|
return hashlib.sha256(path.read_bytes()).hexdigest()
|
|
|
|
|
|
def text_digest(path):
|
|
"""Digest of the text with all whitespace collapsed, or None for non-text."""
|
|
if path.suffix.lower() not in TEXT_SUFFIXES:
|
|
return None
|
|
try:
|
|
text = path.read_text(encoding="utf-8", errors="replace")
|
|
except OSError:
|
|
return None
|
|
return hashlib.sha256(" ".join(text.split()).encode("utf-8")).hexdigest()
|
|
|
|
|
|
def normalise_url(url):
|
|
"""Lower-case scheme and host, drop the fragment, tracking parameters,
|
|
`www.` and a trailing slash, so the same page cited two ways compares equal."""
|
|
try:
|
|
parts = urlsplit(url.strip().rstrip(".,;"))
|
|
except ValueError:
|
|
return url
|
|
host = parts.netloc.lower()
|
|
host = host[4:] if host.startswith("www.") else host
|
|
query = urlencode([(k, v) for k, v in parse_qsl(parts.query, keep_blank_values=True) if not TRACKING.match(k)])
|
|
return urlunsplit((parts.scheme.lower(), host, parts.path.rstrip("/") or "/", query, ""))
|
|
|
|
|
|
def link_url(path):
|
|
"""The URL a link file points at, or None."""
|
|
try:
|
|
text = path.read_text(encoding="utf-8", errors="replace")
|
|
except OSError:
|
|
return None
|
|
suffix = path.suffix.lower()
|
|
captured = frontmatter_value(text, "source_url") if suffix == ".md" else ""
|
|
if captured:
|
|
return captured
|
|
if suffix == ".url":
|
|
m = re.search(r"(?im)^URL=(\S+)", text)
|
|
return m.group(1) if m else None
|
|
if suffix == ".webloc":
|
|
m = re.search(r"<string>(https?://[^<]+)</string>", text)
|
|
return m.group(1) if m else None
|
|
if suffix in TEXT_SUFFIXES and len(text) < 2048:
|
|
found = URL_RE.findall(text)
|
|
rest = URL_RE.sub("", text).strip()
|
|
if len(found) == 1 and not rest:
|
|
return found[0]
|
|
return None
|
|
|
|
|
|
def files_under(base):
|
|
if not base.is_dir():
|
|
return []
|
|
return [p for p in sorted(base.rglob("*"))
|
|
if p.is_file() and not any(part.startswith(".") for part in p.relative_to(base).parts)]
|
|
|
|
|
|
def frontmatter_value(text, key):
|
|
if not text.startswith("---\n"):
|
|
return ""
|
|
end = text.find("\n---\n", 4)
|
|
for line in text[4:end if end != -1 else 0].split("\n"):
|
|
if line.startswith(f"{key}:"):
|
|
return line.split(":", 1)[1].strip().strip("\"'")
|
|
return ""
|
|
|
|
|
|
def wiki_citations(root):
|
|
"""({sha256 prefix: [pages]}, {normalised url: [pages]}, {cited path: [pages]})."""
|
|
by_fp, by_url, by_path = {}, {}, {}
|
|
for page in sorted((root / "wiki").rglob("*.md")):
|
|
rel = page.relative_to(root).as_posix()
|
|
text = page.read_text(encoding="utf-8", errors="replace")
|
|
for key in ("source_fingerprint", "resource"):
|
|
value = frontmatter_value(text, key)
|
|
m = FP_RE.search(value)
|
|
if m:
|
|
by_fp.setdefault(m.group(1).lower(), []).append(rel)
|
|
if URL_RE.match(value):
|
|
by_url.setdefault(normalise_url(value), []).append(rel)
|
|
in_sources = False
|
|
for line in text.split("\n"):
|
|
if line.startswith("## "):
|
|
in_sources = line.strip() == "## Sources"
|
|
continue
|
|
if not in_sources:
|
|
continue
|
|
b = SOURCE_BULLET_RE.match(line)
|
|
if not b:
|
|
continue
|
|
cited, rest = b.group(1).strip(), b.group(2)
|
|
if URL_RE.match(cited):
|
|
by_url.setdefault(normalise_url(cited), []).append(rel)
|
|
else:
|
|
by_path.setdefault(cited.lstrip("/"), []).append(rel)
|
|
m = FP_RE.search(rest)
|
|
if m:
|
|
by_fp.setdefault(m.group(1).lower(), []).append(rel)
|
|
return by_fp, by_url, by_path
|
|
|
|
|
|
def fp_pages(by_fp, full):
|
|
pages = set()
|
|
for prefix, rels in by_fp.items():
|
|
if full.startswith(prefix):
|
|
pages.update(rels)
|
|
return sorted(pages)
|
|
|
|
|
|
def archive_date(rel):
|
|
m = re.match(r"raw/archive/(\d{4}-\d{2}-\d{2})/", rel)
|
|
return m.group(1) if m else "an unknown date"
|
|
|
|
|
|
def scan(root):
|
|
inbox = files_under(root / "raw" / "inbox")
|
|
archive = files_under(root / "raw" / "archive")
|
|
by_fp, by_url, by_path = wiki_citations(root)
|
|
arch_hash, arch_text, arch_url = {}, {}, {}
|
|
for f in archive:
|
|
rel = f.relative_to(root).as_posix()
|
|
arch_hash.setdefault(sha256(f), rel)
|
|
url = link_url(f) if f.suffix.lower() == ".md" else None
|
|
if url:
|
|
arch_url.setdefault(normalise_url(url), rel)
|
|
td = text_digest(f)
|
|
if td:
|
|
arch_text.setdefault(td, rel)
|
|
|
|
hits, fresh, seen, seen_url = [], [], {}, {}
|
|
for f in inbox:
|
|
rel = f.relative_to(root).as_posix()
|
|
full = sha256(f)
|
|
hit = None
|
|
if full in arch_hash:
|
|
match = arch_hash[full]
|
|
hit = {"kind": "DUPLICATE", "file": rel, "match": match,
|
|
"detail": f"identical to {match} (ingested {archive_date(match)})",
|
|
"pages": sorted(set(by_path.get(match, [])) | set(fp_pages(by_fp, full)))}
|
|
elif text_digest(f) and text_digest(f) in arch_text:
|
|
match = arch_text[text_digest(f)]
|
|
hit = {"kind": "SAME TEXT", "file": rel, "match": match,
|
|
"detail": f"same text as {match} (ingested {archive_date(match)}; only whitespace or line endings differ)",
|
|
"pages": sorted(set(by_path.get(match, [])))}
|
|
elif fp_pages(by_fp, full):
|
|
hit = {"kind": "CITED", "file": rel, "match": f"sha256:{full[:8]}",
|
|
"detail": f"sha256:{full[:8]} is already cited as a source", "pages": fp_pages(by_fp, full)}
|
|
else:
|
|
url = link_url(f)
|
|
nurl = normalise_url(url) if url else None
|
|
if nurl and nurl in by_url:
|
|
hit = {"kind": "SAME URL", "file": rel, "match": url,
|
|
"detail": f"{url} is already cited as a source", "pages": sorted(set(by_url[nurl]))}
|
|
elif nurl and nurl in arch_url:
|
|
hit = {"kind": "SAME URL", "file": rel, "match": url,
|
|
"detail": f"{url} was captured before as {arch_url[nurl]} (ingested {archive_date(arch_url[nurl])})",
|
|
"pages": sorted(set(by_path.get(arch_url[nurl], [])))}
|
|
elif nurl and nurl in seen_url:
|
|
hit = {"kind": "INBOX TWIN", "file": rel, "match": seen_url[nurl],
|
|
"detail": f"same URL as {seen_url[nurl]}, also in the inbox", "pages": []}
|
|
elif full in seen:
|
|
hit = {"kind": "INBOX TWIN", "file": rel, "match": seen[full],
|
|
"detail": f"identical to {seen[full]}, also in the inbox", "pages": []}
|
|
seen.setdefault(full, rel)
|
|
url = link_url(f)
|
|
if url:
|
|
seen_url.setdefault(normalise_url(url), rel)
|
|
if hit:
|
|
hits.append(hit)
|
|
else:
|
|
fresh.append(rel)
|
|
return hits, fresh
|
|
|
|
|
|
def main(argv=None):
|
|
ap = argparse.ArgumentParser(description="Report inbox files this KB has already ingested.")
|
|
ap.add_argument("--root", default=".", help="KB root (default: current directory)")
|
|
ap.add_argument("--json", action="store_true")
|
|
a = ap.parse_args(argv)
|
|
root = Path(a.root).resolve()
|
|
if not (root / "wiki").is_dir():
|
|
sys.exit(f"error: {root} has no wiki/ — not a Cascade KB root")
|
|
hits, fresh = scan(root)
|
|
if a.json:
|
|
print(json.dumps({"duplicates": hits, "new": fresh}, indent=2))
|
|
return 1 if hits else 0
|
|
for h in hits:
|
|
pages = f" → {', '.join(h['pages'])}" if h["pages"] else ""
|
|
print(f"{h['kind']:<10} {h['file']} — {h['detail']}{pages}")
|
|
print(f"inbox: {len(fresh)} new, {len(hits)} already ingested or duplicated")
|
|
return 1 if hits else 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|