#!/usr/bin/env python3 """Step 1 of ckb-ingest: which inbox files has this KB already ingested? dedup_inbox.py [--root DIR] [--json] Read-only. For every file in raw/inbox/ it reports, at most once per file: DUPLICATE byte-identical to a file in raw/archive// (ingested on that date) SAME TEXT same text as an archived file once whitespace and line endings are ignored CITED its sha256 prefix is recorded in a wiki page's `## Sources` or `source_fingerprint` SAME URL a link file (.url, .webloc, or a text file holding only a URL) whose normalised URL a wiki page already cites INBOX TWIN byte-identical to another inbox file and lists the wiki pages built from the match, so the agent can say "already ingested on X → pages" and ask before skipping. Exact hashes miss near-duplicates such as a re-exported PDF; that stays a judgment call. Exit 0 when nothing in the inbox is a duplicate, 1 when something is. Python 3.8+, standard library only. """ import argparse import hashlib import json import re import sys from pathlib import Path from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit TEXT_SUFFIXES = {".md", ".txt", ".eml", ".csv", ".json", ".yaml", ".yml", ".html", ".rst", ".log"} URL_RE = re.compile(r"https?://[^\s`<>\"')\]]+") TRACKING = re.compile(r"^(utm_.*|fbclid|gclid|mc_cid|mc_eid|ref|ref_src)$", re.I) FP_RE = re.compile(r"\bsha256:([0-9a-fA-F]{6,64})") SOURCE_BULLET_RE = re.compile(r"^\s*[-*]\s+`([^`]+)`(.*)$") def sha256(path): return hashlib.sha256(path.read_bytes()).hexdigest() def text_digest(path): """Digest of the text with all whitespace collapsed, or None for non-text.""" if path.suffix.lower() not in TEXT_SUFFIXES: return None try: text = path.read_text(encoding="utf-8", errors="replace") except OSError: return None return hashlib.sha256(" ".join(text.split()).encode("utf-8")).hexdigest() def normalise_url(url): """Lower-case scheme and host, drop the fragment, tracking parameters, `www.` and a trailing slash, so the same page cited two ways compares equal.""" try: parts = urlsplit(url.strip().rstrip(".,;")) except ValueError: return url host = parts.netloc.lower() host = host[4:] if host.startswith("www.") else host query = urlencode([(k, v) for k, v in parse_qsl(parts.query, keep_blank_values=True) if not TRACKING.match(k)]) return urlunsplit((parts.scheme.lower(), host, parts.path.rstrip("/") or "/", query, "")) def link_url(path): """The URL a link file points at, or None.""" try: text = path.read_text(encoding="utf-8", errors="replace") except OSError: return None suffix = path.suffix.lower() if suffix == ".url": m = re.search(r"(?im)^URL=(\S+)", text) return m.group(1) if m else None if suffix == ".webloc": m = re.search(r"(https?://[^<]+)", text) return m.group(1) if m else None if suffix in TEXT_SUFFIXES and len(text) < 2048: found = URL_RE.findall(text) rest = URL_RE.sub("", text).strip() if len(found) == 1 and not rest: return found[0] return None def files_under(base): if not base.is_dir(): return [] return [p for p in sorted(base.rglob("*")) if p.is_file() and not any(part.startswith(".") for part in p.relative_to(base).parts)] def frontmatter_value(text, key): if not text.startswith("---\n"): return "" end = text.find("\n---\n", 4) for line in text[4:end if end != -1 else 0].split("\n"): if line.startswith(f"{key}:"): return line.split(":", 1)[1].strip().strip("\"'") return "" def wiki_citations(root): """({sha256 prefix: [pages]}, {normalised url: [pages]}, {cited path: [pages]}).""" by_fp, by_url, by_path = {}, {}, {} for page in sorted((root / "wiki").rglob("*.md")): rel = page.relative_to(root).as_posix() text = page.read_text(encoding="utf-8", errors="replace") for key in ("source_fingerprint", "resource"): value = frontmatter_value(text, key) m = FP_RE.search(value) if m: by_fp.setdefault(m.group(1).lower(), []).append(rel) if URL_RE.match(value): by_url.setdefault(normalise_url(value), []).append(rel) in_sources = False for line in text.split("\n"): if line.startswith("## "): in_sources = line.strip() == "## Sources" continue if not in_sources: continue b = SOURCE_BULLET_RE.match(line) if not b: continue cited, rest = b.group(1).strip(), b.group(2) if URL_RE.match(cited): by_url.setdefault(normalise_url(cited), []).append(rel) else: by_path.setdefault(cited.lstrip("/"), []).append(rel) m = FP_RE.search(rest) if m: by_fp.setdefault(m.group(1).lower(), []).append(rel) return by_fp, by_url, by_path def fp_pages(by_fp, full): pages = set() for prefix, rels in by_fp.items(): if full.startswith(prefix): pages.update(rels) return sorted(pages) def archive_date(rel): m = re.match(r"raw/archive/(\d{4}-\d{2}-\d{2})/", rel) return m.group(1) if m else "an unknown date" def scan(root): inbox = files_under(root / "raw" / "inbox") archive = files_under(root / "raw" / "archive") by_fp, by_url, by_path = wiki_citations(root) arch_hash, arch_text = {}, {} for f in archive: rel = f.relative_to(root).as_posix() arch_hash.setdefault(sha256(f), rel) td = text_digest(f) if td: arch_text.setdefault(td, rel) hits, fresh, seen = [], [], {} for f in inbox: rel = f.relative_to(root).as_posix() full = sha256(f) hit = None if full in arch_hash: match = arch_hash[full] hit = {"kind": "DUPLICATE", "file": rel, "match": match, "detail": f"identical to {match} (ingested {archive_date(match)})", "pages": sorted(set(by_path.get(match, [])) | set(fp_pages(by_fp, full)))} elif text_digest(f) and text_digest(f) in arch_text: match = arch_text[text_digest(f)] hit = {"kind": "SAME TEXT", "file": rel, "match": match, "detail": f"same text as {match} (ingested {archive_date(match)}; only whitespace or line endings differ)", "pages": sorted(set(by_path.get(match, [])))} elif fp_pages(by_fp, full): hit = {"kind": "CITED", "file": rel, "match": f"sha256:{full[:8]}", "detail": f"sha256:{full[:8]} is already cited as a source", "pages": fp_pages(by_fp, full)} else: url = link_url(f) if url and normalise_url(url) in by_url: hit = {"kind": "SAME URL", "file": rel, "match": url, "detail": f"{url} is already cited as a source", "pages": sorted(set(by_url[normalise_url(url)]))} elif full in seen: hit = {"kind": "INBOX TWIN", "file": rel, "match": seen[full], "detail": f"identical to {seen[full]}, also in the inbox", "pages": []} seen.setdefault(full, rel) if hit: hits.append(hit) else: fresh.append(rel) return hits, fresh def main(argv=None): ap = argparse.ArgumentParser(description="Report inbox files this KB has already ingested.") ap.add_argument("--root", default=".", help="KB root (default: current directory)") ap.add_argument("--json", action="store_true") a = ap.parse_args(argv) root = Path(a.root).resolve() if not (root / "wiki").is_dir(): sys.exit(f"error: {root} has no wiki/ — not a Cascade KB root") hits, fresh = scan(root) if a.json: print(json.dumps({"duplicates": hits, "new": fresh}, indent=2)) return 1 if hits else 0 for h in hits: pages = f" → {', '.join(h['pages'])}" if h["pages"] else "" print(f"{h['kind']:<10} {h['file']} — {h['detail']}{pages}") print(f"inbox: {len(fresh)} new, {len(hits)} already ingested or duplicated") return 1 if hits else 0 if __name__ == "__main__": sys.exit(main())