Seven features from the 2026-10-31 agent-knowledge research, no schema change (kb_schema_version stays 1.5): - F1 session-start hook: session_start.py --hook, start_hook.py, opt-in via session.start_hook in ckb.yaml (D-0011) - F2 kept answers in outputs/answers/, lint check 16 (D-0013) - F3 ingest check-in before writing; batch mode skips it - F5 open threads in workload notes, collected at session start (D-0012) - F6 dedup_inbox.py: duplicates reported before ingest - F19 lint check 15 for secrets and <private> spans, --secrets before every sync commit, redaction in saved conversations (D-0013) - F20 context budget: lint_report.py --budget, budget in ckb.yaml Smoke test covers every new script. Docs updated in EN and PL. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
216 lines
8.3 KiB
Python
216 lines
8.3 KiB
Python
#!/usr/bin/env python3
|
|
"""Step 1 of ckb-ingest: which inbox files has this KB already ingested?
|
|
|
|
dedup_inbox.py [--root DIR] [--json]
|
|
|
|
Read-only. For every file in raw/inbox/ it reports, at most once per file:
|
|
|
|
DUPLICATE byte-identical to a file in raw/archive/<date>/ (ingested on that date)
|
|
SAME TEXT same text as an archived file once whitespace and line endings are ignored
|
|
CITED its sha256 prefix is recorded in a wiki page's `## Sources` or `source_fingerprint`
|
|
SAME URL a link file (.url, .webloc, or a text file holding only a URL) whose
|
|
normalised URL a wiki page already cites
|
|
INBOX TWIN byte-identical to another inbox file
|
|
|
|
and lists the wiki pages built from the match, so the agent can say "already
|
|
ingested on X → pages" and ask before skipping. Exact hashes miss
|
|
near-duplicates such as a re-exported PDF; that stays a judgment call.
|
|
|
|
Exit 0 when nothing in the inbox is a duplicate, 1 when something is.
|
|
Python 3.8+, standard library only.
|
|
"""
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
import re
|
|
import sys
|
|
from pathlib import Path
|
|
from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
|
|
|
|
TEXT_SUFFIXES = {".md", ".txt", ".eml", ".csv", ".json", ".yaml", ".yml", ".html", ".rst", ".log"}
|
|
URL_RE = re.compile(r"https?://[^\s`<>\"')\]]+")
|
|
TRACKING = re.compile(r"^(utm_.*|fbclid|gclid|mc_cid|mc_eid|ref|ref_src)$", re.I)
|
|
FP_RE = re.compile(r"\bsha256:([0-9a-fA-F]{6,64})")
|
|
SOURCE_BULLET_RE = re.compile(r"^\s*[-*]\s+`([^`]+)`(.*)$")
|
|
|
|
|
|
def sha256(path):
|
|
return hashlib.sha256(path.read_bytes()).hexdigest()
|
|
|
|
|
|
def text_digest(path):
|
|
"""Digest of the text with all whitespace collapsed, or None for non-text."""
|
|
if path.suffix.lower() not in TEXT_SUFFIXES:
|
|
return None
|
|
try:
|
|
text = path.read_text(encoding="utf-8", errors="replace")
|
|
except OSError:
|
|
return None
|
|
return hashlib.sha256(" ".join(text.split()).encode("utf-8")).hexdigest()
|
|
|
|
|
|
def normalise_url(url):
|
|
"""Lower-case scheme and host, drop the fragment, tracking parameters,
|
|
`www.` and a trailing slash, so the same page cited two ways compares equal."""
|
|
try:
|
|
parts = urlsplit(url.strip().rstrip(".,;"))
|
|
except ValueError:
|
|
return url
|
|
host = parts.netloc.lower()
|
|
host = host[4:] if host.startswith("www.") else host
|
|
query = urlencode([(k, v) for k, v in parse_qsl(parts.query, keep_blank_values=True) if not TRACKING.match(k)])
|
|
return urlunsplit((parts.scheme.lower(), host, parts.path.rstrip("/") or "/", query, ""))
|
|
|
|
|
|
def link_url(path):
|
|
"""The URL a link file points at, or None."""
|
|
try:
|
|
text = path.read_text(encoding="utf-8", errors="replace")
|
|
except OSError:
|
|
return None
|
|
suffix = path.suffix.lower()
|
|
if suffix == ".url":
|
|
m = re.search(r"(?im)^URL=(\S+)", text)
|
|
return m.group(1) if m else None
|
|
if suffix == ".webloc":
|
|
m = re.search(r"<string>(https?://[^<]+)</string>", text)
|
|
return m.group(1) if m else None
|
|
if suffix in TEXT_SUFFIXES and len(text) < 2048:
|
|
found = URL_RE.findall(text)
|
|
rest = URL_RE.sub("", text).strip()
|
|
if len(found) == 1 and not rest:
|
|
return found[0]
|
|
return None
|
|
|
|
|
|
def files_under(base):
|
|
if not base.is_dir():
|
|
return []
|
|
return [p for p in sorted(base.rglob("*"))
|
|
if p.is_file() and not any(part.startswith(".") for part in p.relative_to(base).parts)]
|
|
|
|
|
|
def frontmatter_value(text, key):
|
|
if not text.startswith("---\n"):
|
|
return ""
|
|
end = text.find("\n---\n", 4)
|
|
for line in text[4:end if end != -1 else 0].split("\n"):
|
|
if line.startswith(f"{key}:"):
|
|
return line.split(":", 1)[1].strip().strip("\"'")
|
|
return ""
|
|
|
|
|
|
def wiki_citations(root):
|
|
"""({sha256 prefix: [pages]}, {normalised url: [pages]}, {cited path: [pages]})."""
|
|
by_fp, by_url, by_path = {}, {}, {}
|
|
for page in sorted((root / "wiki").rglob("*.md")):
|
|
rel = page.relative_to(root).as_posix()
|
|
text = page.read_text(encoding="utf-8", errors="replace")
|
|
for key in ("source_fingerprint", "resource"):
|
|
value = frontmatter_value(text, key)
|
|
m = FP_RE.search(value)
|
|
if m:
|
|
by_fp.setdefault(m.group(1).lower(), []).append(rel)
|
|
if URL_RE.match(value):
|
|
by_url.setdefault(normalise_url(value), []).append(rel)
|
|
in_sources = False
|
|
for line in text.split("\n"):
|
|
if line.startswith("## "):
|
|
in_sources = line.strip() == "## Sources"
|
|
continue
|
|
if not in_sources:
|
|
continue
|
|
b = SOURCE_BULLET_RE.match(line)
|
|
if not b:
|
|
continue
|
|
cited, rest = b.group(1).strip(), b.group(2)
|
|
if URL_RE.match(cited):
|
|
by_url.setdefault(normalise_url(cited), []).append(rel)
|
|
else:
|
|
by_path.setdefault(cited.lstrip("/"), []).append(rel)
|
|
m = FP_RE.search(rest)
|
|
if m:
|
|
by_fp.setdefault(m.group(1).lower(), []).append(rel)
|
|
return by_fp, by_url, by_path
|
|
|
|
|
|
def fp_pages(by_fp, full):
|
|
pages = set()
|
|
for prefix, rels in by_fp.items():
|
|
if full.startswith(prefix):
|
|
pages.update(rels)
|
|
return sorted(pages)
|
|
|
|
|
|
def archive_date(rel):
|
|
m = re.match(r"raw/archive/(\d{4}-\d{2}-\d{2})/", rel)
|
|
return m.group(1) if m else "an unknown date"
|
|
|
|
|
|
def scan(root):
|
|
inbox = files_under(root / "raw" / "inbox")
|
|
archive = files_under(root / "raw" / "archive")
|
|
by_fp, by_url, by_path = wiki_citations(root)
|
|
arch_hash, arch_text = {}, {}
|
|
for f in archive:
|
|
rel = f.relative_to(root).as_posix()
|
|
arch_hash.setdefault(sha256(f), rel)
|
|
td = text_digest(f)
|
|
if td:
|
|
arch_text.setdefault(td, rel)
|
|
|
|
hits, fresh, seen = [], [], {}
|
|
for f in inbox:
|
|
rel = f.relative_to(root).as_posix()
|
|
full = sha256(f)
|
|
hit = None
|
|
if full in arch_hash:
|
|
match = arch_hash[full]
|
|
hit = {"kind": "DUPLICATE", "file": rel, "match": match,
|
|
"detail": f"identical to {match} (ingested {archive_date(match)})",
|
|
"pages": sorted(set(by_path.get(match, [])) | set(fp_pages(by_fp, full)))}
|
|
elif text_digest(f) and text_digest(f) in arch_text:
|
|
match = arch_text[text_digest(f)]
|
|
hit = {"kind": "SAME TEXT", "file": rel, "match": match,
|
|
"detail": f"same text as {match} (ingested {archive_date(match)}; only whitespace or line endings differ)",
|
|
"pages": sorted(set(by_path.get(match, [])))}
|
|
elif fp_pages(by_fp, full):
|
|
hit = {"kind": "CITED", "file": rel, "match": f"sha256:{full[:8]}",
|
|
"detail": f"sha256:{full[:8]} is already cited as a source", "pages": fp_pages(by_fp, full)}
|
|
else:
|
|
url = link_url(f)
|
|
if url and normalise_url(url) in by_url:
|
|
hit = {"kind": "SAME URL", "file": rel, "match": url,
|
|
"detail": f"{url} is already cited as a source", "pages": sorted(set(by_url[normalise_url(url)]))}
|
|
elif full in seen:
|
|
hit = {"kind": "INBOX TWIN", "file": rel, "match": seen[full],
|
|
"detail": f"identical to {seen[full]}, also in the inbox", "pages": []}
|
|
seen.setdefault(full, rel)
|
|
if hit:
|
|
hits.append(hit)
|
|
else:
|
|
fresh.append(rel)
|
|
return hits, fresh
|
|
|
|
|
|
def main(argv=None):
|
|
ap = argparse.ArgumentParser(description="Report inbox files this KB has already ingested.")
|
|
ap.add_argument("--root", default=".", help="KB root (default: current directory)")
|
|
ap.add_argument("--json", action="store_true")
|
|
a = ap.parse_args(argv)
|
|
root = Path(a.root).resolve()
|
|
if not (root / "wiki").is_dir():
|
|
sys.exit(f"error: {root} has no wiki/ — not a Cascade KB root")
|
|
hits, fresh = scan(root)
|
|
if a.json:
|
|
print(json.dumps({"duplicates": hits, "new": fresh}, indent=2))
|
|
return 1 if hits else 0
|
|
for h in hits:
|
|
pages = f" → {', '.join(h['pages'])}" if h["pages"] else ""
|
|
print(f"{h['kind']:<10} {h['file']} — {h['detail']}{pages}")
|
|
print(f"inbox: {len(fresh)} new, {len(hits)} already ingested or duplicated")
|
|
return 1 if hits else 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|