Confirmation codes (single use, payload-bound, expiry), wrong kb, redact, allowed hosts and a real-HTTP 421 for a foreign Host, OAuth with real RSA-signed tokens (unverified email, trust opt-out, aud, exp). Skipped without the SDK; CKB_MCP_PYTHON opts the smoke test in. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
304 lines
29 KiB
Bash
Executable file
304 lines
29 KiB
Bash
Executable file
#!/usr/bin/env bash
|
|
# Template smoke test: run the deterministic scripts against the fixture KB and
|
|
# compare with the golden output. Exit non-zero on any drift.
|
|
#
|
|
# tests/smoke.sh # check
|
|
# tests/smoke.sh --update # regenerate the golden files after an intended change
|
|
set -euo pipefail
|
|
cd "$(dirname "$0")/.."
|
|
FIX=tests/fixtures/minimal-kb
|
|
TODAY=2026-09-24
|
|
LINT=.agents/skills/ckb-lint/scripts/lint_report.py
|
|
RETR=.agents/skills/ckb-retrieve/scripts/retrieve_candidates.py
|
|
SAVE=.agents/skills/ckb-add-conversation/scripts/save_conversation.py
|
|
TMP=$(mktemp -d); trap 'rm -rf "$TMP"' EXIT
|
|
|
|
QUICK=.agents/skills/ckb-lint/scripts/lint_quick.sh
|
|
( cd "$FIX" && python3 "../../../$LINT" --json --today "$TODAY" ) > "$TMP/lint.json"
|
|
# The no-Python fallback must agree with the reference on the three counts it covers.
|
|
( cd "$FIX" && python3 "../../../$LINT" --quick --today "$TODAY" || true ) > "$TMP/quick.py.txt"
|
|
( cd "$FIX" && bash "../../../$QUICK" --today "$TODAY" || true ) | sed 's/ (fallback:.*//' > "$TMP/quick.sh.txt"
|
|
python3 "$RETR" "how many times does Atlas retry a failed batch" --root "$FIX" --today "$TODAY" --json > "$TMP/retrieve.json"
|
|
# Conversation rendering must be stable: same fixture transcript, same Markdown.
|
|
mkdir -p "$TMP/conv" && python3 "$SAVE" --from tests/fixtures/conversation/session.jsonl --root "$TMP/conv" --today "$TODAY" --author "Fixture User <fixture@example.com>" > /dev/null
|
|
cp "$TMP"/conv/raw/saved-conversations/"$TODAY"/*.md "$TMP/conversation.md"
|
|
# A title set with --set-title must win over the transcript's own title and rename the saved pair.
|
|
mkdir -p "$TMP/titled" && ENC=$(python3 -c "import re,sys;print(re.sub(r'[^A-Za-z0-9]','-',sys.argv[1]))" "$(cd "$TMP/titled" && pwd -P)") \
|
|
&& mkdir -p "$TMP/cfg/projects/$ENC" && cp tests/fixtures/conversation/session.jsonl "$TMP/cfg/projects/$ENC/"
|
|
( cd "$TMP/titled" && CLAUDE_CONFIG_DIR="$TMP/cfg" python3 "$OLDPWD/$SAVE" --today "$TODAY" --author "Fixture User <fixture@example.com>" >/dev/null \
|
|
&& CLAUDE_CONFIG_DIR="$TMP/cfg" python3 "$OLDPWD/$SAVE" --set-title "Smoke Title" >/dev/null \
|
|
&& CLAUDE_CONFIG_DIR="$TMP/cfg" python3 "$OLDPWD/$SAVE" --today "$TODAY" --author "Fixture User <fixture@example.com>" >/dev/null )
|
|
ls "$TMP/titled/raw/saved-conversations/$TODAY" > "$TMP/titled.txt"
|
|
# Routines: due list is deterministic for the fixture; edits touch only the routine's own lines.
|
|
ROUT=.agents/skills/ckb-routine/scripts/routines.py
|
|
( python3 "$ROUT" --due --root "$FIX" --today "$TODAY" || true ) > "$TMP/routines-due.txt"
|
|
cp "$FIX/ckb.yaml" "$TMP/ckb.yaml"
|
|
python3 "$ROUT" --root "$TMP" --mark weekly-lint --today "$TODAY" >/dev/null
|
|
python3 "$ROUT" --root "$TMP" --add --name smoke --do "Lint" --every 2w >/dev/null
|
|
python3 "$ROUT" --root "$TMP" --remove smoke >/dev/null
|
|
|
|
# Dedup: one copy, one re-saved copy, one cited URL, two inbox twins, one new file.
|
|
DEDUP=.agents/skills/ckb-ingest/scripts/dedup_inbox.py
|
|
mkdir -p "$TMP/dedup" && cp -r "$FIX/." "$TMP/dedup/" && mkdir -p "$TMP/dedup/raw/inbox"
|
|
cp "$FIX/raw/archive/2026-09-01/kickoff-notes.md" "$TMP/dedup/raw/inbox/kickoff-copy.md"
|
|
sed 's/$/\r/' "$FIX/raw/archive/2026-09-02/dlq-review.md" > "$TMP/dedup/raw/inbox/dlq-crlf.md"
|
|
printf '\n## Sources\n\n- `https://www.example.com/atlas/retries/` — etag:abc\n' >> "$TMP/dedup/wiki/projects/atlas.md"
|
|
printf '[InternetShortcut]\nURL=https://example.com/atlas/retries?utm_source=x#top\n' > "$TMP/dedup/raw/inbox/retries.url"
|
|
echo "fresh material" > "$TMP/dedup/raw/inbox/a-new.md" && echo "twin" > "$TMP/dedup/raw/inbox/b-twin.md" && cp "$TMP/dedup/raw/inbox/b-twin.md" "$TMP/dedup/raw/inbox/c-twin.md"
|
|
( python3 "$DEDUP" --root "$TMP/dedup" || true ) | sed "s#$TMP#TMP#g" > "$TMP/dedup.txt"
|
|
# Session start: open threads, hook mode, and the start-hook installer.
|
|
mkdir -p "$TMP/start" && cp -r "$FIX/." "$TMP/start/" && mkdir -p "$TMP/start/workload"
|
|
printf -- '---\ntype: session\n---\n\n## Work\n\n### Open threads\n- [ ] Ask Maria about DLQ size\n- [x] closed one\n\n## Later\n\n- [ ] not under Open threads\n' \
|
|
> "$TMP/start/workload/2026-08-01_fixture-user_summary.md"
|
|
( cd "$TMP/start" && python3 "$OLDPWD/.agents/skills/ckb-lint/scripts/session_start.py" --today "$TODAY" --no-git ) > "$TMP/start-threads.txt"
|
|
echo '{"source":"compact","session_id":"none"}' | ( cd "$TMP/start" && python3 "$OLDPWD/.agents/skills/ckb-lint/scripts/session_start.py" --hook --today "$TODAY" --no-git ) > "$TMP/hook.json"
|
|
mkdir -p "$TMP/start/.claude" && echo '{"hooks":{"Stop":[{"hooks":[{"type":"command","command":"keep me"}]}]}}' > "$TMP/start/.claude/settings.local.json"
|
|
printf '\nsession:\n start_hook: true\n' >> "$TMP/start/ckb.yaml"
|
|
HOOK=.agents/skills/ckb-lint/scripts/start_hook.py
|
|
( cd "$TMP/start" && ! python3 "$OLDPWD/$HOOK" --check >/dev/null && python3 "$OLDPWD/$HOOK" --install >/dev/null \
|
|
&& python3 "$OLDPWD/$HOOK" --check >/dev/null && python3 "$OLDPWD/$HOOK" --remove >/dev/null ) > "$TMP/hook-cycle.txt" 2>&1 && echo ok >> "$TMP/hook-cycle.txt"
|
|
# Hooks run in the session's current directory: both must start from the project root.
|
|
FIXJSONL="$PWD/tests/fixtures/conversation/session.jsonl"
|
|
mkdir -p "$TMP/proj/sub" "$TMP/proj/wiki" "$TMP/proj/.claude" && ln -s "$PWD/.agents" "$TMP/proj/.agents"
|
|
OLDSTOP='python3 .agents/skills/ckb-add-conversation/scripts/save_conversation.py --from-hook >/dev/null 2>&1 || true'
|
|
python3 -c "import json,sys;json.dump({'hooks':{'Stop':[{'hooks':[{'type':'command','command':sys.argv[1]},{'type':'command','command':'keep me'}]}]}},open(sys.argv[2],'w'))" \
|
|
"$OLDSTOP" "$TMP/proj/.claude/settings.local.json"
|
|
printf 'conversations:\n autosave: true\nsession:\n start_hook: true\n' > "$TMP/proj/ckb.yaml"
|
|
( cd "$TMP/proj" && ! python3 .agents/skills/ckb-add-conversation/scripts/autosave_hook.py --check >/dev/null \
|
|
&& python3 .agents/skills/ckb-add-conversation/scripts/autosave_hook.py --install >/dev/null \
|
|
&& python3 .agents/skills/ckb-add-conversation/scripts/autosave_hook.py --check >/dev/null \
|
|
&& python3 .agents/skills/ckb-lint/scripts/start_hook.py --install >/dev/null ) || echo "outdated hook not flagged or not replaced" > "$TMP/hookroot.err"
|
|
hookcmd() { python3 -c "import json,sys;print(next(h['command'] for g in json.load(open(sys.argv[1]))['hooks'][sys.argv[2]] for h in g['hooks'] if sys.argv[3] in h['command']))" \
|
|
"$TMP/proj/.claude/settings.local.json" "$1" "$2"; }
|
|
STOPCMD=$(hookcmd Stop save_conversation.py 2>/dev/null) || true; STARTCMD=$(hookcmd SessionStart session_start.py 2>/dev/null) || true
|
|
( cd "$TMP/proj/sub" && echo "{\"transcript_path\": \"$FIXJSONL\"}" | CLAUDE_PROJECT_DIR="$TMP/proj" sh -c "${STOPCMD:-false}" ) || true
|
|
( cd "$TMP/proj/sub" && echo '{"source":"startup"}' | CLAUDE_PROJECT_DIR="$TMP/proj" sh -c "${STARTCMD:-false}" ) > "$TMP/hookroot.json" || true
|
|
|
|
# Secrets: two planted keys (one in a gitignored export, skipped), a private span in wiki/, then an allowlist line.
|
|
mkdir -p "$TMP/sec" && cp -r "$FIX/." "$TMP/sec/" && mkdir -p "$TMP/sec/workload" "$TMP/sec/outputs/okf"
|
|
echo "aws AKIAABCDEFGHIJKLMNOP here" > "$TMP/sec/workload/2026-09-01_x_summary.md"
|
|
echo "AKIAABCDEFGHIJKLMNOP" > "$TMP/sec/outputs/okf/page.md"
|
|
echo "<private>do not keep</private>" >> "$TMP/sec/wiki/overview.md"
|
|
( cd "$TMP/sec" && python3 "$OLDPWD/$LINT" --secrets || true ) > "$TMP/secrets.txt"
|
|
( cd "$TMP/sec" && python3 "$OLDPWD/$LINT" --quick --today "$TODAY" || true ) > "$TMP/sec.py.txt"
|
|
( cd "$TMP/sec" && bash "$OLDPWD/$QUICK" --today "$TODAY" || true ) | sed 's/ (fallback:.*//' > "$TMP/sec.sh.txt"
|
|
ALLOW=$(grep -o 'add `workload/[^`]*`' "$TMP/secrets.txt" | sed 's/^add `//; s/`$//')
|
|
echo "$ALLOW # smoke: fixture key" > "$TMP/sec/.secrets-allow"
|
|
( cd "$TMP/sec" && python3 "$OLDPWD/$LINT" --secrets || true ) > "$TMP/secrets-allowed.txt"
|
|
# Schema 1.6: review fields, a contradiction marker, tags against a vocabulary, profile and lessons.
|
|
S16="$TMP/s16"; mkdir -p "$S16" && cp -r "$FIX/." "$S16/" && W="$S16/wiki"
|
|
printf -- '---\ntype: vocabulary\ntldr: Tags.\n---\n\n# Tags\n\n| Tag | Meaning | Use for |\n|---|---|---|\n| `retries` | Retry behaviour | Retry questions |\n' > "$W/tags.md"
|
|
echo '| [Tags](tags.md) | Tag vocabulary | Tagging |' >> "$W/index.md"
|
|
sed -i 's/^retention: high$/retention: high\ntags: retries, made-up\nreview: reviewed\nreviewed_by: Fixture User\nreviewed_on: 2026-09-01/' "$W/entities/atlas-retry-policy.md"
|
|
sed -i 's/^retention: medium$/retention: medium\nreview: draft/' "$W/entities/maria-nowak.md"
|
|
printf '\n> CONTRADICTION: three retries (`raw/archive/2026-09-01/kickoff-notes.md`) vs five retries (`raw/archive/2026-09-02/dlq-review.md`) — found 2026-09-24\n' >> "$W/entities/maria-nowak.md"
|
|
sed -i 's/^type: project$/type: project\nreview: approved/' "$W/projects/atlas.md"
|
|
printf -- '---\ntype: lessons\ntldr: Lessons.\nreview: needs_update\n---\n\n# Lessons\n\n### 2026-09-05 — Old lesson already seen\n- **Scope:** KB\n\n### 2026-09-20 — Cite the runbook for retry numbers\n- **Scope:** KB\n' > "$W/lessons.md"
|
|
( cd "$S16" && python3 "$OLDPWD/$LINT" --json --today "$TODAY" || true ) > "$TMP/s16-lint.json"
|
|
python3 "$RETR" "how many times does Atlas retry a failed batch" --root "$S16" --today "$TODAY" --tag retries --json > "$TMP/s16-retr.json"
|
|
SLUG=$(cd "$S16" && python3 -c "import sys;sys.path.insert(0,sys.argv[1]);import session_start as s;from pathlib import Path;print(s.author_slug(Path('.')))" "$OLDPWD/.agents/skills/ckb-lint/scripts")
|
|
mkdir -p "$S16/workload/profiles" && printf -- '---\ntype: profile\n---\n\n# Fixture\n\n## How to work with me\n- Answer in short bullets.\n' > "$S16/workload/profiles/$SLUG.md"
|
|
printf -- '---\ntype: session\n---\n\n## Earlier\n- worked\n' > "$S16/workload/2026-09-10_${SLUG}_summary.md"
|
|
( cd "$S16" && python3 "$OLDPWD/.agents/skills/ckb-lint/scripts/session_start.py" --today "$TODAY" --no-git ) > "$TMP/s16-start.txt"
|
|
|
|
# Private spans never reach a saved conversation.
|
|
mkdir -p "$TMP/priv" && sed 's#How many times does Atlas retry a failed batch?#<private>hunter2 is the vault code</private> How many times does Atlas retry a failed batch?#' \
|
|
tests/fixtures/conversation/session.jsonl > "$TMP/priv-session.jsonl"
|
|
python3 "$SAVE" --from "$TMP/priv-session.jsonl" --root "$TMP/priv" --today "$TODAY" --author "Fixture User <fixture@example.com>" > /dev/null
|
|
|
|
if [[ "${1:-}" == "--update" ]]; then
|
|
cp "$TMP/dedup.txt" tests/expected/dedup.txt
|
|
cp "$TMP/lint.json" tests/expected/lint.json
|
|
cp "$TMP/retrieve.json" tests/expected/retrieve.json
|
|
cp "$TMP/conversation.md" tests/expected/conversation.md
|
|
cp "$TMP/routines-due.txt" tests/expected/routines-due.txt
|
|
echo "golden files updated"; exit 0
|
|
fi
|
|
|
|
fail=0
|
|
for f in lint retrieve; do
|
|
if ! diff -u "tests/expected/$f.json" "$TMP/$f.json"; then echo "DRIFT: $f"; fail=1; fi
|
|
done
|
|
if ! diff -u tests/expected/conversation.md "$TMP/conversation.md"; then echo "DRIFT: conversation"; fail=1; fi
|
|
[[ "$(cat "$TMP/titled.txt")" == $'smoke-title_0f1e2d3c.jsonl\nsmoke-title_0f1e2d3c.md' ]] || { echo "set-title: unexpected files: $(cat "$TMP/titled.txt")"; fail=1; }
|
|
grep -q '^# Smoke Title$' "$TMP/titled/raw/saved-conversations/$TODAY/smoke-title_0f1e2d3c.md" || { echo "set-title: heading not applied"; fail=1; }
|
|
if ! diff -u tests/expected/routines-due.txt "$TMP/routines-due.txt"; then echo "DRIFT: routines-due"; fail=1; fi
|
|
[[ "$(diff "$FIX/ckb.yaml" "$TMP/ckb.yaml" | grep -c '^[<>]')" == "2" ]] || { echo "routines: --mark/--add/--remove changed more than the last_run line"; diff "$FIX/ckb.yaml" "$TMP/ckb.yaml"; fail=1; }
|
|
grep -q 'last_run: 2026-09-24' "$TMP/ckb.yaml" || { echo "routines: --mark did not stamp today"; fail=1; }
|
|
if ! diff -u "$TMP/quick.py.txt" "$TMP/quick.sh.txt"; then echo "DRIFT: lint_quick.sh disagrees with lint_report.py --quick"; fail=1; fi
|
|
top=$(python3 -c "import json;print(json.load(open('$TMP/retrieve.json'))['candidates'][0]['path'])")
|
|
[[ "$top" == "wiki/entities/atlas-retry-policy.md" ]] || { echo "retrieve: expected atlas-retry-policy first, got $top"; fail=1; }
|
|
# Session start in one call: the fixture has one due routine and a clean wiki.
|
|
START=.agents/skills/ckb-lint/scripts/session_start.py
|
|
python3 "$START" --root "$FIX" --today "$TODAY" --no-git > "$TMP/start.txt" || { echo "session_start.py failed"; fail=1; }
|
|
grep -q '^ckb check: clean' "$TMP/start.txt" || { echo "session_start: no clean check line"; fail=1; }
|
|
grep -q '^routines: DUE weekly-lint' "$TMP/start.txt" || { echo "session_start: due routine missing"; fail=1; }
|
|
grep -q '^announce: yes' "$TMP/start.txt" || { echo "session_start: announce line missing"; fail=1; }
|
|
# Release 1.22 features.
|
|
if ! diff -u tests/expected/dedup.txt "$TMP/dedup.txt"; then echo "DRIFT: dedup"; fail=1; fi
|
|
# One open thread: the ticked one and the one outside "Open threads" must not count.
|
|
grep -q '^threads: 1 open' "$TMP/start-threads.txt" || { echo "session_start: expected exactly 1 open thread"; fail=1; }
|
|
python3 -c "import json,sys;c=json.load(open(sys.argv[1]))['hookSpecificOutput']['additionalContext'];assert 'ckb check: clean' in c and 'announce: no' in c" "$TMP/hook.json" \
|
|
|| { echo "session_start --hook: bad JSON or wrong announce on compact"; fail=1; }
|
|
[[ "$(tail -1 "$TMP/hook-cycle.txt")" == ok ]] || { echo "start_hook: check/install/remove cycle failed"; cat "$TMP/hook-cycle.txt"; fail=1; }
|
|
grep -q '"keep me"' "$TMP/start/.claude/settings.local.json" || { echo "start_hook: removed a hook it does not own"; fail=1; }
|
|
[[ "$(tail -1 "$TMP/secrets.txt")" == "secrets: 2 possible" ]] || { echo "secrets: expected 2 (key + private span)"; cat "$TMP/secrets.txt"; fail=1; }
|
|
[[ "$(tail -1 "$TMP/secrets-allowed.txt")" == "secrets: 1 possible" ]] || { echo "secrets: allowlist line not honoured"; cat "$TMP/secrets-allowed.txt"; fail=1; }
|
|
if ! diff -u "$TMP/sec.py.txt" "$TMP/sec.sh.txt"; then echo "DRIFT: lint_quick.sh secrets count disagrees with lint_report.py"; fail=1; fi
|
|
! grep -rq hunter2 "$TMP/priv/raw/saved-conversations" || { echo "save_conversation: <private> span was saved"; fail=1; }
|
|
grep -q '<private>\[redacted\]</private>' "$TMP"/priv/raw/saved-conversations/"$TODAY"/*.md || { echo "save_conversation: redaction mark missing"; fail=1; }
|
|
python3 -c "import json,sys;[json.loads(l) for l in open(sys.argv[1]) if l.strip()]" "$TMP"/priv/raw/saved-conversations/"$TODAY"/*.jsonl \
|
|
|| { echo "save_conversation: redacted JSONL is not valid JSON"; fail=1; }
|
|
[[ ! -e "$TMP/hookroot.err" ]] || { cat "$TMP/hookroot.err"; fail=1; }
|
|
python3 -c "import json,sys;h=json.load(open(sys.argv[1]))['hooks']['Stop'];c=[x['command'] for g in h for x in g['hooks']];s=[x for x in c if 'save_conversation' in x];assert c.count('keep me')==1 and len(s)==1 and 'CLAUDE_PROJECT_DIR' in s[0]" \
|
|
"$TMP/proj/.claude/settings.local.json" || { echo "autosave_hook --install: did not replace the outdated entry cleanly"; fail=1; }
|
|
ls "$TMP"/proj/raw/saved-conversations/*/*.md >/dev/null 2>&1 && [[ ! -e "$TMP/proj/sub/raw" ]] \
|
|
|| { echo "autosave hook: run from a subfolder, it did not save into the project root"; fail=1; }
|
|
grep -q 'ckb check:' "$TMP/hookroot.json" || { echo "start hook: run from a subfolder, it printed no brief"; fail=1; }
|
|
python3 - "$TMP/s16-lint.json" <<'PY' || fail=1
|
|
import json, sys
|
|
t = json.load(open(sys.argv[1]))["trees"][0]
|
|
want = {"review": 2, "contradictions": 1, "tags": 1}
|
|
got = {k: len(t[k]) for k in want}
|
|
problems = [f"schema 1.6 lint: {k} expected {v}, got {got[k]}: {t[k]}" for k, v in want.items() if got[k] != v]
|
|
if not any("changed since its review" in f for f in t["review"]):
|
|
problems.append("schema 1.6 lint: reviewed page edited after review not flagged")
|
|
if not any("made-up" in f for f in t["tags"]):
|
|
problems.append("schema 1.6 lint: unknown tag not flagged")
|
|
if sorted(t["review_queue"]) != ["entities/maria-nowak.md (draft)", "lessons.md (needs_update)"]:
|
|
problems.append(f"schema 1.6 lint: review queue {t['review_queue']}")
|
|
problems and print("\n".join(problems)); sys.exit(1 if problems else 0)
|
|
PY
|
|
python3 -c "import json,sys;c=json.load(open(sys.argv[1]))['candidates'];assert [x['path'] for x in c]==['wiki/entities/atlas-retry-policy.md'] and c[0]['review']=='reviewed'" "$TMP/s16-retr.json" \
|
|
|| { echo "retrieve --tag: expected only the tagged page"; fail=1; }
|
|
grep -q '^profile: workload/profiles/' "$TMP/s16-start.txt" && grep -q 'Answer in short bullets' "$TMP/s16-start.txt" || { echo "session_start: profile not printed"; fail=1; }
|
|
grep -q '^lessons: 1 in wiki/lessons.md since your note of 2026-09-10' "$TMP/s16-start.txt" && grep -q 'Cite the runbook' "$TMP/s16-start.txt" \
|
|
|| { echo "session_start: new lesson not listed"; grep lessons "$TMP/s16-start.txt"; fail=1; }
|
|
grep -q '^### Open threads' "$TMP/start-threads.txt" && { echo "session_start: open threads printed twice"; fail=1; }
|
|
# A growing transcript: one `match:` allowlist entry covers the same key on every line.
|
|
mkdir -p "$TMP/sec/raw/saved-conversations/d" && for i in 1 2 3; do echo "turn $i AKIAQRSTUVWXYZABCDEF" >> "$TMP/sec/raw/saved-conversations/d/t.jsonl"; done
|
|
MH=$(cd "$TMP/sec" && { python3 "$OLDPWD/$LINT" --secrets || true; } | grep 't.jsonl' | grep -o 'match:[0-9a-f]*' | head -1)
|
|
echo "raw/saved-conversations/*/*.jsonl $MH" >> "$TMP/sec/.secrets-allow"
|
|
[[ "$(cd "$TMP/sec" && python3 "$OLDPWD/$LINT" --secrets | tail -1)" == "secrets: 1 possible" ]] || { echo "secrets: match: allowlist entry not honoured"; fail=1; }
|
|
[[ "$(cd "$TMP/sec" && bash "$OLDPWD/$QUICK" --today "$TODAY" | sed 's/ (fallback:.*//')" == "ckb check: 1 possible secrets — run \"lint\" for detail" ]] \
|
|
|| { echo "lint_quick.sh: match: allowlist entry not honoured"; fail=1; }
|
|
# Section search (bm25): anchors returned, cache refreshes on change, no regression on the benchmark.
|
|
python3 tests/bench_retrieval.py --check > "$TMP/bench.txt" || { echo "bench: bm25 made retrieval worse"; cat "$TMP/bench.txt"; fail=1; }
|
|
python3 -c "import json,sys;c=json.load(open(sys.argv[1]))['candidates'][0];assert c['sections'] and c['tokens'], c" "$TMP/retrieve.json" \
|
|
|| { echo "retrieve: top candidate has no section hits or token estimate"; fail=1; }
|
|
FTS="$TMP/fts"; mkdir -p "$FTS/tmp" && cp -r "$FIX/." "$FTS/"
|
|
python3 "$RETR" "parked batches" --root "$FTS" --json > /dev/null
|
|
[[ -s "$FTS/tmp/ckb-fts.sqlite" ]] || { echo "bm25: no index cache in tmp/"; fail=1; }
|
|
printf '\n## Quarantine\n\nZebra-striped batches go to the quarantine shelf.\n' >> "$FTS/wiki/entities/maria-nowak.md"
|
|
python3 -c "import json,sys;c=json.load(open(sys.argv[1]))['candidates'];assert c and c[0]['path']=='wiki/entities/maria-nowak.md' and any(s['anchor']=='quarantine' for s in c[0]['sections'])" \
|
|
<(python3 "$RETR" "zebra striped quarantine shelf" --root "$FTS" --json) || { echo "bm25: index did not refresh after a page changed"; fail=1; }
|
|
# Access log and cold pages: proposals only, after enough history.
|
|
python3 "$RETR" --opened wiki/entities/atlas-retry-policy.md --root "$FTS" --today 2026-05-01 > /dev/null
|
|
( cd "$FTS" && python3 "$OLDPWD/$LINT" --cold 30 --today "$TODAY" ) > "$TMP/cold.txt"
|
|
grep -q '^- projects/atlas.md (never read' "$TMP/cold.txt" && ! grep -q 'atlas-retry-policy\|overview.md' "$TMP/cold.txt" \
|
|
|| { echo "lint --cold: unexpected proposals"; cat "$TMP/cold.txt"; fail=1; }
|
|
# Consolidation: a scope cluster's topic goes missing -> current -> outdated; a missing link is found.
|
|
CONS=.agents/skills/ckb-consolidate/scripts/consolidate.py; CK="$TMP/cons"; mkdir -p "$CK" && cp -r "$FIX/." "$CK/"
|
|
state() { python3 "$CONS" --root "$CK" --today "$TODAY" --json | python3 -c "import json,sys;print(next(t['state'] for t in json.load(sys.stdin)['topics'] if t['name']=='atlas'))"; }
|
|
[[ "$(state)" == missing ]] || { echo "consolidate: atlas topic should be missing"; fail=1; }
|
|
mkdir -p "$CK/wiki/topics" && { printf -- '---\ntype: topic\ntldr: Atlas.\n---\n\n# Atlas\n\n## Sources\n\n'
|
|
for m in entities/atlas-retry-policy.md entities/maria-nowak.md; do echo "- \`wiki/$m\` — sha256:$(sha256sum "$CK/wiki/$m" | cut -c1-8) (checked $TODAY)"; done; } > "$CK/wiki/topics/atlas.md"
|
|
[[ "$(state)" == current ]] || { echo "consolidate: atlas topic should be current"; fail=1; }
|
|
echo "Extra line." >> "$CK/wiki/entities/maria-nowak.md"
|
|
[[ "$(state)" == outdated ]] || { echo "consolidate: atlas topic should be outdated after a member changed"; fail=1; }
|
|
printf -- '---\ntype: concept\ntldr: Backoff timing for Atlas.\n---\n\n# Backoff\n\n## Sources\n\n- `raw/archive/2026-09-01/kickoff-notes.md` — sha256:a97984a4\n- `raw/archive/2026-09-02/dlq-review.md` — sha256:2bbe6665\n' > "$CK/wiki/entities/backoff.md"
|
|
python3 "$CONS" --root "$CK" --today "$TODAY" --json | python3 -c "import json,sys;l=json.load(sys.stdin)['links'];assert ['entities/atlas-retry-policy.md','entities/backoff.md'] in [x[:2] for x in l], l" \
|
|
|| { echo "consolidate: missing link between pages sharing two sources not found"; fail=1; }
|
|
python3 "$CONS" --root "$CK" --today "$TODAY" --write > /dev/null && grep -q '^## Merge candidates' "$CK/outputs/reports/${TODAY}_consolidation.md" \
|
|
|| { echo "consolidate: --write did not produce the report"; fail=1; }
|
|
# capture module, installed into a scratch KB: direct fetch, duplicate refused, Wayback fallback after a 404.
|
|
CAP="$TMP/cap"; WEB="$TMP/web"; mkdir -p "$CAP/.agents/skills" "$WEB/web/20260101id_" && cp -r "$FIX/." "$CAP/"
|
|
cp -r .agents/skills/ckb-ingest .agents/modules/capture/skills/ckb-capture "$CAP/.agents/skills/"
|
|
PORT=$(python3 -c "import socket;s=socket.socket();s.bind(('127.0.0.1',0));print(s.getsockname()[1])")
|
|
printf '<html><head><title>Atlas runbook</title><script>var x=1;</script></head><body><nav>Menu</nav><main><h1>Atlas runbook</h1><p>When a batch fails, Atlas retries it five times with jittered backoff before it goes to the dead-letter queue, and the queue is drained every Monday by the platform team.</p><p>Replays need a ticket and a second reviewer.</p><ul><li>Never replay without a ticket.</li></ul></main></body></html>' > "$WEB/page.html"
|
|
printf '<html><head><title>Old page</title></head><body><p>This page only survives in the archive. It explains the legacy retry budget of seven attempts, which was dropped in 2025 when the dead-letter queue arrived.</p></body></html>' > "$WEB/web/20260101id_/snap.html"
|
|
printf '{"archived_snapshots": {"closest": {"available": true, "timestamp": "20260101", "url": "http://127.0.0.1:%s/web/20260101/snap.html"}}}' "$PORT" > "$WEB/wayback.json"
|
|
( cd "$WEB" && exec python3 -m http.server "$PORT" --bind 127.0.0.1 >/dev/null 2>&1 ) & HTTP_PID=$!
|
|
trap 'kill $HTTP_PID 2>/dev/null; rm -rf "$TMP"' EXIT
|
|
python3 -c "import socket,time,sys
|
|
for _ in range(50):
|
|
try: socket.create_connection(('127.0.0.1',int(sys.argv[1])),0.2).close(); break
|
|
except OSError: time.sleep(0.1)" "$PORT"
|
|
CAPPY="$CAP/.agents/skills/ckb-capture/scripts/capture_url.py"
|
|
export CKB_WAYBACK_API="http://127.0.0.1:$PORT/wayback.json?url="
|
|
rc=0; python3 "$CAPPY" "http://127.0.0.1:$PORT/page.html" --root "$CAP" --today "$TODAY" > /dev/null 2>&1 || rc=$?
|
|
[[ $rc == 3 ]] && ! ls "$CAP"/raw/inbox/*.md >/dev/null 2>&1 || { echo "capture: a loopback URL was fetched without --allow-private (exit $rc)"; fail=1; }
|
|
python3 "$CAPPY" "http://127.0.0.1:$PORT/page.html" --root "$CAP" --today "$TODAY" --allow-private > /dev/null || { echo "capture: direct fetch failed"; fail=1; }
|
|
CF=$(ls "$CAP"/raw/inbox/*atlas-runbook*.md 2>/dev/null | head -1)
|
|
[[ -n "$CF" ]] && grep -q '^fetched_via: direct' "$CF" && grep -q '^untrusted: true' "$CF" && grep -q 'five times' "$CF" && ! grep -q 'var x\|Menu' "$CF" \
|
|
|| { echo "capture: snapshot missing, unmarked, or not cleaned"; fail=1; }
|
|
[[ "$(grep -c '^# Atlas runbook' "$CF" 2>/dev/null)" == 1 ]] || { echo "capture: title heading duplicated"; fail=1; }
|
|
rc=0; python3 "$CAPPY" "http://127.0.0.1:$PORT/page.html" --root "$CAP" --today "$TODAY" --allow-private > /dev/null || rc=$?
|
|
[[ $rc == 1 ]] || { echo "capture: duplicate URL was kept (exit $rc)"; fail=1; }
|
|
python3 "$CAPPY" "http://127.0.0.1:$PORT/gone.html" --root "$CAP" --today "$TODAY" --allow-private > /dev/null || { echo "capture: Wayback fallback failed"; fail=1; }
|
|
grep -l '^fetched_via: wayback' "$CAP"/raw/inbox/*.md 2>/dev/null | xargs grep -q 'seven attempts' 2>/dev/null || { echo "capture: Wayback snapshot not used"; fail=1; }
|
|
unset CKB_WAYBACK_API
|
|
# share module: two KBs and a bare-repo channel — join, send, receive, version, retract, forged mail, secret block.
|
|
SH="$TMP/share"; mkdir -p "$SH" && git init -q --bare "$SH/chan.git"
|
|
for k in alpha beta; do cp -r "$FIX" "$SH/$k" && mkdir -p "$SH/$k/.agents/skills" \
|
|
&& cp -r .agents/skills/ckb-lint .agents/skills/ckb-ingest .agents/modules/share/skills/ckb-share "$SH/$k/.agents/skills/"; done
|
|
SHPY=.agents/modules/share/skills/ckb-share/scripts/share.py
|
|
al() { python3 "$SHPY" --root "$SH/alpha" --today "$TODAY" --author "Al <al@example.com>" "$@"; }
|
|
be() { python3 "$SHPY" --root "$SH/beta" --today "$TODAY" --author "Be <be@example.com>" "$@"; }
|
|
SHDOC=wiki/entities/atlas-retry-policy.md
|
|
printf '\n## Links\n\nSee [Maria](/wiki/entities/maria-nowak.md) <private>hidden</private>.\n\n## Notes\n\nKeep this out.\n' >> "$SH/alpha/$SHDOC"
|
|
{ al join --channel cx --url "$SH/chan.git" --kb-name alpha && be join --channel cx --url "$SH/chan.git" --kb-name beta; } > /dev/null \
|
|
|| { echo "share: join failed"; fail=1; }
|
|
al send "$SHDOC" --to beta > /dev/null || { echo "share: send failed"; fail=1; }
|
|
SM="$SH/alpha/tmp/share/cx/mail/alpha/${TODAY}_atlas-retry-policy_v1.md"
|
|
grep -q '^untrusted: true' "$SM" && grep -q 'alpha: `raw/archive/' "$SM" && grep -q 'Maria (alpha: wiki/entities/maria-nowak.md)' "$SM" \
|
|
&& ! grep -q 'Keep this out\|hidden' "$SM" || { echo "share: packed copy wrong (notes, private, links or marking)"; fail=1; }
|
|
be receive --take all > /dev/null || { echo "share: receive failed"; fail=1; }
|
|
SI="$SH/beta/raw/inbox/${TODAY}_alpha_atlas-retry-policy_v1.md"
|
|
grep -q '^submitted_by: Al <al@example.com>' "$SI" && grep -q '^share_channel: cx' "$SI" \
|
|
&& grep -q "cx mail/alpha/${TODAY}_atlas-retry-policy_v1.md taken" "$SH/beta/ckb.yaml" || { echo "share: received file or ledger wrong"; fail=1; }
|
|
rc=0; be list > /dev/null || rc=$?; [[ $rc == 1 ]] || { echo "share: a taken message was offered again"; fail=1; }
|
|
mkdir -p "$SH/beta/raw/archive/$TODAY" && mv "$SI" "$SH/beta/raw/archive/$TODAY/"
|
|
echo "More detail." >> "$SH/alpha/$SHDOC"
|
|
{ al send "$SHDOC" --to beta && be receive --take all; } > /dev/null
|
|
grep -q "^supersedes_source: raw/archive/$TODAY/" "$SH/beta/raw/inbox/${TODAY}_alpha_atlas-retry-policy_v2.md" 2>/dev/null \
|
|
|| { echo "share: v2 does not point at v1"; fail=1; }
|
|
cp "$SH/beta/raw/archive/$TODAY/${TODAY}_alpha_atlas-retry-policy_v1.md" "$SH/beta/raw/inbox/again.md"
|
|
( python3 .agents/skills/ckb-ingest/scripts/dedup_inbox.py --root "$SH/beta" || true ) | grep -q '^SAME SHARE raw/inbox/again.md' \
|
|
|| { echo "dedup: a re-received share message not reported"; fail=1; }
|
|
rm "$SH/beta/raw/inbox/again.md"
|
|
{ al unshare atlas-retry-policy --reason "Wrong client" && be receive --take all; } > /dev/null
|
|
[[ $(grep -l '^via: "share:cx"' "$SH"/beta/raw/retractions/*.md 2>/dev/null | wc -l) == 2 ]] \
|
|
|| { echo "share: retraction did not file a request per held version"; fail=1; }
|
|
C="$SH/beta/tmp/share/cx"; printf -- '---\ntype: share\nkind: document\nshare_id: alpha/fake\nversion: 1\nfrom: alpha\nto: [beta]\ntitle: Fake\n---\n\n# Fake\n' > "$C/mail/alpha/${TODAY}_fake_v1.md"
|
|
git -C "$C" add -A && git -C "$C" -c user.name=Mallory -c user.email=m@example.com commit -qm forge && git -C "$C" push -q origin HEAD
|
|
rc=0; be receive --take all > "$TMP/share-forged.txt" || rc=$?
|
|
[[ $rc == 1 ]] && grep -q 'UNVERIFIED' "$TMP/share-forged.txt" && [[ ! -e "$SH/beta/raw/inbox/${TODAY}_alpha_fake_v1.md" ]] \
|
|
|| { echo "share: forged message was taken"; fail=1; }
|
|
printf -- '---\ntype: concept\n---\n# Keys\n\nkey AKIAABCDEFGHIJKLMNOP\n' > "$SH/alpha/wiki/entities/keys.md"
|
|
rc=0; al send wiki/entities/keys.md --to beta > /dev/null || rc=$?
|
|
[[ $rc == 3 ]] && [[ ! -e "$SH/alpha/tmp/share/outgoing/${TODAY}_keys_v1.md" ]] || { echo "share: secret not blocked (exit $rc)"; fail=1; }
|
|
rc=0; al send raw/archive/2026-09-01/kickoff-notes.md --to beta > /dev/null 2>&1 || rc=$?
|
|
[[ $rc == 1 ]] || { echo "share: raw source sent without --raw-ok"; fail=1; }
|
|
# Skill evals: every case is valid; the runner fails an untouched KB and passes the reference run.
|
|
python3 tests/run_evals.py validate > "$TMP/evals.txt" || { echo "evals: invalid case"; cat "$TMP/evals.txt"; fail=1; }
|
|
python3 tests/run_evals.py prepare ckb-decide/record "$TMP/eval" > /dev/null
|
|
rc=0; python3 tests/run_evals.py check ckb-decide/record "$TMP/eval" > /dev/null || rc=$?
|
|
[[ $rc == 1 ]] || { echo "evals: an untouched KB passed the decide case"; fail=1; }
|
|
# Every recorded reference run (files changed plus a .deleted list) still passes its case.
|
|
for ref in tests/evals/*/*/reference; do
|
|
c=${ref#tests/evals/}; c=${c%/reference}; rm -rf "$TMP/replay"
|
|
python3 tests/run_evals.py replay "$c" "$TMP/replay" > "$TMP/eval.txt" || { echo "evals: reference run of $c failed"; cat "$TMP/eval.txt"; fail=1; }
|
|
done
|
|
# The template must stay inside its own context budget.
|
|
python3 "$LINT" --budget > "$TMP/budget.txt" || { echo "budget: template is over its own always-loaded budget"; head -3 "$TMP/budget.txt"; fail=1; }
|
|
# mcp module: what a remote client may read and write (no MCP SDK needed).
|
|
python3 tests/mcp_policy.py "$FIX" || fail=1
|
|
# mcp server itself (confirmation codes, allowed hosts, OAuth email): needs the MCP SDK.
|
|
# Opt in with CKB_MCP_PYTHON=<venv>/bin/python; without it the unit run below skips them.
|
|
if [[ -n "${CKB_MCP_PYTHON:-}" ]]; then
|
|
"$CKB_MCP_PYTHON" -m unittest tests.unit.test_mcp_server 2>"$TMP/mcp-server.txt" || { cat "$TMP/mcp-server.txt"; echo "mcp server tests failed"; fail=1; }
|
|
fi
|
|
python3 -m unittest discover -s tests/unit -q 2>"$TMP/unit.txt" || { cat "$TMP/unit.txt"; echo "unit tests failed"; fail=1; }
|
|
[[ $fail -eq 0 ]] && echo "smoke: OK"
|
|
exit $fail
|