ckb/tests/run_evals.py
2026-11-11 18:00:20 +01:00

282 lines
12 KiB
Python

#!/usr/bin/env python3
"""Skill evals: structural checks of what an agent run left behind.
run_evals.py list # every case, with its prompt
run_evals.py prepare CASE DIR # copy the fixture + setup into DIR, print the prompt
run_evals.py check CASE DIR # score DIR after the agent ran there; exit 1 on a failure
run_evals.py validate # every case parses and names known assertion kinds
run_evals.py record CASE DIR # save what a passing run changed as the case's reference/
run_evals.py replay CASE DIR # prepare DIR, apply reference/ (incl. deletions), check
CASE is a path under tests/evals/, e.g. `ckb-decide/record`. A case may list
`replies`: what the operator says, in order, each time the agent stops and
waits (a check-in, a typed confirmation). `setup` steps, applied in order:
{"write": PATH, "content": TEXT} a file; "{upstream}" in TEXT is replaced
{"template": [PATH, ...], "ref": REF} copy template-repo paths into the
KB, from the working tree or a git REF
{"upstream": REF} a bare repo at tmp/eval-upstream.git whose
`main` is REF of this template repo The agent run
itself is manual or a CI job with a token budget: open DIR in the agent and
give it the prompt. Model output varies, so every assertion is structural —
files created or moved, frontmatter values, log entries, pages left alone,
lint checks clean — never the wording.
Assertion kinds (each one an object in the case's "assert" list):
file_exists: GLOB file_absent: GLOB
file_contains: {path: GLOB, regex: RE} some matching file matches RE
file_lacks: {path: GLOB, regex: RE} no matching file matches RE
frontmatter: {path: GLOB, keys: {k: v}} some matching file has every key;
v is exact, "*" (present) or "~RE"
unchanged: GLOB same bytes as when prepared
lint_clean: [check, ...] lint_report.py --json: these wiki checks have no findings
Python 3.8+, standard library only.
"""
import hashlib
import io
import json
import re
import shutil
import subprocess
import sys
import tarfile
from pathlib import Path
HERE = Path(__file__).resolve().parent
REPO = HERE.parent
EVALS = HERE / "evals"
FIXTURES = HERE / "fixtures"
LINT = HERE.parent / ".agents" / "skills" / "ckb-lint" / "scripts" / "lint_report.py"
BASELINE = ".eval-baseline.json"
KINDS = {"file_exists", "file_absent", "file_contains", "file_lacks", "frontmatter", "unchanged", "lint_clean"}
def load(case):
path = EVALS / case / "case.json"
if not path.is_file():
sys.exit(f"error: no case at {path}")
return json.loads(path.read_text(encoding="utf-8"))
def cases():
return sorted(p.parent.relative_to(EVALS).as_posix() for p in EVALS.glob("*/*/case.json"))
def glob_match(path, pattern):
"""Path.match treats ** as one segment before 3.13; mirror glob semantics instead."""
parts = re.split(r"(\*\*/|\*\*|\*|\?)", pattern)
rx = "".join("(?:.*/)?" if t == "**/" else ".*" if t == "**" else "[^/]*" if t == "*"
else "[^/]" if t == "?" else re.escape(t) for t in parts)
return re.fullmatch(rx, path) is not None
def files(root, pattern):
return sorted(p for p in root.glob(pattern) if p.is_file() and BASELINE not in p.name)
def frontmatter(text):
if not text.startswith("---\n"):
return {}
close = re.compile(r"^---[ \t]*$", re.M).search(text, 4) # may be the last line
out = {}
for line in text[4:close.start() if close else 4].split("\n"):
k, sep, v = line.partition(":")
if sep and not line[:1].isspace():
out[k.strip()] = v.strip().strip("\"'")
return out
def value_ok(want, got):
if got is None:
return False
if want == "*":
return True
if isinstance(want, str) and want.startswith("~"):
return re.search(want[1:], got) is not None
return str(want) == got
def prepare(case, target):
c, root = load(case), Path(target).resolve()
if root.exists() and any(root.iterdir()):
sys.exit(f"error: {root} is not empty")
shutil.copytree(FIXTURES / c["fixture"], root, dirs_exist_ok=True)
upstream = root / "tmp" / "eval-upstream.git"
for step in c.get("setup", []):
if "write" in step:
f = root / step["write"]
f.parent.mkdir(parents=True, exist_ok=True)
f.write_text(step["content"].replace("{upstream}", upstream.as_posix()), encoding="utf-8")
elif "template" in step:
copy_template(step["template"], root, step.get("ref"))
elif "upstream" in step:
upstream.parent.mkdir(parents=True, exist_ok=True)
subprocess.run(["git", "init", "-q", "--bare", str(upstream)], check=True)
subprocess.run(["git", "-C", str(REPO), "push", "-q", str(upstream), f"{step['upstream']}:refs/heads/main"],
check=True)
subprocess.run(["git", "-C", str(upstream), "symbolic-ref", "HEAD", "refs/heads/main"], check=True)
base = {p.relative_to(root).as_posix(): hashlib.sha256(p.read_bytes()).hexdigest()
for p in root.rglob("*") if p.is_file()}
(root / BASELINE).write_text(json.dumps(base, indent=1), encoding="utf-8")
print(f"prepared {root} from fixtures/{c['fixture']}\nprompt ({c['skill']}):\n\n{c['prompt']}")
for n, reply in enumerate(c.get("replies", []), 1):
print(f"\nreply {n}, when the agent stops and waits:\n\n{reply}")
def copy_template(paths, root, ref=None):
"""Template-repo files into the scratch KB: from the working tree, or from
a git ref (an older release, for upgrade cases)."""
if ref is None:
for rel in paths:
src, dst = REPO / rel, root / rel
dst.parent.mkdir(parents=True, exist_ok=True)
if src.is_dir():
shutil.copytree(src, dst, dirs_exist_ok=True, symlinks=True)
else:
shutil.copy2(src, dst, follow_symlinks=False)
return
data = subprocess.run(["git", "-C", str(REPO), "archive", "--format=tar", ref, "--", *paths],
capture_output=True, check=True).stdout
with tarfile.open(fileobj=io.BytesIO(data)) as tar:
for m in tar.getmembers():
if m.name.startswith(("/", "..")) or "/../" in m.name:
continue
tar.extract(m, root)
def check(case, target):
c, root = load(case), Path(target).resolve()
base = json.loads((root / BASELINE).read_text(encoding="utf-8")) if (root / BASELINE).is_file() else {}
lint = None
results = []
for a in c["assert"]:
(kind, arg), = a.items()
if kind == "file_exists":
ok, why = bool(files(root, arg)), arg
elif kind == "file_absent":
ok, why = not files(root, arg), arg
elif kind in ("file_contains", "file_lacks"):
hit = any(re.search(arg["regex"], f.read_text(encoding="utf-8", errors="replace"), re.M) for f in files(root, arg["path"]))
ok, why = (hit if kind == "file_contains" else not hit), f"{arg['path']} /{arg['regex']}/"
elif kind == "frontmatter":
ok = any(all(value_ok(v, frontmatter(f.read_text(encoding="utf-8", errors="replace")).get(k)) for k, v in arg["keys"].items())
for f in files(root, arg["path"]))
why = f"{arg['path']} {arg['keys']}"
elif kind == "unchanged":
bad = [f.relative_to(root).as_posix() for f in files(root, arg)
if base.get(f.relative_to(root).as_posix()) != hashlib.sha256(f.read_bytes()).hexdigest()]
gone = [p for p in base if glob_match(p, arg) and not (root / p).exists()]
ok, why = not bad and not gone, arg + (f" (changed: {', '.join(bad + gone)})" if bad or gone else "")
elif kind == "lint_clean":
if lint is None:
out = subprocess.run([sys.executable, "-I", str(LINT), "--json", "--scope", "wiki"], cwd=str(root),
capture_output=True, text=True).stdout
lint = json.loads(out)["trees"][0]
dirty = {k: lint.get(k, []) for k in arg if lint.get(k)}
ok, why = not dirty, ", ".join(arg) + (f" ({dirty})" if dirty else "")
else:
ok, why = False, f"unknown assertion kind {kind}"
results.append((ok, kind, why))
for ok, kind, why in results:
print(f"{'PASS' if ok else 'FAIL'} {kind}: {why}")
passed = sum(ok for ok, _, _ in results)
print(f"{case}: {passed}/{len(results)} passed")
return 0 if passed == len(results) else 1
IGNORED = ("tmp/", ".eval-baseline.json")
def record(case, target):
"""Copy every file the run added or changed into reference/, and list the
files it deleted in reference/.deleted, so `replay` can rebuild the result."""
root = Path(target).resolve()
if check(case, target) != 0:
sys.exit("error: only a passing run is recorded")
base = json.loads((root / BASELINE).read_text(encoding="utf-8"))
ref = EVALS / case / "reference"
if ref.exists():
shutil.rmtree(ref)
now = {p.relative_to(root).as_posix(): p for p in root.rglob("*")
if p.is_file() and not p.relative_to(root).as_posix().startswith(IGNORED)}
for rel, p in sorted(now.items()):
if base.get(rel) != hashlib.sha256(p.read_bytes()).hexdigest():
dst = ref / rel
dst.parent.mkdir(parents=True, exist_ok=True)
shutil.copy2(p, dst)
gone = sorted(rel for rel in base if rel not in now and not rel.startswith(IGNORED))
if gone:
ref.mkdir(parents=True, exist_ok=True)
(ref / ".deleted").write_text("\n".join(gone) + "\n", encoding="utf-8")
print(f"recorded {case}: {sum(1 for _ in ref.rglob('*') if _.is_file())} file(s) in {ref.relative_to(HERE.parent)}")
def replay(case, target):
ref = EVALS / case / "reference"
if not ref.is_dir():
sys.exit(f"error: {case} has no reference/")
prepare(case, target)
root = Path(target).resolve()
for p in ref.rglob("*"):
if p.is_file() and p.name != ".deleted":
dst = root / p.relative_to(ref)
dst.parent.mkdir(parents=True, exist_ok=True)
shutil.copy2(p, dst)
deleted = ref / ".deleted"
for rel in (deleted.read_text(encoding="utf-8").split() if deleted.is_file() else []):
if (root / rel).exists():
(root / rel).unlink()
return check(case, target)
def validate():
problems = []
for case in cases():
try:
c = load(case)
except ValueError as e:
problems.append(f"{case}: invalid JSON ({e})")
continue
for key in ("skill", "fixture", "prompt", "assert"):
if key not in c:
problems.append(f"{case}: missing `{key}`")
if not all(isinstance(r, str) for r in c.get("replies", [])):
problems.append(f"{case}: `replies` must be a list of strings")
for step in c.get("setup", []):
if not isinstance(step, dict) or not ({"write", "template", "upstream"} & set(step)):
problems.append(f"{case}: bad setup step {step}")
if not (FIXTURES / c.get("fixture", "?")).is_dir():
problems.append(f"{case}: fixture `{c.get('fixture')}` does not exist")
for a in c.get("assert", []):
if not isinstance(a, dict) or len(a) != 1 or next(iter(a)) not in KINDS:
problems.append(f"{case}: bad assertion {a}")
for p in problems:
print(p)
print(f"evals: {len(cases())} case(s), {len(problems)} problem(s)")
return 1 if problems else 0
def main(argv):
if not argv or argv[0] not in ("list", "prepare", "check", "validate", "record", "replay"):
print(__doc__)
return 2
if argv[0] == "list":
for case in cases():
print(f"{case}: {load(case)['prompt'].splitlines()[0][:90]}")
return 0
if argv[0] == "validate":
return validate()
if len(argv) != 3:
sys.exit(f"usage: run_evals.py {argv[0]} CASE DIR")
if argv[0] == "prepare":
prepare(argv[1], argv[2])
return 0
if argv[0] == "record":
record(argv[1], argv[2])
return 0
if argv[0] == "replay":
return replay(argv[1], argv[2])
return check(argv[1], argv[2])
if __name__ == "__main__":
sys.exit(main(sys.argv[1:]))