282 lines
12 KiB
Python
282 lines
12 KiB
Python
#!/usr/bin/env python3
|
|
"""Skill evals: structural checks of what an agent run left behind.
|
|
|
|
run_evals.py list # every case, with its prompt
|
|
run_evals.py prepare CASE DIR # copy the fixture + setup into DIR, print the prompt
|
|
run_evals.py check CASE DIR # score DIR after the agent ran there; exit 1 on a failure
|
|
run_evals.py validate # every case parses and names known assertion kinds
|
|
run_evals.py record CASE DIR # save what a passing run changed as the case's reference/
|
|
run_evals.py replay CASE DIR # prepare DIR, apply reference/ (incl. deletions), check
|
|
|
|
CASE is a path under tests/evals/, e.g. `ckb-decide/record`. A case may list
|
|
`replies`: what the operator says, in order, each time the agent stops and
|
|
waits (a check-in, a typed confirmation). `setup` steps, applied in order:
|
|
{"write": PATH, "content": TEXT} a file; "{upstream}" in TEXT is replaced
|
|
{"template": [PATH, ...], "ref": REF} copy template-repo paths into the
|
|
KB, from the working tree or a git REF
|
|
{"upstream": REF} a bare repo at tmp/eval-upstream.git whose
|
|
`main` is REF of this template repo The agent run
|
|
itself is manual or a CI job with a token budget: open DIR in the agent and
|
|
give it the prompt. Model output varies, so every assertion is structural —
|
|
files created or moved, frontmatter values, log entries, pages left alone,
|
|
lint checks clean — never the wording.
|
|
|
|
Assertion kinds (each one an object in the case's "assert" list):
|
|
file_exists: GLOB file_absent: GLOB
|
|
file_contains: {path: GLOB, regex: RE} some matching file matches RE
|
|
file_lacks: {path: GLOB, regex: RE} no matching file matches RE
|
|
frontmatter: {path: GLOB, keys: {k: v}} some matching file has every key;
|
|
v is exact, "*" (present) or "~RE"
|
|
unchanged: GLOB same bytes as when prepared
|
|
lint_clean: [check, ...] lint_report.py --json: these wiki checks have no findings
|
|
Python 3.8+, standard library only.
|
|
"""
|
|
import hashlib
|
|
import io
|
|
import json
|
|
import re
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import tarfile
|
|
from pathlib import Path
|
|
|
|
HERE = Path(__file__).resolve().parent
|
|
REPO = HERE.parent
|
|
EVALS = HERE / "evals"
|
|
FIXTURES = HERE / "fixtures"
|
|
LINT = HERE.parent / ".agents" / "skills" / "ckb-lint" / "scripts" / "lint_report.py"
|
|
BASELINE = ".eval-baseline.json"
|
|
KINDS = {"file_exists", "file_absent", "file_contains", "file_lacks", "frontmatter", "unchanged", "lint_clean"}
|
|
|
|
|
|
def load(case):
|
|
path = EVALS / case / "case.json"
|
|
if not path.is_file():
|
|
sys.exit(f"error: no case at {path}")
|
|
return json.loads(path.read_text(encoding="utf-8"))
|
|
|
|
|
|
def cases():
|
|
return sorted(p.parent.relative_to(EVALS).as_posix() for p in EVALS.glob("*/*/case.json"))
|
|
|
|
|
|
def glob_match(path, pattern):
|
|
"""Path.match treats ** as one segment before 3.13; mirror glob semantics instead."""
|
|
parts = re.split(r"(\*\*/|\*\*|\*|\?)", pattern)
|
|
rx = "".join("(?:.*/)?" if t == "**/" else ".*" if t == "**" else "[^/]*" if t == "*"
|
|
else "[^/]" if t == "?" else re.escape(t) for t in parts)
|
|
return re.fullmatch(rx, path) is not None
|
|
|
|
|
|
def files(root, pattern):
|
|
return sorted(p for p in root.glob(pattern) if p.is_file() and BASELINE not in p.name)
|
|
|
|
|
|
def frontmatter(text):
|
|
if not text.startswith("---\n"):
|
|
return {}
|
|
close = re.compile(r"^---[ \t]*$", re.M).search(text, 4) # may be the last line
|
|
out = {}
|
|
for line in text[4:close.start() if close else 4].split("\n"):
|
|
k, sep, v = line.partition(":")
|
|
if sep and not line[:1].isspace():
|
|
out[k.strip()] = v.strip().strip("\"'")
|
|
return out
|
|
|
|
|
|
def value_ok(want, got):
|
|
if got is None:
|
|
return False
|
|
if want == "*":
|
|
return True
|
|
if isinstance(want, str) and want.startswith("~"):
|
|
return re.search(want[1:], got) is not None
|
|
return str(want) == got
|
|
|
|
|
|
def prepare(case, target):
|
|
c, root = load(case), Path(target).resolve()
|
|
if root.exists() and any(root.iterdir()):
|
|
sys.exit(f"error: {root} is not empty")
|
|
shutil.copytree(FIXTURES / c["fixture"], root, dirs_exist_ok=True)
|
|
upstream = root / "tmp" / "eval-upstream.git"
|
|
for step in c.get("setup", []):
|
|
if "write" in step:
|
|
f = root / step["write"]
|
|
f.parent.mkdir(parents=True, exist_ok=True)
|
|
f.write_text(step["content"].replace("{upstream}", upstream.as_posix()), encoding="utf-8")
|
|
elif "template" in step:
|
|
copy_template(step["template"], root, step.get("ref"))
|
|
elif "upstream" in step:
|
|
upstream.parent.mkdir(parents=True, exist_ok=True)
|
|
subprocess.run(["git", "init", "-q", "--bare", str(upstream)], check=True)
|
|
subprocess.run(["git", "-C", str(REPO), "push", "-q", str(upstream), f"{step['upstream']}:refs/heads/main"],
|
|
check=True)
|
|
subprocess.run(["git", "-C", str(upstream), "symbolic-ref", "HEAD", "refs/heads/main"], check=True)
|
|
base = {p.relative_to(root).as_posix(): hashlib.sha256(p.read_bytes()).hexdigest()
|
|
for p in root.rglob("*") if p.is_file()}
|
|
(root / BASELINE).write_text(json.dumps(base, indent=1), encoding="utf-8")
|
|
print(f"prepared {root} from fixtures/{c['fixture']}\nprompt ({c['skill']}):\n\n{c['prompt']}")
|
|
for n, reply in enumerate(c.get("replies", []), 1):
|
|
print(f"\nreply {n}, when the agent stops and waits:\n\n{reply}")
|
|
|
|
|
|
def copy_template(paths, root, ref=None):
|
|
"""Template-repo files into the scratch KB: from the working tree, or from
|
|
a git ref (an older release, for upgrade cases)."""
|
|
if ref is None:
|
|
for rel in paths:
|
|
src, dst = REPO / rel, root / rel
|
|
dst.parent.mkdir(parents=True, exist_ok=True)
|
|
if src.is_dir():
|
|
shutil.copytree(src, dst, dirs_exist_ok=True, symlinks=True)
|
|
else:
|
|
shutil.copy2(src, dst, follow_symlinks=False)
|
|
return
|
|
data = subprocess.run(["git", "-C", str(REPO), "archive", "--format=tar", ref, "--", *paths],
|
|
capture_output=True, check=True).stdout
|
|
with tarfile.open(fileobj=io.BytesIO(data)) as tar:
|
|
for m in tar.getmembers():
|
|
if m.name.startswith(("/", "..")) or "/../" in m.name:
|
|
continue
|
|
tar.extract(m, root)
|
|
|
|
|
|
def check(case, target):
|
|
c, root = load(case), Path(target).resolve()
|
|
base = json.loads((root / BASELINE).read_text(encoding="utf-8")) if (root / BASELINE).is_file() else {}
|
|
lint = None
|
|
results = []
|
|
for a in c["assert"]:
|
|
(kind, arg), = a.items()
|
|
if kind == "file_exists":
|
|
ok, why = bool(files(root, arg)), arg
|
|
elif kind == "file_absent":
|
|
ok, why = not files(root, arg), arg
|
|
elif kind in ("file_contains", "file_lacks"):
|
|
hit = any(re.search(arg["regex"], f.read_text(encoding="utf-8", errors="replace"), re.M) for f in files(root, arg["path"]))
|
|
ok, why = (hit if kind == "file_contains" else not hit), f"{arg['path']} /{arg['regex']}/"
|
|
elif kind == "frontmatter":
|
|
ok = any(all(value_ok(v, frontmatter(f.read_text(encoding="utf-8", errors="replace")).get(k)) for k, v in arg["keys"].items())
|
|
for f in files(root, arg["path"]))
|
|
why = f"{arg['path']} {arg['keys']}"
|
|
elif kind == "unchanged":
|
|
bad = [f.relative_to(root).as_posix() for f in files(root, arg)
|
|
if base.get(f.relative_to(root).as_posix()) != hashlib.sha256(f.read_bytes()).hexdigest()]
|
|
gone = [p for p in base if glob_match(p, arg) and not (root / p).exists()]
|
|
ok, why = not bad and not gone, arg + (f" (changed: {', '.join(bad + gone)})" if bad or gone else "")
|
|
elif kind == "lint_clean":
|
|
if lint is None:
|
|
out = subprocess.run([sys.executable, "-I", str(LINT), "--json", "--scope", "wiki"], cwd=str(root),
|
|
capture_output=True, text=True).stdout
|
|
lint = json.loads(out)["trees"][0]
|
|
dirty = {k: lint.get(k, []) for k in arg if lint.get(k)}
|
|
ok, why = not dirty, ", ".join(arg) + (f" ({dirty})" if dirty else "")
|
|
else:
|
|
ok, why = False, f"unknown assertion kind {kind}"
|
|
results.append((ok, kind, why))
|
|
for ok, kind, why in results:
|
|
print(f"{'PASS' if ok else 'FAIL'} {kind}: {why}")
|
|
passed = sum(ok for ok, _, _ in results)
|
|
print(f"{case}: {passed}/{len(results)} passed")
|
|
return 0 if passed == len(results) else 1
|
|
|
|
|
|
IGNORED = ("tmp/", ".eval-baseline.json")
|
|
|
|
|
|
def record(case, target):
|
|
"""Copy every file the run added or changed into reference/, and list the
|
|
files it deleted in reference/.deleted, so `replay` can rebuild the result."""
|
|
root = Path(target).resolve()
|
|
if check(case, target) != 0:
|
|
sys.exit("error: only a passing run is recorded")
|
|
base = json.loads((root / BASELINE).read_text(encoding="utf-8"))
|
|
ref = EVALS / case / "reference"
|
|
if ref.exists():
|
|
shutil.rmtree(ref)
|
|
now = {p.relative_to(root).as_posix(): p for p in root.rglob("*")
|
|
if p.is_file() and not p.relative_to(root).as_posix().startswith(IGNORED)}
|
|
for rel, p in sorted(now.items()):
|
|
if base.get(rel) != hashlib.sha256(p.read_bytes()).hexdigest():
|
|
dst = ref / rel
|
|
dst.parent.mkdir(parents=True, exist_ok=True)
|
|
shutil.copy2(p, dst)
|
|
gone = sorted(rel for rel in base if rel not in now and not rel.startswith(IGNORED))
|
|
if gone:
|
|
ref.mkdir(parents=True, exist_ok=True)
|
|
(ref / ".deleted").write_text("\n".join(gone) + "\n", encoding="utf-8")
|
|
print(f"recorded {case}: {sum(1 for _ in ref.rglob('*') if _.is_file())} file(s) in {ref.relative_to(HERE.parent)}")
|
|
|
|
|
|
def replay(case, target):
|
|
ref = EVALS / case / "reference"
|
|
if not ref.is_dir():
|
|
sys.exit(f"error: {case} has no reference/")
|
|
prepare(case, target)
|
|
root = Path(target).resolve()
|
|
for p in ref.rglob("*"):
|
|
if p.is_file() and p.name != ".deleted":
|
|
dst = root / p.relative_to(ref)
|
|
dst.parent.mkdir(parents=True, exist_ok=True)
|
|
shutil.copy2(p, dst)
|
|
deleted = ref / ".deleted"
|
|
for rel in (deleted.read_text(encoding="utf-8").split() if deleted.is_file() else []):
|
|
if (root / rel).exists():
|
|
(root / rel).unlink()
|
|
return check(case, target)
|
|
|
|
|
|
def validate():
|
|
problems = []
|
|
for case in cases():
|
|
try:
|
|
c = load(case)
|
|
except ValueError as e:
|
|
problems.append(f"{case}: invalid JSON ({e})")
|
|
continue
|
|
for key in ("skill", "fixture", "prompt", "assert"):
|
|
if key not in c:
|
|
problems.append(f"{case}: missing `{key}`")
|
|
if not all(isinstance(r, str) for r in c.get("replies", [])):
|
|
problems.append(f"{case}: `replies` must be a list of strings")
|
|
for step in c.get("setup", []):
|
|
if not isinstance(step, dict) or not ({"write", "template", "upstream"} & set(step)):
|
|
problems.append(f"{case}: bad setup step {step}")
|
|
if not (FIXTURES / c.get("fixture", "?")).is_dir():
|
|
problems.append(f"{case}: fixture `{c.get('fixture')}` does not exist")
|
|
for a in c.get("assert", []):
|
|
if not isinstance(a, dict) or len(a) != 1 or next(iter(a)) not in KINDS:
|
|
problems.append(f"{case}: bad assertion {a}")
|
|
for p in problems:
|
|
print(p)
|
|
print(f"evals: {len(cases())} case(s), {len(problems)} problem(s)")
|
|
return 1 if problems else 0
|
|
|
|
|
|
def main(argv):
|
|
if not argv or argv[0] not in ("list", "prepare", "check", "validate", "record", "replay"):
|
|
print(__doc__)
|
|
return 2
|
|
if argv[0] == "list":
|
|
for case in cases():
|
|
print(f"{case}: {load(case)['prompt'].splitlines()[0][:90]}")
|
|
return 0
|
|
if argv[0] == "validate":
|
|
return validate()
|
|
if len(argv) != 3:
|
|
sys.exit(f"usage: run_evals.py {argv[0]} CASE DIR")
|
|
if argv[0] == "prepare":
|
|
prepare(argv[1], argv[2])
|
|
return 0
|
|
if argv[0] == "record":
|
|
record(argv[1], argv[2])
|
|
return 0
|
|
if argv[0] == "replay":
|
|
return replay(argv[1], argv[2])
|
|
return check(argv[1], argv[2])
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main(sys.argv[1:]))
|