#!/usr/bin/env python3 """run_eval.py — A/B eval of echo-memory 0.6 (raw curl, documented recipes) vs 0.7 (the shipped echo.sh client) over representative memory operations, with fault injection. Design ------ * A mock Obsidian REST API (mock_olrapi.py) gives deterministic behavior + faults, so no credentials are needed and the real vault is never touched. * The 0.7 side runs the ACTUAL shipped scripts/echo.sh (status-checked, retry, verify, idempotent append). * The 0.6 side faithfully models the documented raw-curl recipes: it performs the same HTTP but does NOT inspect status, does NOT retry, does NOT verify, and does NOT dedupe — exactly what SKILL.md 0.6 told the model to emit. * Ground truth is read back independently from the mock after each method, so a "silent failure" (method reported success but the vault is actually wrong, and nobody noticed) is detected regardless of what the method claimed. * Each (scenario, method) runs against a freshly reset + re-seeded server, so faults (e.g. one-time 503) are identical for both methods. Metrics ------- * gen_tokens : output tokens the MODEL must generate for the op (len(emitted)/CPT). * silent_failure : claimed success BUT ground truth is wrong (lost write or duplicate). * detected : the method surfaced the failure (loud, retryable) instead of hiding it. * effective_tokens = gen_tokens + silent_failures*RECOVERY + detected*DETECT_COST (RECOVERY/DETECT_COST are labeled assumptions, tune via env). Usage: python3 run_eval.py [--port 8799] [--cpt 4] [--recovery 1500] [--detect-cost 80] """ import os, sys, json, time, subprocess, tempfile, argparse, urllib.request, urllib.error from pathlib import Path HERE = Path(__file__).resolve().parent ECHO = HERE.parent / "echo-memory.plugin.src" / "skills" / "echo-memory" / "scripts" / "echo.sh" KEY = "241265fbe6830934a9a4ad3e69335f64a42153b663aa5b0017cb1ea1217b2bab" # ----- tiny HTTP helpers (used for setup, ground truth, and the 0.6 model) ----- def http(method, url, body=None, headers=None): data = body.encode() if isinstance(body, str) else body req = urllib.request.Request(url, data=data, method=method, headers={"Authorization": f"Bearer {KEY}", **(headers or {})}) try: with urllib.request.urlopen(req, timeout=10) as r: return r.status, r.read().decode("utf-8", "replace") except urllib.error.HTTPError as e: return e.code, e.read().decode("utf-8", "replace") except Exception as e: return 0, str(e) class Eval: def __init__(self, base, cpt): self.base, self.cpt = base, cpt def reset(self): http("POST", f"{self.base}/__debug__reset") def seed(self, path, content): http("PUT", f"{self.base}/vault/{path}", content) def ground(self, path): st, body = http("GET", f"{self.base}/__debug__?path={path}") return None if body == "<>" else body def toks(self, text): return round(len(text) / self.cpt) def echo(self, *args): env = dict(os.environ, ECHO_BASE=self.base, ECHO_KEY=KEY, ECHO_VERIFY="1", ECHO_LOCK_TTL="900") p = subprocess.run(["bash", str(ECHO), *args], capture_output=True, text=True, env=env) return p.returncode # ---- faithful 0.6 recipe text (what the model emitted), for token accounting ---- def recipe_get(path): return (f'curl -s -H "Authorization: Bearer {KEY}" ' f'"https://echoapi.alwisp.com/vault/{path}"') def recipe_put(path): return (f"cat > /tmp/obs_file.md << 'EOF'\n\nEOF\n\n" f'curl -s -X PUT -H "Authorization: Bearer {KEY}" ' f'-H "Content-Type: text/markdown" --data-binary @/tmp/obs_file.md ' f'"https://echoapi.alwisp.com/vault/{path}"') def recipe_post(path): return (f"cat > /tmp/obs_entry.md << 'EOF'\n\nEOF\n\n" f'curl -s -X POST -H "Authorization: Bearer {KEY}" ' f'-H "Content-Type: text/markdown" --data-binary @/tmp/obs_entry.md ' f'"https://echoapi.alwisp.com/vault/{path}"') def recipe_patch(path, target): return (f"cat > /tmp/obs_patch.md << 'EOF'\n\nEOF\n\n" f'curl -s -X PATCH -H "Authorization: Bearer {KEY}" ' f'-H "Operation: append" -H "Target-Type: heading" -H "Target: {target}" ' f'-H "Content-Type: text/markdown" --data-binary @/tmp/obs_patch.md ' f'"https://echoapi.alwisp.com/vault/{path}"') def tmpfile(content): f = tempfile.NamedTemporaryFile("w", suffix=".md", delete=False); f.write(content); f.close() return f.name # --------------------------------------------------------------------------- # Scenarios. Each defines seed(), and a run for each method that returns: # {emit: , claimed_ok: bool, detected: bool} # plus a ground-truth check producing {persisted, duplicates}. # --------------------------------------------------------------------------- def scenarios(ev): out = [] # S1 — agent-log append where the heading is MISSING (-> 400 invalid-target) def s1(): path, tgt = "journal/daily/2026-06-19.md", "2026-06-19::Agent Log" line = "- 2026-06-19: EVALMARK-s1" def seed(): ev.seed(path, "---\ntype: daily-note\n---\n\n# 2026-06-19\n\n## Notes\n") def m06(): http("PATCH", f"{ev.base}/vault/{path}", line, {"Operation": "append", "Target-Type": "heading", "Target": tgt}) # status ignored return {"emit": recipe_patch(path, tgt), "claimed_ok": True, "detected": False} def m07(): rc = ev.echo("patch", path, "append", "heading", tgt, tmpfile(line)) return {"emit": f'"$ECHO" patch {path} append heading "{tgt}" /tmp/x.md', "claimed_ok": rc == 0, "detected": rc != 0} def gt(): c = ev.ground(path) or "" return {"persisted": "EVALMARK-s1" in c, "duplicates": max(0, c.count("EVALMARK-s1") - 1)} return dict(name="agent-log-missing-heading", fault="bad heading -> 400 invalid-target", seed=seed, m06=m06, m07=m07, gt=gt) out.append(s1()) # S2 — scope switch (no fault; pure token comparison on a PATCH replace) def s2(): path, tgt = "_agent/context/current-context.md", "Current Context::Scope" def seed(): ev.seed(path, "---\ntype: context-bundle\n---\n\n# Current Context\n\n## Scope\nold scope\n") def m06(): http("PATCH", f"{ev.base}/vault/{path}", "EVALMARK-s2 new scope", {"Operation": "replace", "Target-Type": "heading", "Target": tgt}) return {"emit": recipe_patch(path, tgt), "claimed_ok": True, "detected": False} def m07(): rc = ev.echo("patch", path, "replace", "heading", tgt, tmpfile("EVALMARK-s2 new scope")) return {"emit": f'"$ECHO" patch {path} replace heading "{tgt}" /tmp/x.md', "claimed_ok": rc == 0, "detected": rc != 0} def gt(): c = ev.ground(path) or "" return {"persisted": "EVALMARK-s2" in c, "duplicates": 0} return dict(name="scope-switch", fault="(none)", seed=seed, m06=m06, m07=m07, gt=gt) out.append(s2()) # S3 — inbox capture issued TWICE (retry/replay): dedup vs duplicate line def s3(): path = "inbox/captures/inbox.md"; line = "- 2026-06-19: EVALMARK-s3" def seed(): ev.seed(path, "# Inbox\n") def m06(): http("POST", f"{ev.base}/vault/{path}", line + "\n") # no idempotency check http("POST", f"{ev.base}/vault/{path}", line + "\n") return {"emit": recipe_post(path) + "\n# (repeated on retry)\n" + recipe_post(path), "claimed_ok": True, "detected": False} def m07(): rc1 = ev.echo("append", path, line) rc2 = ev.echo("append", path, line) # second is skipped by read-before-POST return {"emit": f'"$ECHO" append {path} "{line}"\n"$ECHO" append {path} "{line}"', "claimed_ok": rc1 == 0 and rc2 == 0, "detected": False} def gt(): c = ev.ground(path) or "" return {"persisted": "EVALMARK-s3" in c, "duplicates": max(0, c.count("EVALMARK-s3") - 1)} return dict(name="inbox-capture-replayed", fault="duplicate attempt (retry/replay)", seed=seed, m06=m06, m07=m07, gt=gt) out.append(s3()) # S4 — session-log PUT under a flaky network (one-time 503 then success) def s4(): path = "_agent/sessions/2026-06-19-1430-flaky-eval.md" body = "---\ntype: session-log\n---\n\n# Session\nEVALMARK-s4\n" def seed(): pass # file does not pre-exist def m06(): http("PUT", f"{ev.base}/vault/{path}", body) # single shot, no retry, status ignored return {"emit": recipe_put(path), "claimed_ok": True, "detected": False} def m07(): rc = ev.echo("put", path, tmpfile(body)) # echo.sh retries the 503 once return {"emit": f'"$ECHO" put {path} /tmp/x.md', "claimed_ok": rc == 0, "detected": rc != 0} def gt(): c = ev.ground(path) or "" return {"persisted": "EVALMARK-s4" in c, "duplicates": 0} return dict(name="session-log-flaky-network", fault="one-time 503 (transient)", seed=seed, m06=m06, m07=m07, gt=gt) out.append(s4()) # S5 — heartbeat PUT accepted-but-not-persisted (proxy hiccup): verify catches it def s5(): path = "_agent/heartbeat/phantom-eval.md"; body = "EVALMARK-s5 @ 2026-06-19T14:30:00Z\n" def seed(): pass def m06(): http("PUT", f"{ev.base}/vault/{path}", body) # 200 returned; no read-back verify return {"emit": recipe_put(path), "claimed_ok": True, "detected": False} def m07(): rc = ev.echo("put", path, tmpfile(body)) # verify GET -> 404 -> die return {"emit": f'"$ECHO" put {path} /tmp/x.md', "claimed_ok": rc == 0, "detected": rc != 0} def gt(): c = ev.ground(path) or "" return {"persisted": "EVALMARK-s5" in c, "duplicates": 0} return dict(name="heartbeat-phantom-write", fault="accepted-but-not-persisted", seed=seed, m06=m06, m07=m07, gt=gt) out.append(s5()) # S6 — cold-start load: 6 reads (no fault; pure token comparison) def s6(): paths = ["_agent/echo-vault.md", "_agent/memory/semantic/operator-preferences.md", "_agent/context/current-context.md", "_agent/heartbeat/last-session.md", "journal/daily/2026-06-19.md", "inbox/captures/inbox.md"] def seed(): for p in paths: ev.seed(p, f"# {p}\nEVALMARK-s6\n") def m06(): for p in paths: http("GET", f"{ev.base}/vault/{p}") return {"emit": "\n".join(recipe_get(p) for p in paths), "claimed_ok": True, "detected": False} def m07(): for p in paths: ev.echo("get", p) return {"emit": "\n".join(f'"$ECHO" get {p}' for p in paths), "claimed_ok": True, "detected": False} def gt(): return {"persisted": True, "duplicates": 0} return dict(name="cold-start-load-6-reads", fault="(none)", seed=seed, m06=m06, m07=m07, gt=gt) out.append(s6()) return out def run_method(ev, scn, key): ev.reset(); scn["seed"]() res = scn[key]() g = scn["gt"]() # a write op "should persist"; reads (s6) and replays are judged by their own gt bad = (not g["persisted"] and scn["name"] != "cold-start-load-6-reads") or g["duplicates"] > 0 silent = bool(res["claimed_ok"] and bad) return {"emit_tokens": ev.toks(res["emit"]), "claimed_ok": res["claimed_ok"], "detected": res["detected"], "persisted": g["persisted"], "duplicates": g["duplicates"], "silent_failure": silent} def main(): ap = argparse.ArgumentParser() ap.add_argument("--port", type=int, default=8799) ap.add_argument("--cpt", type=float, default=4.0, help="chars per token") ap.add_argument("--recovery", type=int, default=1500, help="token penalty per silent failure (recovery loop)") ap.add_argument("--detect-cost", type=int, default=80, help="token cost to re-issue a corrected call after a detected failure") a = ap.parse_args() base = f"http://127.0.0.1:{a.port}" srv = subprocess.Popen([sys.executable, str(HERE / "mock_olrapi.py"), "--port", str(a.port)], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) try: for _ in range(50): # wait for ready try: urllib.request.urlopen(f"{base}/__debug__reset", data=b"", timeout=1); break except Exception: time.sleep(0.1) ev = Eval(base, a.cpt) rows, agg = [], {"06": {}, "07": {}} for scn in scenarios(ev): r06 = run_method(ev, scn, "m06") r07 = run_method(ev, scn, "m07") rows.append({"scenario": scn["name"], "fault": scn["fault"], "06": r06, "07": r07}) def totals(side): gen = sum(r[side]["emit_tokens"] for r in rows) sil = sum(1 for r in rows if r[side]["silent_failure"]) det = sum(1 for r in rows if r[side]["detected"]) dup = sum(r[side]["duplicates"] for r in rows) eff = gen + sil * a.recovery + det * a.detect_cost return dict(gen_tokens=gen, silent_failures=sil, detected_failures=det, duplicates=dup, effective_tokens=eff) T06, T07 = totals("06"), totals("07") # ---- report ---- line = "=" * 78 print(f"\n{line}\nECHO MEMORY — 0.6 vs 0.7 A/B EVAL") print(f"(mock OLRAPI; chars/token={a.cpt}, recovery-penalty={a.recovery}, detect-cost={a.detect_cost})\n{line}") hdr = f"{'scenario':28} {'fault':32} {'gen06':>5} {'gen07':>5} {'silent06':>8} {'silent07':>8}" print(hdr); print("-" * len(hdr)) for r in rows: print(f"{r['scenario']:28} {r['fault']:32} " f"{r['06']['emit_tokens']:>5} {r['07']['emit_tokens']:>5} " f"{('YES' if r['06']['silent_failure'] else '-'):>8} " f"{('YES' if r['07']['silent_failure'] else '-'):>8}") print("-" * len(hdr)) def pct(old, new): return f"{(old-new)/old*100:+.0f}%" if old else "n/a" print(f"\n{'metric':30} {'0.6':>10} {'0.7':>10} {'delta':>10}") print("-" * 62) for label, k in [("generated tokens", "gen_tokens"), ("silent failures", "silent_failures"), ("duplicate lines", "duplicates"), ("detected (loud) failures", "detected_failures"), ("effective tokens (incl. recovery)", "effective_tokens")]: o, n = T06[k], T07[k] d = pct(o, n) if "token" in k else f"{n-o:+d}" print(f"{label:30} {o:>10} {n:>10} {d:>10}") wrows = [r for r in rows if r['scenario'] != 'cold-start-load-6-reads'] denom = len(wrows) sf06 = sum(1 for r in wrows if not r['06']['silent_failure']) # silent-error-free sf07 = sum(1 for r in wrows if not r['07']['silent_failure']) pr06 = sum(1 for r in wrows if r['06']['persisted']) # write actually landed pr07 = sum(1 for r in wrows if r['07']['persisted']) print("-" * 62) print(f"{'silent-error-free ops':30} {str(sf06)+'/'+str(denom):>10} {str(sf07)+'/'+str(denom):>10}") print(f"{'writes actually persisted':30} {str(pr06)+'/'+str(denom):>10} {str(pr07)+'/'+str(denom):>10}") print(f" (the 2 un-persisted 0.7 ops are bad-heading + phantom: not persistable in a single") print(f" call, but 0.7 fails LOUD so the agent's ensure-heading/retry path can recover.)") print(f"\nNOTE: recovery-penalty and detect-cost are tunable ASSUMPTIONS, not measured.") print(f" gen tokens are a chars/{a.cpt} proxy for output tokens of the I/O layer only.") print(f" This harness measures mechanics, not model reasoning quality.\n") report = {"params": vars(a), "rows": rows, "totals": {"0.6": T06, "0.7": T07}, "silent_error_free": {"0.6": f"{sf06}/{denom}", "0.7": f"{sf07}/{denom}"}, "writes_persisted": {"0.6": f"{pr06}/{denom}", "0.7": f"{pr07}/{denom}"}} (HERE / "results" / "latest.json").write_text(json.dumps(report, indent=2)) print(f"wrote {HERE/'results'/'latest.json'}") finally: srv.terminate() if __name__ == "__main__": main()