echo/eval/test_features.py

#!/usr/bin/env python3
"""test_features.py — end-to-end test of the v0.9 features against the mock OLRAPI.

Exercises the real shipped scripts (echo.py capture/resolve/recall/link + sweep.py)
through subprocess, then reads ground truth back from the mock. No network, no creds,
no live vault.

Run:  python test_features.py [--port 8801]
"""
import argparse, json, os, subprocess, sys, time, urllib.request, urllib.error
from pathlib import Path

HERE = Path(__file__).resolve().parent
SCRIPTS = HERE.parent / "echo-memory.plugin.src" / "skills" / "echo-memory" / "scripts"
ECHO = SCRIPTS / "echo.py"
SWEEP = SCRIPTS / "sweep.py"
KEY = "test-key-not-a-real-secret"

failures = []


def check(name, cond, detail=""):
    print(f"{'ok  ' if cond else 'FAIL'} {name}" + (f"  -- {detail}" if not cond else ""))
    if not cond:
        failures.append(name)


class Harness:
    def __init__(self, base):
        self.base = base

    def http(self, method, url, body=None, headers=None):
        data = body.encode() if isinstance(body, str) else body
        req = urllib.request.Request(url, data=data, method=method,
                                     headers={"Authorization": f"Bearer {KEY}", **(headers or {})})
        try:
            with urllib.request.urlopen(req, timeout=10) as r:
                return r.status, r.read().decode("utf-8", "replace")
        except urllib.error.HTTPError as e:
            return e.code, e.read().decode("utf-8", "replace")
        except Exception as e:
            return 0, str(e)

    def reset(self):
        self.http("POST", f"{self.base}/__debug__reset")

    def seed(self, path, content):
        self.http("PUT", f"{self.base}/vault/{path}", content)

    def ground(self, path):
        _, body = self.http("GET", f"{self.base}/__debug__?path={path}")
        return None if body == "<<MISSING>>" else body

    def echo(self, *args):
        env = dict(os.environ, ECHO_BASE=self.base, ECHO_KEY=KEY, ECHO_VERIFY="1",
                   ECHO_TODAY="2026-06-21")
        return subprocess.run([sys.executable, str(args[0]), *args[1:]],
                              capture_output=True, text=True, env=env)


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--port", type=int, default=8801)
    a = ap.parse_args()
    base = f"http://127.0.0.1:{a.port}"
    srv = subprocess.Popen([sys.executable, str(HERE / "mock_olrapi.py"), "--port", str(a.port)],
                           stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)
    try:
        for _ in range(50):
            try:
                urllib.request.urlopen(f"{base}/__debug__reset", data=b"", timeout=1); break
            except Exception:
                time.sleep(0.1)
        h = Harness(base)
        h.reset()
        h.seed("_agent/echo-vault.md", "---\nschema_version: 2\n---\n# marker\n")

        # 1. capture a person (with an alias)
        r = h.echo(ECHO, "capture", "Bob Smith", "--kind", "person", "--aliases", "bob,rs")
        note = h.ground("resources/people/bob-smith.md")
        check("capture person creates note", note is not None and "type: person" in note, r.stderr)
        check("capture stamps agent_written", note and "agent_written: true" in note)
        idx = h.ground("_agent/index/entities.json")
        check("index records entity", idx and "bob-smith" in idx and '"bob"' in idx)

        # 2. resolve by alias
        r = h.echo(ECHO, "resolve", "bob")
        out = json.loads(r.stdout) if r.stdout.strip().startswith("{") else {}
        check("resolve alias -> path", out.get("match") and out.get("path") == "resources/people/bob-smith.md", r.stdout)

        # 3. capture a company whose body mentions Bob Smith -> auto bidirectional link
        r = subprocess.run([sys.executable, str(ECHO), "capture", "MPM", "--kind", "company", "-"],
                           input="Bob Smith is the principal here.\n", capture_output=True, text=True,
                           env=dict(os.environ, ECHO_BASE=base, ECHO_KEY=KEY, ECHO_TODAY="2026-06-21"))
        mpm = h.ground("resources/companies/mpm.md")
        bob = h.ground("resources/people/bob-smith.md")
        check("auto-link forward (mpm -> bob)", mpm and "[[resources/people/bob-smith]]" in mpm, r.stdout + r.stderr)
        check("auto-link reciprocal (bob -> mpm)", bob and "[[resources/companies/mpm]]" in bob)

        # 4. recall an entity surfaces its linked neighborhood (search is mocked empty,
        #    so this exercises the index-resolve + 1-hop expansion path)
        r = h.echo(ECHO, "recall", "MPM")
        check("recall surfaces linked person", "resources/people/bob-smith.md" in r.stdout, r.stdout)

        # 4b. BM25 finds a note by a BODY term. /search/simple is mocked empty, so a hit
        #     here can only come from the local BM25 recall index (proves H1 lexical layer).
        r = h.echo(ECHO, "recall", "principal")
        check("recall finds note by body term (BM25)",
              "resources/companies/mpm.md" in r.stdout, r.stdout)

        # 5. explicit link between two fresh notes
        h.echo(ECHO, "capture", "Alpha", "--kind", "concept")
        h.echo(ECHO, "capture", "Beta", "--kind", "concept")
        h.echo(ECHO, "link", "resources/concepts/alpha.md", "resources/concepts/beta.md")
        alpha = h.ground("resources/concepts/alpha.md")
        beta = h.ground("resources/concepts/beta.md")
        check("link adds A->B", alpha and "[[resources/concepts/beta]]" in alpha)
        check("link adds B->A", beta and "[[resources/concepts/alpha]]" in beta)

        # 6. inbox capture (unknown home)
        h.echo(ECHO, "capture", "stray thought", "--inbox")
        inbox = h.ground("inbox/captures/inbox.md")
        check("inbox capture lands", inbox and "stray thought" in inbox)

        # 7. sweep --apply rebuilds the index, symmetrizes, stamps schema 3
        r = h.echo(SWEEP, "--apply")
        marker = h.ground("_agent/echo-vault.md")
        check("sweep runs clean", r.returncode == 0, r.stdout + r.stderr)
        check("sweep stamps schema 4", marker and "schema_version=4" in marker.replace(": ", "="), marker)
        idx2 = h.ground("_agent/index/entities.json")
        check("sweep rebuilt index has all entities",
              idx2 and all(s in idx2 for s in ["bob-smith", "mpm", "alpha", "beta"]))
        rix = h.ground("_agent/index/recall-index.json")
        check("sweep builds the recall (BM25) index",
              rix is not None and '"postings"' in rix and "principal" in rix, rix)

        # 8. H3 — atomic_index_update re-reads fresh and MERGES, so an entry written by a
        #    "concurrent" session survives our write; and the advisory lock is released after.
        h.http("PUT", f"{base}/vault/_agent/index/entities.json",
               '{"schema":1,"updated":"2026-06-21","entities":{"ext-entity":'
               '{"path":"resources/concepts/ext-entity.md","kind":"concept","title":"Ext",'
               '"aliases":[],"last_seen":"2026-06-21"}}}')
        env = dict(os.environ, ECHO_BASE=base, ECHO_KEY=KEY, ECHO_TODAY="2026-06-21", PYTHONPATH=str(SCRIPTS))
        snippet = ("import echo_index as ix, echo_concurrency as c; "
                   "c.atomic_index_update(lambda d: ix.upsert(d,'mine',"
                   "'resources/concepts/mine.md','concept','Mine'))")
        subprocess.run([sys.executable, "-c", snippet], env=env, capture_output=True, text=True)
        idx3 = h.ground("_agent/index/entities.json")
        check("H3 atomic update merges a concurrent entry (no clobber)",
              bool(idx3) and "ext-entity" in idx3 and "mine" in idx3, idx3)
        check("H3 advisory lock released after the transaction",
              h.ground("_agent/locks/vault.lock") is None)

        # 9. M4 — --dry-run previews and writes nothing; --json emits a parseable envelope.
        r = h.echo(ECHO, "capture", "DryRun Co", "--kind", "company", "--dry-run", "--json")
        try:
            env = json.loads(r.stdout)
        except Exception:
            env = {}
        check("M4 --dry-run emits a dry-run JSON envelope",
              env.get("action", "").startswith("dry-run") and env.get("path") == "resources/companies/dryrun-co.md", r.stdout)
        check("M4 --dry-run writes nothing", h.ground("resources/companies/dryrun-co.md") is None)
        r = h.echo(ECHO, "capture", "Json Co", "--kind", "company", "--json")
        try:
            env = json.loads(r.stdout)
        except Exception:
            env = {}
        check("M4 --json envelope reports created + path",
              env.get("ok") is True and env.get("action") == "created"
              and env.get("path") == "resources/companies/json-co.md", r.stdout)
        check("M4 --json capture actually wrote the note", h.ground("resources/companies/json-co.md") is not None)

        print(f"\n{len(failures)} failure(s)" if failures else "\nall feature tests passed")
        return 1 if failures else 0
    finally:
        srv.terminate()


if __name__ == "__main__":
    raise SystemExit(main())