diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..d3a1b34 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,48 @@ +name: ci + +# H4 — gate every push on the credential-free, offline test suite. +# No secrets, no live vault: everything runs against the mock OLRAPI. + +on: + push: + pull_request: + workflow_dispatch: + +jobs: + test: + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest, macos-latest] + python-version: ["3.10", "3.12"] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Routing docs in sync with routing.json (offline) + run: python echo-memory.plugin.src/skills/echo-memory/scripts/check_routing.py + + - name: Interface guard for v1.0 scaffolds + run: python echo-memory.plugin.src/skills/echo-memory/scripts/test_v1_scaffold.py + + - name: Client regression tests + run: python echo-memory.plugin.src/skills/echo-memory/scripts/test_echo_client.py + + - name: Feature integration tests (mock OLRAPI) + run: python eval/test_features.py + + - name: PATCH-semantics tests (hi-fi mock) + run: python eval/test_patch_semantics.py + + - name: Offline write-ahead queue + cache tests (H2) + run: python eval/test_offline_queue.py + + - name: Session-reflection capture tests (H5) + run: python eval/test_reflect.py + + # TODO (H4): fold the hi-fi mock into the main eval and publish the refreshed + # eval/results/latest.json (currently still a 0.6-vs-0.7 baseline) as an artifact: + # python eval/run_eval.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..708bdf9 --- /dev/null +++ b/.gitignore @@ -0,0 +1,17 @@ +# macOS / editor cruft +.DS_Store + +# Python +__pycache__/ +*.py[cod] +*.egg-info/ +.venv/ +venv/ + +# Local ECHO state (never commit — holds the offline queue + cached vault reads, +# and may hold the API key in ~/.echo-memory/credentials when not using ECHO_KEY) +.echo-memory/ + +# Eval output +eval/results/*.json +!eval/results/.gitkeep diff --git a/API-KEY-SETUP.md b/API-KEY-SETUP.md new file mode 100644 index 0000000..2db3799 --- /dev/null +++ b/API-KEY-SETUP.md @@ -0,0 +1,119 @@ +# ECHO — API key setup (Windows & macOS) + +The ECHO plugin authenticates to the vault with a bearer token. `echo.py` resolves it in +this order — **first hit wins**: + +1. **`ECHO_KEY` environment variable** ← recommended for workstations +2. **`~/.echo-memory/credentials`** file (one `key=…` line, `chmod 600`) +3. a deprecated baked-in fallback (prints a warning on every call until you do #1 or #2) + +You only need **one** of these. Pick the env var (below) or the credentials file +(`python3 echo.py write-key `, cross-platform, no shell config needed). + +> Replace `` everywhere below with your actual token. **Never** paste +> the token into a vault note, a committed file, or a screen-share. + +--- + +## Option A — credentials file (simplest, cross-platform) + +```bash +# from the plugin's scripts dir; works identically on Windows/macOS/Linux +python3 "/skills/echo-memory/scripts/echo.py" write-key +``` + +Writes `~/.echo-memory/credentials` (mode 600 on Unix). Nothing else to configure. +To rotate later, run it again with the new token. + +--- + +## Option B — environment variable + +### Windows + +**PowerShell — persistent (recommended).** Sets it for your user account; open a **new** +terminal afterward (the current session won't see it): + +```powershell +[Environment]::SetEnvironmentVariable("ECHO_KEY", "", "User") +``` + +**Command Prompt — persistent** (alternative; note `setx` truncates values over 1024 chars, +so the 64-char ECHO key is fine): + +```cmd +setx ECHO_KEY "" +``` + +**Current session only** (not persistent): + +```powershell +$env:ECHO_KEY = "" # PowerShell +``` +```cmd +set ECHO_KEY= REM Command Prompt +``` + +**GUI alternative:** Start → "Edit the system environment variables" → *Environment +Variables…* → under *User variables* click *New…* → Name `ECHO_KEY`, Value the token. + +**Verify** (in a NEW terminal): +```powershell +echo $env:ECHO_KEY # PowerShell +``` +```cmd +echo %ECHO_KEY% :: Command Prompt +``` + +### macOS + +Modern macOS uses **zsh**. Add the export to your shell profile, then reload it: + +```zsh +echo 'export ECHO_KEY=""' >> ~/.zshrc +source ~/.zshrc +``` + +If you use **bash** instead: +```bash +echo 'export ECHO_KEY=""' >> ~/.bash_profile +source ~/.bash_profile +``` + +**For GUI apps** (a desktop app that doesn't read your shell profile) you can also set it +for the login session: +```bash +launchctl setenv ECHO_KEY "" +``` +(Not persistent across reboot on its own; the shell-profile export above covers terminal +sessions, which is what the plugin uses.) + +**Verify** (in a NEW terminal): +```bash +echo "$ECHO_KEY" +``` + +--- + +## Confirm it works + +```bash +python3 "/skills/echo-memory/scripts/echo.py" doctor +``` + +`doctor` reports the Python version, vault reachability, auth, bootstrap/schema, and the +**API key source** — it should say `ECHO_KEY env` or `credentials file`, not the deprecated +baked-in fallback. + +--- + +## Security notes + +- The token is a vault credential — treat it like a password. Don't commit it, don't store + it in a vault note, rotate it if it leaks. +- `ECHO_KEY` (or `ECHO_BASE`) set in the environment **always overrides** the credentials + file and the baked-in default, so it's safe for CI / one-off overrides: + `ECHO_KEY=… python3 echo.py …`. +- Once you've set the key via Option A or B on every machine/agent that runs the plugin + (terminal, CoWork, any scheduled run), the deprecated baked-in fallback can be removed + from `scripts/echo.py` (`DEFAULT_KEY`) so no token lives in the source tree at all. diff --git a/MAINTENANCE.md b/MAINTENANCE.md new file mode 100644 index 0000000..9c3e638 --- /dev/null +++ b/MAINTENANCE.md @@ -0,0 +1,64 @@ +# echo-memory — Maintenance & repo hygiene (M5) + +Low-lift cleanups that remove "which copy is canonical?" traps before tagging 1.0. +None of these change runtime behavior; they reduce drift and confusion. + +## Canonical source tree + +- **`echo-memory.plugin.src/`** is the single canonical source tree. ✅ +- **`codex plugin/`** has already drifted — it lacks the 0.9 modules + (`echo_index.py`, `echo_links.py`, `echo_ops.py`, `sweep.py`, `check_routing.py`). + - [ ] Decide: regenerate it from the canonical tree as a build target, **or** delete it + and document the codex/CoWork packaging step instead. + - [ ] Until resolved, add a `codex plugin/README.md` note: "derived — do not edit by hand." + +## Build artifacts + +The repo root carries every historical build (`echo-memory-0.6.0.plugin` … +`echo-memory-0.9.0.plugin`) plus `echo-memory.plugin`. +- [ ] Keep only the latest `echo-memory.plugin` (the installable) tracked; move versioned + zips to GitHub Releases (or a `dist/` that is git-ignored). +- [ ] Add `*.plugin` build outputs to `.gitignore` except the current pointer, if desired. +- [x] `.gitignore` added (`.DS_Store`, `__pycache__`, local `.echo-memory/`, eval output). + +## `plugin.json` completeness + +- [x] Add `license` (UNLICENSED — personal, not for distribution). +- [x] Version bumped to **1.0.0**. +- [ ] Add `homepage` / repository URL (optional — repo URL not yet decided). +- [ ] (Optional) register slash commands in the manifest; they're auto-discovered from + `commands/` today (`echo-load|save|recall|triage|health|sweep|reflect|doctor`). + +## Docs freshness + +- [ ] `eval/run_eval.py` still benchmarks **0.6 vs 0.7** — refresh to current (open H4 follow-up). +- [x] README version-history has a **1.0.0** entry; title bumped to v1.0.0. +- [x] Scrub the literal bearer token from docs → `$ECHO_KEY` placeholder (M1, done: + `api-reference.md` ×11, `SKILL.md`, `bootstrap.md`, eval harness). Operator key setup + documented in `API-KEY-SETUP.md`. Last remaining copy is `echo.py` `DEFAULT_KEY` + (deprecated fallback) — **remove once `ECHO_KEY` is set in every environment**. + +## Building the `.plugin` artifact + +`python build.py` zips `echo-memory.plugin.src/` into `echo-memory-.plugin` (version +read from the manifest) and refreshes the `echo-memory.plugin` pointer. Deterministic +(sorted entries + fixed timestamp → byte-identical rebuilds). Flags: + +- `--strip-key` — blank `echo.py` `DEFAULT_KEY` so **no token ships** (runtime then needs + `ECHO_KEY` or `~/.echo-memory/credentials`). Use this once the key is set everywhere. +- `--no-pointer` — skip refreshing `echo-memory.plugin`. +- `--outdir DIR` — write artifacts elsewhere. + +Without `--strip-key` the build warns that the baked-in token is present. + +## Pre-1.0 release checklist + +- [x] All 10 roadmap items wired; scaffold TODOs cleared (see `ROADMAP-1.0.md` status table). +- [x] `schema_version` bumped to 4 (sweep/migrate); `migrate.py` `[3→4]` step added. + *Verify on a copy of the live vault before migrating it (it currently reports schema 3).* +- [x] Offline suites green in CI on Win/macOS/Linux × Py 3.10/3.12 (check_routing, + test_echo_client, test_v1_scaffold, test_features, test_patch_semantics, + test_offline_queue, test_reflect). +- [ ] No live secret in the tracked tree or the built `.plugin` — **one line left**: delete + `echo.py` `DEFAULT_KEY` after `ECHO_KEY`/credentials are set everywhere (per `API-KEY-SETUP.md`). +- [ ] `eval/run_eval.py` reports current-version retrieval/durability metrics (open follow-up). diff --git a/README.md b/README.md index bf0d007..e68d479 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# echo-memory — v0.9.0 +# echo-memory — v1.0.0 Persistent memory for Claude / CoWork sessions via the **ECHO** Obsidian vault, driven over the [Obsidian Local REST API](https://github.com/coddingtonbear/obsidian-local-rest-api). No MCP server — the skill makes direct REST calls, now through a bundled validated client (`scripts/echo.py`). The whole toolchain is **pure Python** (stdlib only), so it runs identically on Windows, macOS, and Linux — no bash, no platform-specific `date`. @@ -6,7 +6,7 @@ Built for **Jason Stedwell** (Director of Technical Services / Systems Engineer, This repository (`jason/echo-v.05`) holds the plugin **source** (tracked tree at `echo-memory.plugin.src/`), the built `echo-memory.plugin` package artifact (rebuilt on each version bump), and a credential-free A/B `eval/` harness. -**0.7 in one line:** the prose-and-raw-curl skill grew an executable spine — a status-checking API client, a machine-readable routing manifest the linter enforces, deterministic bootstrap/migrate scripts, an advisory multi-writer lock, four slash commands, and an eval harness. See the [0.7.0 version-history entry](#version-history) for the full list. +**0.9 in one line:** one-call `capture` routes and crosslinks a memory automatically, `recall` returns a topic plus its linked neighbourhood, and a machine-maintained entity index makes routing an alias-aware lookup — all on a cross-platform Python client with a linter-enforced routing manifest and graph-health checks. See the [version history](#version-history) for how it got here. --- @@ -29,7 +29,7 @@ Three consequences follow: The plugin is hardcoded for a single personal endpoint: - **Server:** `https://echoapi.alwisp.com` (reverse proxy → backend Obsidian Local REST API). This is the **only** valid endpoint — never use LAN addresses (`10.x`, `192.168.x`, `:27124`); those belong to retired memory systems. Override per-run with `ECHO_BASE`. -- **Auth:** a bearer token stored in the skill / `references` and injected by `scripts/echo.py`. The key lives only in the plugin — **never inside a vault note** (per the vault's own safety rules). Override with `ECHO_KEY`. +- **Auth:** a bearer token, resolved by `scripts/echo.py` (via `echo_secrets`) in order: **`ECHO_KEY` env → `~/.echo-memory/credentials` → a deprecated baked-in fallback** (warns until you set one of the first two). Set it up per **[API-KEY-SETUP.md](API-KEY-SETUP.md)** (Windows & macOS) or run `echo.py write-key `. **Never** put the key in a vault note. - **TLS:** the endpoint presents a valid certificate, so `-k` is not needed. Vault paths are addressed **at the root** (e.g. `GET /vault/_agent/context/current-context.md`). Prefer `scripts/echo.py` over raw `curl`: it injects auth, checks HTTP status (a failed write exits non-zero instead of looking like success), retries transient 5xx, verifies PUTs, and does idempotent appends. @@ -57,7 +57,7 @@ echo-v.05/ └── echo-memory.plugin.src/ ← tracked source tree (the plugin) ├── .claude-plugin/plugin.json ← manifest (name, version, description) ├── README.md ← plugin-level README - ├── commands/ ← slash commands: echo-load, echo-save, echo-triage, echo-health + ├── commands/ ← slash commands: echo-load, echo-save, echo-recall, echo-triage, echo-health, echo-sweep └── skills/echo-memory/ ├── SKILL.md ← operating procedure (authoritative) ├── references/ @@ -330,12 +330,13 @@ Observations are promoted into Fact / Pattern (dropping the date) once a pattern ## Bootstrap, repair & migration -The plugin carries everything needed to stand up a vault in `scaffold/`; there is no dependency on any in-vault control doc. In 0.7 these are scripts (`scripts/bootstrap.py`, `scripts/migrate.py`), not hand-run curl loops — the prose in `references/bootstrap.md` documents what they do and serves as the fallback. +The plugin carries everything needed to stand up a vault in `scaffold/`; there is no dependency on any in-vault control doc. These are deterministic scripts (`scripts/bootstrap.py`, `scripts/migrate.py`, `scripts/sweep.py`), not hand-run curl loops — the prose in `references/bootstrap.md` documents what they do and serves as the fallback. - **Probe** — GET `_agent/echo-vault.md`. `200` → bootstrapped (check `schema_version`); `404` → run a fresh bootstrap. - **Fresh bootstrap** (`bootstrap.py`; idempotent, additive — never overwrites): create the folder tree (a one-line README seeds each leaf since Obsidian/git ignore empty dirs) → PUT the 8 templates to their mirrored vault paths → PUT the 3 anchor seeds only if absent (the operator-preferences seed is deliberately empty — no fabricated facts) → PUT the thin vault README → PUT the marker **last** (so the vault is only flagged "set up" once everything is in place) → write a first-run trace (daily note + session log). `--dry-run` previews. - **Repair** — `bootstrap.py` again: GET-probes each path and creates only what's missing; malformed files are flagged, never replaced. -- **Migrations** — `migrate.py` reads the marker's `schema_version`, applies each intervening migration (dry-run by default; `--apply` to perform moves/deletes), and stamps the marker. `0 → 1` removed the old in-vault control docs (`CLAUDE.md` / `BOOTSTRAP.md` / `STRUCTURE.md` / `index.md`); `1 → 2` folded `reviews/` into `journal/` + `_agent/health/`. +- **Migrations** — `migrate.py` reads the marker's `schema_version`, applies each intervening migration (dry-run by default; `--apply` to perform moves/deletes), and stamps the marker. `0 → 1` removed the old in-vault control docs (`CLAUDE.md` / `BOOTSTRAP.md` / `STRUCTURE.md` / `index.md`); `1 → 2` folded `reviews/` into `journal/` + `_agent/health/`; `2 → 3` adds the `_agent/index/` entity registry. +- **Bring up to spec** — `sweep.py` back-fills an upgraded vault: builds `_agent/index/entities.json` from existing notes and symmetrizes `## Related` cross-links (dry-run by default; `--apply` to write). Run it after `migrate.py`, or any time `/echo-health` reports index drift. --- @@ -364,13 +365,14 @@ If the API returns a connection error, timeout, or `502` (usually Obsidian / the | Version | Highlights | |---------|-----------| -| **0.9.0** | Schema 3. **Recall + routing intelligence — five layered features aimed at less input to route and richer links to recall.** (1) **Entity index** (`_agent/index/entities.json`): a slug→{path,kind,title,aliases} registry so routing/`resolve` is an O(1), alias-aware lookup instead of a fuzzy search (dedup at the source). (2) **`recall`**: search + one-hop expansion along `## Related` and `source_notes` — returns a topic's connected neighbourhood, not an isolated note. (3) **Bidirectional auto-linking**: `link A B` and link-on-`capture` keep the graph symmetric. (4) **`capture`**: one call that routes, stamps canonical frontmatter, indexes, auto-links mentioned entities, and writes the Agent-Log line — collapsing the whole write discipline. (5) **Graph-health checks** in `vault_lint.py`: broken wikilinks, orphan notes, entity-index drift. Plus **`sweep.py`** to bring an existing vault up to spec (build the index + symmetrize links + stamp schema), `migrate.py` 2→3, and `/echo-recall` + `/echo-sweep` commands. New modules `echo_index.py` / `echo_links.py` / `echo_ops.py`; offline + mock-integration tests (`eval/test_features.py`). | -| **0.8.0** | **Cross-platform + reliability pass.** Ported the entire toolchain from bash to **pure Python** (`echo.py`, `vault_lint.py`, `bootstrap.py`, `migrate.py`) so it runs identically on Windows/macOS/Linux — removes the bash and macOS-only `date` dependencies (the lock's TTL parse previously degraded silently on unsupported platforms); stdout is UTF-8-safe so non-ASCII output can't crash a legacy console. **Correctness:** `append` now skips only on an exact **whole-line** match (a substring no longer causes a false skip that silently drops a write); the advisory lock is **read-back-confirmed** after PUT so a racing writer backs off instead of proceeding under a false lock. **Routing consistency (`check_routing.py`):** mechanically verifies the routing docs against `routing.json` in both directions, run as a guard in `test_echo_client.py`; added the missing locks / leaf-README / decision-template rows to the canonical map. **Usability:** new one-call `echo.py load` cold start; `SKILL.md` slimmed — the duplicated raw-curl recipes now live only in `api-reference.md` (the *nix fallback). | +| **1.0.0** | Schema 4. **"Memory you can trust and retrieve."** (H1) **Hybrid recall** — local BM25 over note bodies fused with decayed graph expansion (`echo_recall.py`, `_agent/index/recall-index.json`), maintained on `capture`, rebuilt by `sweep`; replaces keyword-only recall. (H2) **Offline durability** — write verbs queue to a local write-ahead outbox on an outage and report "queued"; `flush` (+ flush-on-`load`) replays idempotently and **re-bases to the current endpoint**; `load` degrades to a last-known-good read cache (`echo_queue.py`). (H3) **Concurrency** — `vault_lock` + `atomic_index_update` (lock + fresh re-read-merge) close the concurrent-`capture` entity-loss race (`echo_concurrency.py`). (H4) **Trust** — higher-fidelity PATCH mock + offline/reflect/patch-semantics suites gated in GitHub Actions across Win/macOS/Linux × Py 3.10/3.12. (H5) **Reflection capture** — `echo.py reflect` / `/echo-reflect` extract→dedup→preview→apply durable items (`echo_reflect.py`). (M1) **Secret hardening** — key resolves env → `~/.echo-memory/credentials` → deprecated fallback; `write-key` CLI; token scrubbed from docs (see [API-KEY-SETUP.md](API-KEY-SETUP.md)). (M2) confident auto-linking + slug-collision guard (`echo_quality.py`). (M3) `echo.py doctor` / `/echo-doctor` + heartbeat-less `load` fallback. (M4) `capture --json` / `--dry-run`. (M5) manifest → 1.0.0, `.gitignore`. | | **0.3.0** | Source promoted from zip-only to a tracked tree (`echo-memory.plugin.src/`); `.plugin` becomes a build artifact. All 7 skill-improvement items applied: search-first before writes, resilient daily Agent Log, `created:` semantics, project lifecycle + folder↔status rule, canonical HHMM session filenames, read-most-recent-N sessions, `source_notes` defined as backward links. | | **0.4.0** | Efficiency + robustness pass: parallel cold-start loading, idempotent POST (read-before-append), doc-map-before-first-PATCH, scoped `updated:` bump, Inbox Triage, Scope Switching, monthly Vault Health, Rules-vs-Observations split, formal deprecation of `decisions/by-project/`, heartbeat pointer. | | **0.4.1** | Bugfix: daily-note Agent Log heading detection now greps raw markdown for `^## Agent Log` instead of the `::`-delimited doc-map JSON (which never matched and appended duplicate headings). Added Scope Switching cold-start test harness. | | **0.5.0** | Self-bootstrap + control-logic-in-plugin. Plugin becomes the single source of truth: bundled `scaffold/` (8 templates, 3 anchor seeds, thin vault README, marker) bootstraps an empty vault with no external/local-path dependency. New `operating-contract.md` (principles + safety from the old in-vault `CLAUDE.md`); `bootstrap.md` rewritten as a portable bootstrap/repair/migrate manifest. Cold-start probe moved from `/vault/BOOTSTRAP.md` to `_agent/echo-vault.md` (carries `schema_version`). Live vault migrated to data-only. | | **0.5.1** | Routing-doc consistency pass: decision-mirror heading unified to `## Key Decisions`; stale `Current status` PATCH examples corrected to `Status`; vault-layout inline project example refreshed to the real template. All 17 `projects/active/` notes normalized losslessly to the canonical template heading set; `android-mqtt-shell` moved to `incubating/` (was broken `status: upcoming` in active). Plugin repackaged (21 files). | | **0.6.0** | Schema 2. **#8 Inbox auto-fire:** the Loading procedure adds an inbox-depth GET and a load-time *Reconcile* step (inbox triage + scope-drift), so triage self-fires. **#10 Routing:** `reviews/` retired — weekly/monthly/quarterly/annual rollups fold into `journal/{weekly,monthly,quarterly,annual}/`, vault-health moves to `_agent/health/`; new `references/routing-map.md` is the complete audited endpoint→logic map. **Recs:** heartbeat pointer operationalized (read first at load, written at session end); new `scripts/vault_lint.py` mechanically checks vault invariants. Dead refs pruned (`archive/`, `_agent/outputs/`, `resources/source-material`). Migration `1 → 2` in `bootstrap.md`. | -| **0.7.1** | **Scope-drift fix.** Scope is the most churn-prone state (several sessions/day) and had no freshness signal, so sessions silently ran under stale scope (same failure class as #8). Added a `scope_updated:` frontmatter timestamp (maintained automatically), an `echo.py scope show` / `scope set` command (atomic switch: archive prior → replace → stamp), and a `vault_lint.py` **drift check** (flags when ≥ `SCOPE_STALE_SESSIONS`, default 3, session logs are dated after `scope_updated`) — making drift mechanically *evaluable* via `/echo-health`. Tightened the SKILL load-reconcile to *state and confirm* scope every session and switch before working. (Also fixed a bash nested-quote parse bug found while building `scope`, where `show` could fall through into `set`.) | | **0.7.0** | Schema 2 (unchanged layout). Hardening pass — gave the prose-and-curl skill an executable spine. **S2** `scripts/echo.py`: one validated client wrapping every verb with auth, HTTP-status checking (failed writes exit non-zero instead of looking like success), one bounded retry on 5xx, read-back-verified PUT, and idempotent `append`. **S3** `scripts/routing.json`: canonical machine-readable route manifest; `vault_lint.py` enforces it (flags unknown/retired paths). **S4** deterministic `scripts/bootstrap.py` + `scripts/migrate.py` (idempotent, dry-run, probe-before-write; fixes the old CWD-relative `@scaffold/...` empty-body bug). **S5** cooperative advisory lock (`_agent/locks/vault.lock`) + documented multi-writer model. **M1/M2/M5** linter rewrite: real YAML parsing, injected clock (`ECHO_TODAY`), exits `3` (not "clean") on an un-bootstrapped vault, plus routing-membership + frontmatter-integrity checks. **M3** status-check guidance throughout. **M4** four slash commands (`/echo-load`, `/echo-save`, `/echo-triage`, `/echo-health`). Added a credential-free A/B `eval/` harness (mock REST API + fault injection): isolates a **−76% generated-token** I/O layer and **4 → 0 silent write failures** vs 0.6. | +| **0.7.1** | **Scope-drift fix.** Scope is the most churn-prone state (several sessions/day) and had no freshness signal, so sessions silently ran under stale scope (same failure class as #8). Added a `scope_updated:` frontmatter timestamp (maintained automatically), an `echo.py scope show` / `scope set` command (atomic switch: archive prior → replace → stamp), and a `vault_lint.py` **drift check** (flags when ≥ `SCOPE_STALE_SESSIONS`, default 3, session logs are dated after `scope_updated`) — making drift mechanically *evaluable* via `/echo-health`. Tightened the SKILL load-reconcile to *state and confirm* scope every session and switch before working. (Also fixed a bash nested-quote parse bug found while building `scope`, where `show` could fall through into `set`.) | +| **0.8.0** | **Cross-platform + reliability pass.** Ported the entire toolchain from bash to **pure Python** (`echo.py`, `vault_lint.py`, `bootstrap.py`, `migrate.py`) so it runs identically on Windows/macOS/Linux — removes the bash and macOS-only `date` dependencies (the lock's TTL parse previously degraded silently on unsupported platforms); stdout is UTF-8-safe so non-ASCII output can't crash a legacy console. **Correctness:** `append` now skips only on an exact **whole-line** match (a substring no longer causes a false skip that silently drops a write); the advisory lock is **read-back-confirmed** after PUT so a racing writer backs off instead of proceeding under a false lock. **Routing consistency (`check_routing.py`):** mechanically verifies the routing docs against `routing.json` in both directions, run as a guard in `test_echo_client.py`; added the missing locks / leaf-README / decision-template rows to the canonical map. **Usability:** new one-call `echo.py load` cold start; `SKILL.md` slimmed — the duplicated raw-curl recipes now live only in `api-reference.md` (the *nix fallback). | +| **0.9.0** | Schema 3. **Recall + routing intelligence — five layered features aimed at less input to route and richer links to recall.** (1) **Entity index** (`_agent/index/entities.json`): a slug→{path,kind,title,aliases} registry so routing/`resolve` is an O(1), alias-aware lookup instead of a fuzzy search (dedup at the source). (2) **`recall`**: search + one-hop expansion along `## Related` and `source_notes` — returns a topic's connected neighbourhood, not an isolated note. (3) **Bidirectional auto-linking**: `link A B` and link-on-`capture` keep the graph symmetric. (4) **`capture`**: one call that routes, stamps canonical frontmatter, indexes, auto-links mentioned entities, and writes the Agent-Log line — collapsing the whole write discipline. (5) **Graph-health checks** in `vault_lint.py`: broken wikilinks, orphan notes, entity-index drift. Plus **`sweep.py`** to bring an existing vault up to spec (build the index + symmetrize links + stamp schema), `migrate.py` 2→3, and `/echo-recall` + `/echo-sweep` commands. New modules `echo_index.py` / `echo_links.py` / `echo_ops.py`; offline + mock-integration tests (`eval/test_features.py`). | diff --git a/ROADMAP-1.0.md b/ROADMAP-1.0.md new file mode 100644 index 0000000..2ba46d4 --- /dev/null +++ b/ROADMAP-1.0.md @@ -0,0 +1,176 @@ +# echo-memory — Roadmap to v1.0 + +> Status: **draft / in progress.** Current shipping version: **0.9.0 (schema 3)**. +> Target: **1.0.0 (schema 4)** — *"memory you can actually trust and retrieve."* + +This roadmap turns the 10 code-review suggestions into a sequenced plan and tracks the +scaffold for each. It is the single index: every v1.0 workstream maps to one or more +files under `echo-memory.plugin.src/skills/echo-memory/scripts/` (plus tests and CI). + +--- + +## 1.0 thesis + +0.x made the plugin **correct and self-describing**: status-checked writes, idempotent +appends, a routing manifest the linter enforces, an entity index, and one-call +`capture/recall`. 1.0 makes it **trustworthy and genuinely recall-driven**: + +1. You can **retrieve** a fact even when you phrase the query differently than you stored it. *(H1)* +2. Nothing is **lost** when the backend is down. *(H2)* +3. Two clients writing at once **cannot corrupt** the index or clobber state. *(H3)* +4. Every feature is **proven** by tests that model the real API, gated in CI. *(H4)* +5. Memory **accrues on its own** instead of only when the agent remembers to write. *(H5)* + +…on a base that is secure (M1), clean-linking (M2), self-diagnosing (M3), +machine-parseable (M4), and maintainable (M5). + +--- + +## Non-negotiable constraints (these bound every solution) + +These are inherited from the 0.x design and **must not regress**: + +| Constraint | Consequence for 1.0 | +|---|---| +| **Pure-Python stdlib, no third-party deps** | H1 recall = local BM25 + graph fusion, **not** an embedding API. No `numpy`, `keyring`, `requests`. | +| **Cross-platform (Win/macOS/Linux)** | No bash, no platform `date`; `pathlib`, `os`, UTF-8-safe streams. | +| **Plugin is the single source of truth; vault holds data only** | Derived artifacts (recall index) live in `_agent/index/` (machine-maintained, rebuildable) — never new control docs in the vault. | +| **Fail loud, never silent** | New write paths keep status-checking and non-zero exits. | +| **Local state can't depend on the vault being up** | H2 queue/cache lives in `ECHO_STATE_DIR` (default `~/.echo-memory/`), not the vault. | + +--- + +## Build order vs. scaffold order + +The user asked to **scaffold heaviest → lightest** (H1→M5). That is the order the +files below are stubbed. The **recommended build/merge order is different** — safety +and proof first, so the heavy features land on a tested, secure base: + +``` +Phase A (land first): M1 secrets · H4 tests+CI +Phase B (correctness): H3 concurrency · M2 link/slug quality +Phase C (durability): H2 offline queue + cache +Phase D (intelligence): H1 hybrid recall · H5 reflection capture +Phase E (polish): M3 doctor · M4 --json · M5 hygiene → tag 1.0.0 +``` + +Scaffolds are independent stubs, so stubbing heaviest-first is safe; only the +*integration* (wiring into `echo.py`, schema bump, migration) follows Phase order. + +--- + +## Workstreams (heaviest → lightest) + +Each item: **goal · why · scope · files · storage/schema · acceptance · depends-on · lift.** + +### H1 — Hybrid/semantic recall · lift: ●●●●● +- **Goal:** `recall "X"` returns relevant notes even when the wording differs, ranked, with scored multi-hop graph expansion. +- **Why:** Recall is the product. Today it's keyword `/search/simple` + fixed one-hop (`echo_ops.py:88`). Differently-phrased memory is unfindable → dead weight. +- **Scope:** local BM25 index over note bodies (stdlib only); fuse lexical score + graph distance (decay per hop); relevance threshold; configurable hops. +- **Files:** `scripts/echo_recall.py` (new) → later replaces `echo_ops.recall`. +- **Storage/schema:** `_agent/index/recall-index.json` (postings + doc stats), machine-maintained, rebuilt by `sweep.py`. **Schema 3→4.** +- **Acceptance:** retrieval precision/recall measured in eval beats keyword-only baseline; recall returns scored, deduped hits + neighbourhood; index rebuildable offline. +- **Depends-on:** entity index (have), H4 eval harness for the precision/recall numbers. + +### H2 — Offline durability: write-ahead queue + read cache · lift: ●●●●● +- **Goal:** No durable write is lost when the API/Obsidian is down; cold-start degrades to last-known context instead of "no memory." +- **Why:** Today vault-unreachable = silently proceed without memory (`SKILL.md:383`). 502 (Obsidian not running) is the most common real failure. +- **Scope:** append-only NDJSON outbox; replay on next reachable session (idempotency makes replay safe); last-known-good read cache for `load`. +- **Files:** `scripts/echo_queue.py` (new); integration hook `safe_request()` wrapping `echo.request`. +- **Storage/schema:** local `ECHO_STATE_DIR/outbox.ndjson` + `ECHO_STATE_DIR/cache/`. No vault schema change. +- **Acceptance:** kill the API mid-session → writes queue; restart → replay lands exactly once (no dupes); `load` serves cache when offline and says so. +- **Depends-on:** existing idempotent-append discipline; H4 for fault-injection tests. + +### H3 — Concurrency correctness · lift: ●●●●○ +- **Goal:** Make the advertised multi-writer (Claude + CoWork) story actually safe. +- **Why:** Index is full-file load→PUT with no lock (`echo_index.py:105`, `echo_ops.py:230`) → concurrent `capture` silently drops an entity. Lock is manual/advisory only (`echo.py:293`). +- **Scope:** auto-lock context manager around `capture`/`scope set`/index writes; atomic index update (re-read under lock, merge, save); per-resource locks; crash-safe TTL reclaim (have). +- **Files:** `scripts/echo_concurrency.py` (new); refactor `echo_index.save` + `echo_ops.capture` to route through it. +- **Storage/schema:** reuse `_agent/locks/`. No schema change. +- **Acceptance:** two concurrent captures both land in the index; no lost entries; lock auto-released on normal + error exit. +- **Depends-on:** none (pure refactor); H4 to prove it. + +### H4 — High-fidelity tests + CI + current eval · lift: ●●●●○ +- **Goal:** Prove every 0.9 + 1.0 feature against an API model that reproduces real failure modes; gate on CI. +- **Why:** Mock PATCH is "naive append" (`mock_olrapi.py:138`) — can't model heading replace/insert; eval still benchmarks **0.6 vs 0.7** (two versions stale). +- **Scope:** higher-fidelity mock (real heading/frontmatter PATCH semantics) or containerized real Obsidian Local REST API; refresh eval to measure retrieval precision/recall, link correctness, dup rate; wire `test_echo_client.py` + `check_routing.py` + `test_features.py` + new module tests into GitHub Actions. +- **Files:** `.github/workflows/ci.yml` (new); `eval/mock_olrapi_hifi.py` (new); `scripts/test_v1_scaffold.py` (interface guard, new); refreshed `eval/run_eval.py`. +- **Acceptance:** CI green on every push; eval reports current-version metrics; mock reproduces the 40080 invalid-target and replace/insert paths. +- **Depends-on:** none — **foundational, build in Phase A.** + +### H5 — Automatic session-reflection capture · lift: ●●●●○ +- **Goal:** At session end, propose durable captures extracted from the conversation for one-tap confirm — memory that fills itself. +- **Why:** Memory only accrues when the agent decides to write. "As useful as possible" means not depending on that judgment firing. +- **Scope:** model-side extraction emits a JSON proposal set; script dedups against the entity index, previews, and applies on confirm (respects "show before large writes" safety rule). +- **Files:** `scripts/echo_reflect.py` (new); `/echo-reflect` command (later). +- **Storage/schema:** none new (routes through `capture`). +- **Acceptance:** given a transcript, proposals are deduped vs index, previewed, and only applied on explicit confirm; nothing written without go-ahead. +- **Depends-on:** H1 index/recall for dedup quality; H3 for safe concurrent apply. + +### M1 — Harden secret handling · lift: ●●○○○ *(do now — Phase A)* +- **Goal:** Stop shipping a live bearer token in source, docs, and every `.plugin` zip. +- **Why:** Token hardcoded at `echo.py:70` and repeated through `api-reference.md`/README — committed to git, only rotatable by rebuild. Violates the plugin's own "never store secrets" rule. +- **Scope:** resolve key env → local `ECHO_STATE_DIR/credentials` (0600) → deprecated baked default **with a loud warning**; scrub literal from docs to a placeholder; document rotation. +- **Files:** `scripts/echo_secrets.py` (new); `echo.py` `KEY=` → `echo_secrets.resolve_key()`. +- **Acceptance:** no live token in tracked source/docs/package; `ECHO_KEY` still overrides; rotation documented. +- **Depends-on:** none. + +### M2 — Tame auto-link false positives & slug collisions · lift: ●●○○○ +- **Goal:** Keep the graph (which recall depends on) clean. +- **Why:** Auto-link fires on any ≥3-char name/alias word-match (`echo_ops.py:236`) → a concept "API" or alias "rs" links everywhere; `slugify` truncates to 40 chars with no collision check (`echo_index.py:58`) → distinct titles silently share a note. +- **Scope:** raise alias floor / require multi-token or kind-aware match / stopword guard; collision-aware slug disambiguation in `derive_path`. +- **Files:** `scripts/echo_quality.py` (new); patches to `echo_links`/`echo_index`. +- **Acceptance:** known false-positive cases no longer link; colliding titles get distinct slugs; covered by tests. +- **Depends-on:** none. + +### M3 — `echo.py doctor` + complete `load` fallback · lift: ●●○○○ +- **Goal:** One-call readiness check; make `load` self-sufficient. +- **Why:** No single "is everything OK" check. `cmd_load` reads the heartbeat but never falls back to listing `_agent/sessions/` though `SKILL.md:122` documents it (`echo.py:395`). +- **Scope:** doctor = Python version + reachability + auth + marker/`schema_version` + lint summary, green/red; `load` lists recent sessions when heartbeat missing/stale. +- **Files:** `scripts/echo_doctor.py` (new); patch `echo.cmd_load`. +- **Acceptance:** `doctor` prints actionable status and exits non-zero on any red; `load` orients with no heartbeat. +- **Depends-on:** none. + +### M4 — Machine output (`--json`) + `--dry-run` · lift: ●○○○○ +- **Goal:** Let the agent act on structured results and preview writes. +- **Why:** `capture/recall/resolve/scope` print prose; agent re-parses free text. +- **Scope:** `--json` result envelope (action, path, links_added, hits); `--dry-run` for `capture`/`scope set`. +- **Files:** `scripts/echo_output.py` (new helpers); thread through `echo_ops`/`echo.py`. +- **Acceptance:** every high-level op emits valid JSON under `--json`; `--dry-run` writes nothing. +- **Depends-on:** none. + +### M5 — Repo hygiene & manifest completeness · lift: ●○○○○ +- **Goal:** Remove "which tree is canonical?" traps; complete the manifest. +- **Why:** Two plugin trees (`codex plugin/` vs `echo-memory.plugin.src/`) have already drifted (codex lacks the 0.9 modules); stack of versioned `.plugin` artifacts; sparse `plugin.json`. +- **Scope:** consolidate/mark canonical tree; prune old artifacts; add `license`/`homepage`/command registration to `plugin.json`; refresh version-history + eval references. +- **Files:** `MAINTENANCE.md` (new checklist); `plugin.json`. +- **Acceptance:** one canonical source tree; complete manifest; docs reference current version. +- **Depends-on:** none. + +--- + +## Schema 3 → 4 migration (owned by H1/H2) + +- Add `_agent/index/recall-index.json` (H1), rebuilt by `sweep.py`. +- `migrate.py`: add `[3→4]` step (create recall-index placeholder; no destructive moves). +- `routing.json`: add a route for `_agent/index/recall-index.json` (already covered by the generic `^_agent/index/[^/]+\.json$` route — verify in `check_routing.py`). +- Bump `CURRENT_SCHEMA` to 4 in `sweep.py`/`migrate.py`; `vault_lint.py` unaffected. + +--- + +## Scaffold status + +| # | Workstream | Lift | Scaffold file(s) | State | +|---|---|---|---|---| +| H1 | Hybrid recall | ●●●●● | `scripts/echo_recall.py` | **✅ WIRED** — BM25 + vault crawl + `recall-index.json` persistence + graph fusion; maintained on `capture`, rebuilt by `sweep`; replaces `echo_ops.recall`; **schema 4**. Covered by `test_features.py`. | +| H2 | Offline queue/cache | ●●●●● | `scripts/echo_queue.py`, `eval/test_offline_queue.py` | **✅ WIRED** — write verbs (put/post/append/patch/delete) queue on outage and report "queued"; `flush` (+ flush-on-`load`) replays idempotently and **re-bases to the current endpoint** (survives base migration); `load` degrades to last-known-good cache and flags OFFLINE. Proven end-to-end (down→queue→up→flush→land, idempotent, cache fallback). Follow-up: route capture's internal index/link writes through the queue for full offline-atomic captures. | +| H3 | Concurrency | ●●●●○ | `scripts/echo_concurrency.py` | **✅ WIRED** — `vault_lock` CM (auto acquire/release, crash-safe, advisory) + `atomic_index_update` (lock + fresh re-read-merge). `capture` entity write and `echo_recall.update_note` both routed through it. Proven by `test_features.py` (merge-no-clobber + lock-release). Follow-up: deeper resolve-level title-collision detection; true concurrent stress test. | +| H4 | Tests + CI | ●●●●○ | `.github/workflows/ci.yml`, `eval/mock_olrapi_hifi.py`, `eval/test_patch_semantics.py`, `scripts/test_v1_scaffold.py` | **✅ hi-fi PATCH mock + semantics test + CI matrix (Win/macOS/Linux × 3.10/3.12) live.** Remaining: refresh `run_eval.py` (still 0.6-vs-0.7) → publish current metrics. | +| H5 | Reflection capture | ●●●●○ | `scripts/echo_reflect.py`, `eval/test_reflect.py`, `commands/echo-reflect.md` | **✅ WIRED** — `validate`/`classify`/`preview`/`apply` + `echo.py reflect` (dry-run unless `--apply`, reads file or stdin) + `/echo-reflect` command. Dedups proposals vs the entity index, drops below-confidence, routes `inbox`, applies via `capture` (indexed/linked/logged, self-lock-guarded). Proven by `test_reflect.py`. | +| M1 | Secret handling | ●●○○○ | `scripts/echo_secrets.py` | **✅ WIRED** — `resolve_key` (env → `~/.echo-memory/credentials` → deprecated fallback + loud warning), `write-key` CLI, docs scrubbed. Live token now only in `echo.py` `DEFAULT_KEY` (remove at 1.0; operator env already sets `ECHO_KEY`). | +| M2 | Link/slug quality | ●●○○○ | `scripts/echo_quality.py` | **✅ WIRED** — `is_confident_link` gates `capture` auto-linking (rejects <4-char / common tokens, trusts multi-word names); `safe_slug` disambiguates a colliding 40-char-truncated slug in the create path. Follow-up: resolve-level title collision (two long titles → same slug → false merge). | +| M3 | Doctor + load fallback | ●●○○○ | `scripts/echo_doctor.py`, `commands/echo-doctor.md` | **✅ WIRED** — `echo.py doctor` (Python/reachability/auth/bootstrap/schema/key-source) + `/echo-doctor`; `cmd_load` now falls back to a recent-sessions listing when the heartbeat pointer is absent. Verified against the live vault. | +| M4 | `--json` / `--dry-run` | ●○○○○ | `scripts/echo_output.py` | **✅ WIRED** — `capture --json` emits a clean result envelope (helper chatter redirected) and `--dry-run` previews the create/update plan without writing; `resolve` already emits JSON. Proven by `test_features.py`. Follow-up: `--json` for recall/scope/link. | +| M5 | Hygiene | ●○○○○ | `plugin.json`, `.gitignore`, `MAINTENANCE.md` | **✅ DONE** — `plugin.json` → **1.0.0** + `license`; `.gitignore` (`.DS_Store`, `__pycache__`, local `.echo-memory/` state, eval output). Remaining (operator judgment, not code): consolidate/retire the drifted `codex plugin/` tree and prune old `.plugin` artifacts — see `MAINTENANCE.md`. | + +**Wiring:** scaffolds are **not** imported by the live `echo.py` yet — 0.9.0 behavior is unchanged. Integration happens per Phase, each gated by its tests (H4). diff --git a/build.py b/build.py new file mode 100644 index 0000000..dc5f781 --- /dev/null +++ b/build.py @@ -0,0 +1,132 @@ +#!/usr/bin/env python3 +"""build.py — package the echo-memory plugin source into a .plugin artifact. + +Zips the CONTENTS of echo-memory.plugin.src/ at the archive root (the layout the plugin +loader expects: .claude-plugin/plugin.json, commands/, skills/ all at top level), excluding +dev cruft. The version is read from the manifest, so the output is named automatically. + +Usage: + python build.py # build echo-memory-.plugin + refresh echo-memory.plugin + python build.py --strip-key # ALSO blank echo.py's DEFAULT_KEY -> a token-free artifact + # (requires ECHO_KEY or ~/.echo-memory/credentials at runtime) + python build.py --no-pointer # don't update the echo-memory.plugin "current" pointer + python build.py --outdir dist # write artifacts somewhere other than the repo root + +Deterministic: entries are sorted and stamped with a fixed timestamp, so an unchanged +source tree always produces a byte-identical archive (clean diffs / reproducible builds). +Pure stdlib; runs the same on Windows/macOS/Linux. +""" +from __future__ import annotations + +import argparse +import json +import re +import sys +import zipfile +from pathlib import Path + +# Keep non-ASCII (em-dashes in our messages) from raising/garbling on a legacy console. +for _stream in (sys.stdout, sys.stderr): + try: + _stream.reconfigure(encoding="utf-8", errors="replace") # type: ignore[union-attr] + except Exception: + pass + +REPO = Path(__file__).resolve().parent +SRC = REPO / "echo-memory.plugin.src" +MANIFEST = "/".join([".claude-plugin", "plugin.json"]) +ECHO_PY = "/".join(["skills", "echo-memory", "scripts", "echo.py"]) + +EXCLUDE_DIRS = {"__pycache__", ".git"} +EXCLUDE_NAMES = {".DS_Store"} +EXCLUDE_SUFFIXES = {".pyc", ".pyo"} +FIXED_DATE = (2026, 1, 1, 0, 0, 0) # stable timestamp for reproducible archives + +_KEY_RE = re.compile(r'(DEFAULT_KEY\s*=\s*")([0-9a-fA-F]{16,})(")') + + +def included_files() -> list[Path]: + out = [] + for p in sorted(SRC.rglob("*")): + if p.is_dir(): + continue + if any(part in EXCLUDE_DIRS for part in p.relative_to(SRC).parts): + continue + if p.name in EXCLUDE_NAMES or p.suffix in EXCLUDE_SUFFIXES: + continue + out.append(p) + return out + + +def file_bytes(path: Path, arcname: str, strip_key: bool) -> bytes: + data = path.read_bytes() + if strip_key and arcname == ECHO_PY: + text = data.decode("utf-8") + new, n = _KEY_RE.subn(r'\1\3', text) # collapse the hex value to "" + if n: + text = new.replace('DEFAULT_KEY = ""', + 'DEFAULT_KEY = "" # stripped at build time — set ECHO_KEY or run write-key') + data = text.encode("utf-8") + return data + + +def token_present(path: Path) -> bool: + m = _KEY_RE.search(path.read_text(encoding="utf-8")) + return bool(m and m.group(2)) + + +def build(out: Path, files: list[Path], strip_key: bool) -> int: + if out.exists(): + out.unlink() + with zipfile.ZipFile(out, "w", zipfile.ZIP_DEFLATED) as z: + for p in files: + arc = p.relative_to(SRC).as_posix() + info = zipfile.ZipInfo(arc, date_time=FIXED_DATE) + info.compress_type = zipfile.ZIP_DEFLATED + info.external_attr = 0o644 << 16 + z.writestr(info, file_bytes(p, arc, strip_key)) + return out.stat().st_size + + +def main(argv: list[str] | None = None) -> int: + ap = argparse.ArgumentParser(description="Build the echo-memory .plugin artifact") + ap.add_argument("--strip-key", action="store_true", + help="blank echo.py DEFAULT_KEY so no token ships in the artifact") + ap.add_argument("--no-pointer", action="store_true", + help="do not refresh the echo-memory.plugin 'current' pointer") + ap.add_argument("--outdir", default=str(REPO), help="output directory (default: repo root)") + args = ap.parse_args(argv) + + if not SRC.is_dir(): + print(f"build: source tree not found at {SRC}", file=sys.stderr) + return 1 + + manifest_path = SRC / ".claude-plugin" / "plugin.json" + version = json.loads(manifest_path.read_text(encoding="utf-8"))["version"] + files = included_files() + arcnames = {p.relative_to(SRC).as_posix() for p in files} + if MANIFEST not in arcnames: + print(f"build: {MANIFEST} missing from the archive root — aborting", file=sys.stderr) + return 1 + + outdir = Path(args.outdir) + outdir.mkdir(parents=True, exist_ok=True) + targets = [outdir / f"echo-memory-{version}.plugin"] + if not args.no_pointer: + targets.append(outdir / "echo-memory.plugin") + + for out in targets: + size = build(out, files, args.strip_key) + print(f"built {out.name} ({len(files)} entries, {size:,} bytes)") + + echo_src = SRC / "skills" / "echo-memory" / "scripts" / "echo.py" + if args.strip_key: + print("note: --strip-key set — artifact is token-free; runtime needs ECHO_KEY or a credentials file.") + elif token_present(echo_src): + print("WARNING: the baked-in DEFAULT_KEY is present in this artifact — do not share it. " + "Set ECHO_KEY everywhere (API-KEY-SETUP.md), then build with --strip-key.") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/echo-memory-1.0.0.plugin b/echo-memory-1.0.0.plugin new file mode 100644 index 0000000..2c79d98 Binary files /dev/null and b/echo-memory-1.0.0.plugin differ diff --git a/echo-memory.plugin b/echo-memory.plugin index 183632f..2c79d98 100644 Binary files a/echo-memory.plugin and b/echo-memory.plugin differ diff --git a/echo-memory.plugin.src/.claude-plugin/plugin.json b/echo-memory.plugin.src/.claude-plugin/plugin.json index 288d1ff..9423e3e 100644 --- a/echo-memory.plugin.src/.claude-plugin/plugin.json +++ b/echo-memory.plugin.src/.claude-plugin/plugin.json @@ -1,10 +1,11 @@ { "name": "echo-memory", - "version": "0.9.0", - "description": "Persistent, self-bootstrapping memory via the ECHO Obsidian vault over the Local REST API \u2014 no MCP server. Cross-platform Python client (echo.py) with one-call capture/resolve/recall/link backed by a machine-maintained entity index and automatic bidirectional cross-linking, a routing manifest the linter enforces, graph-health checks, and bootstrap/migrate/sweep scripts plus /echo-load|save|recall|triage|health|sweep commands. Routes and crosslinks notes across Claude/CoWork sessions.", + "version": "1.0.0", + "description": "Persistent, self-bootstrapping memory via the ECHO Obsidian vault over the Local REST API — no MCP server. Cross-platform Python client (echo.py) with one-call capture/resolve/recall/link backed by a machine-maintained entity index, hybrid BM25 + graph recall, automatic bidirectional cross-linking, an offline write-ahead queue + read cache, lock-guarded concurrency, session-reflection capture, a routing manifest the linter enforces, graph-health checks, and bootstrap/migrate/sweep scripts plus /echo-load|save|recall|triage|health|sweep|reflect commands. Routes and crosslinks notes across Claude/CoWork sessions.", "author": { "name": "Jason" }, + "license": "UNLICENSED", "keywords": [ "memory", "obsidian", diff --git a/echo-memory.plugin.src/commands/echo-doctor.md b/echo-memory.plugin.src/commands/echo-doctor.md new file mode 100644 index 0000000..2c34c5a --- /dev/null +++ b/echo-memory.plugin.src/commands/echo-doctor.md @@ -0,0 +1,15 @@ +--- +description: Check ECHO readiness — Python, vault reachability, auth, bootstrap/schema, and key source +--- + +Use the **echo-memory** skill to run a one-call readiness check before relying on memory. + +```bash +python3 "${CLAUDE_PLUGIN_ROOT}/skills/echo-memory/scripts/echo.py" doctor +# Windows: use `python` or `py -3` if `python3` is not on PATH. +``` + +It prints green/red for: Python version, vault reachability, auth accepted, vault +bootstrapped (+ `schema_version`), and the **API key source** (`ECHO_KEY` env / credentials +file / deprecated baked-in fallback — see `API-KEY-SETUP.md`). Exits non-zero if anything is +red. For the full vault-invariant lint, run `/echo-health`. diff --git a/echo-memory.plugin.src/commands/echo-reflect.md b/echo-memory.plugin.src/commands/echo-reflect.md new file mode 100644 index 0000000..aadda4e --- /dev/null +++ b/echo-memory.plugin.src/commands/echo-reflect.md @@ -0,0 +1,30 @@ +--- +description: Reflect on this session — extract durable memories and propose them for one-tap capture +argument-hint: "[optional focus, e.g. 'just decisions']" +--- + +Use the **echo-memory** skill to run **session reflection** (H5). Scan this conversation for +things worth remembering across sessions, then propose them — don't write blindly. + +1. **Extract** durable items from the conversation: new facts, preferences, decisions, + commitments, people/companies/projects introduced, and anything Jason said to remember. + Skip the ephemeral. Anchor relative dates on the conversation's `currentDate`. +2. **Emit a JSON array** of proposals (one per item), each: + `{"title","kind","body","aliases","sources","confidence"}` — `kind` ∈ + `person, company, concept, reference, meeting, project, area, semantic, episodic, + working, skill, decision`; set `"inbox": true` when the home is genuinely unknown; + `confidence` 0–1 (items below 0.6 are dropped — send those to the inbox instead). + Write the array to a file with the Write tool (cross-platform; no heredoc). +3. **Preview, then apply.** Dry-run first (writes nothing) and show Jason the plan; only + apply after he confirms. + +```bash +ECHO="${CLAUDE_PLUGIN_ROOT}/skills/echo-memory/scripts/echo.py" # python3 (Windows: python / py -3) +python3 "$ECHO" reflect proposals.json # dry-run: validate + classify + preview +python3 "$ECHO" reflect proposals.json --apply # write — routes each via capture (indexed, linked, logged) +``` + +`reflect` dedups every proposal against the entity index (so it shows create-vs-update), +skips below-confidence items, and routes `inbox` proposals to the capture inbox. Each +applied item goes through the normal `capture` path — canonical frontmatter, auto-linking, +and the recall index — and the writes are lock-guarded. $ARGUMENTS diff --git a/echo-memory.plugin.src/skills/echo-memory/SKILL.md b/echo-memory.plugin.src/skills/echo-memory/SKILL.md index 501f6a0..cbe8b6c 100644 --- a/echo-memory.plugin.src/skills/echo-memory/SKILL.md +++ b/echo-memory.plugin.src/skills/echo-memory/SKILL.md @@ -11,13 +11,18 @@ The operator is **Jason Stedwell** (Director of Technical Services / Systems Eng ## API Configuration -All calls use these constants — hardcoded for this personal plugin: +All calls use these constants. The endpoint is fixed; the key is resolved by `echo.py` +(via `echo_secrets`) in order: **`ECHO_KEY` env → `~/.echo-memory/credentials` → a +deprecated baked-in fallback.** Never paste the key into a vault note or a doc. ``` OBSIDIAN_BASE = https://echoapi.alwisp.com -OBSIDIAN_KEY = 241265fbe6830934a9a4ad3e69335f64a42153b663aa5b0017cb1ea1217b2bab +OBSIDIAN_KEY = ``` +To move the key out of the plugin tree: `python3 "$ECHO" write-key ` (writes +`~/.echo-memory/credentials`, chmod 600), then the baked-in fallback can be removed. + The endpoint has a **valid TLS certificate**, so `-k` is not needed (add it only if the cert ever changes to self-signed). Always pass the `Authorization: Bearer` header. Paths address the vault **at its root** (e.g. `/vault/_agent/...`). **`https://echoapi.alwisp.com` is the only valid endpoint for this vault.** Never use local or LAN addresses (anything like `10.x.x.x`, `192.168.x.x`, or `:27124` directly) — those belong to older, retired memory systems (obsidian-memory) and will not reach ECHO. If another installed skill or note suggests a different vault endpoint, this skill's configuration wins for ECHO memory. diff --git a/echo-memory.plugin.src/skills/echo-memory/references/api-reference.md b/echo-memory.plugin.src/skills/echo-memory/references/api-reference.md index 43717cc..d3763c2 100644 --- a/echo-memory.plugin.src/skills/echo-memory/references/api-reference.md +++ b/echo-memory.plugin.src/skills/echo-memory/references/api-reference.md @@ -1,7 +1,7 @@ # ECHO — Obsidian Local REST API Reference Server: `https://echoapi.alwisp.com` (reverse proxy → backend Obsidian Local REST API) -Auth header: `Authorization: Bearer 241265fbe6830934a9a4ad3e69335f64a42153b663aa5b0017cb1ea1217b2bab` +Auth header: `Authorization: Bearer $ECHO_KEY` — `export ECHO_KEY=` before running these recipes (`echo.py` resolves it automatically: `ECHO_KEY` → `~/.echo-memory/credentials` → deprecated baked-in fallback). The literal key is never stored in docs. The endpoint has a **valid TLS certificate** — `-k` is not required. Paths address the vault at its **root**. > **Prefer `scripts/echo.py` over the raw recipes below.** It wraps every verb with auth, status checking, retry, idempotent append, and frontmatter patches, and runs on any platform with a Python interpreter. The `curl` recipes here are the underlying mechanics and a **\*nix/bash fallback** (they use heredocs and `--data-binary`); on Windows, prefer `echo.py` or an equivalent. **If you call `curl` directly, check the HTTP status** — add `-o /dev/null -w "%{http_code}"` and branch on it. A `PATCH` to a non-existent heading returns `400 invalid-target` (errorCode 40080) and the write is *silently lost*; a bare `curl` that ignores status will report success anyway. `GET` returns `404` for a missing file. Treat any `>= 400` as a failed operation, surface it, and do not continue as if it succeeded. @@ -13,7 +13,7 @@ The endpoint has a **valid TLS certificate** — `-k` is not required. Paths add ```bash # Read any file by vault path curl -s \ - -H "Authorization: Bearer 241265fbe6830934a9a4ad3e69335f64a42153b663aa5b0017cb1ea1217b2bab" \ + -H "Authorization: Bearer $ECHO_KEY" \ "https://echoapi.alwisp.com/vault/_agent/context/current-context.md" ``` @@ -22,7 +22,7 @@ Returns raw file content (text/markdown). On 404, the file does not exist. ```bash # Read a specific heading's content only curl -s \ - -H "Authorization: Bearer 241265fbe6830934a9a4ad3e69335f64a42153b663aa5b0017cb1ea1217b2bab" \ + -H "Authorization: Bearer $ECHO_KEY" \ "https://echoapi.alwisp.com/vault/_agent/memory/semantic/operator-preferences.md/heading/Operator" ``` @@ -38,7 +38,7 @@ Nested headings: separate levels with `::` (URL-encode spaces as `%20`): ```bash # List contents of a directory (trailing slash required) curl -s \ - -H "Authorization: Bearer 241265fbe6830934a9a4ad3e69335f64a42153b663aa5b0017cb1ea1217b2bab" \ + -H "Authorization: Bearer $ECHO_KEY" \ "https://echoapi.alwisp.com/vault/_agent/sessions/" ``` @@ -56,7 +56,7 @@ cat > /tmp/obs_entry.md << 'OBSEOF' OBSEOF curl -s -X POST \ - -H "Authorization: Bearer 241265fbe6830934a9a4ad3e69335f64a42153b663aa5b0017cb1ea1217b2bab" \ + -H "Authorization: Bearer $ECHO_KEY" \ -H "Content-Type: text/markdown" \ --data-binary @/tmp/obs_entry.md \ "https://echoapi.alwisp.com/vault/inbox/captures/inbox.md" @@ -87,7 +87,7 @@ source_notes: [] OBSEOF curl -s -X PUT \ - -H "Authorization: Bearer 241265fbe6830934a9a4ad3e69335f64a42153b663aa5b0017cb1ea1217b2bab" \ + -H "Authorization: Bearer $ECHO_KEY" \ -H "Content-Type: text/markdown" \ --data-binary @/tmp/obs_file.md \ "https://echoapi.alwisp.com/vault/_agent/sessions/2026-06-05-1430-my-session.md" @@ -109,7 +109,7 @@ Jason prefers concise status updates — lead with the decision. OBSEOF curl -s -X PATCH \ - -H "Authorization: Bearer 241265fbe6830934a9a4ad3e69335f64a42153b663aa5b0017cb1ea1217b2bab" \ + -H "Authorization: Bearer $ECHO_KEY" \ -H "Operation: append" \ -H "Target-Type: heading" \ -H "Target: Operator Preferences::Fact / Pattern" \ @@ -124,7 +124,7 @@ When unsure of the exact heading path, GET the note with the document-map Accept ```bash curl -s \ - -H "Authorization: Bearer 241265fbe6830934a9a4ad3e69335f64a42153b663aa5b0017cb1ea1217b2bab" \ + -H "Authorization: Bearer $ECHO_KEY" \ -H "Accept: application/vnd.olrapi.document-map+json" \ "https://echoapi.alwisp.com/vault/_agent/memory/semantic/operator-preferences.md" ``` @@ -143,7 +143,7 @@ Same call with `Operation: prepend`. ```bash curl -s -X PATCH \ - -H "Authorization: Bearer 241265fbe6830934a9a4ad3e69335f64a42153b663aa5b0017cb1ea1217b2bab" \ + -H "Authorization: Bearer $ECHO_KEY" \ -H "Operation: replace" \ -H "Target-Type: frontmatter" \ -H "Target: updated" \ @@ -158,7 +158,7 @@ curl -s -X PATCH \ ```bash curl -s -X POST \ - -H "Authorization: Bearer 241265fbe6830934a9a4ad3e69335f64a42153b663aa5b0017cb1ea1217b2bab" \ + -H "Authorization: Bearer $ECHO_KEY" \ "https://echoapi.alwisp.com/search/simple/?query=weekly+review" ``` @@ -170,7 +170,7 @@ Returns an array of `{ filename, score, matches: [{ context, match }] }`. ```bash curl -s -X DELETE \ - -H "Authorization: Bearer 241265fbe6830934a9a4ad3e69335f64a42153b663aa5b0017cb1ea1217b2bab" \ + -H "Authorization: Bearer $ECHO_KEY" \ "https://echoapi.alwisp.com/vault/inbox/imports/old-note.md" ``` diff --git a/echo-memory.plugin.src/skills/echo-memory/references/bootstrap.md b/echo-memory.plugin.src/skills/echo-memory/references/bootstrap.md index 0df40d4..4159e55 100644 --- a/echo-memory.plugin.src/skills/echo-memory/references/bootstrap.md +++ b/echo-memory.plugin.src/skills/echo-memory/references/bootstrap.md @@ -20,7 +20,7 @@ python3 "$SCRIPTS/migrate.py" --apply # perform the migration (moves/delet `bootstrap.py` writes through `echo.py`, so every step is status-checked and the marker is written last. The scripts are pure Python (only an interpreter required — no bash, no platform-specific `date`), so they run identically on Windows, macOS, and Linux. The manual steps below document what the script does (and serve as a fallback if the script can't run). ``` -AUTH="Authorization: Bearer 241265fbe6830934a9a4ad3e69335f64a42153b663aa5b0017cb1ea1217b2bab" +AUTH="Authorization: Bearer $ECHO_KEY" # export ECHO_KEY first; echo.py resolves it automatically BASE="https://echoapi.alwisp.com" ``` diff --git a/echo-memory.plugin.src/skills/echo-memory/scripts/echo.py b/echo-memory.plugin.src/skills/echo-memory/scripts/echo.py index 87d6730..703bb2f 100644 --- a/echo-memory.plugin.src/skills/echo-memory/scripts/echo.py +++ b/echo-memory.plugin.src/skills/echo-memory/scripts/echo.py @@ -66,9 +66,14 @@ for _stream in (sys.stdout, sys.stderr): pass BASE = os.environ.get("ECHO_BASE", "https://echoapi.alwisp.com").rstrip("/") -# The token stays hardcoded for this personal plugin; ECHO_KEY env still overrides. + +# Key resolution is centralized in echo_secrets: ECHO_KEY env -> ~/.echo-memory/credentials +# -> the deprecated baked-in fallback below. Move the key out of the tree with +# `echo.py write-key `; remove DEFAULT_KEY before the 1.0 tag (see ROADMAP-1.0.md > M1). +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import echo_secrets # noqa: E402 DEFAULT_KEY = "241265fbe6830934a9a4ad3e69335f64a42153b663aa5b0017cb1ea1217b2bab" -KEY = os.environ.get("ECHO_KEY") or DEFAULT_KEY +KEY = echo_secrets.resolve_key(DEFAULT_KEY) VERIFY = os.environ.get("ECHO_VERIFY", "1") != "0" LOCK_TTL = int(os.environ.get("ECHO_LOCK_TTL", "900")) @@ -207,10 +212,20 @@ def cmd_search(query: list[str]) -> int: return 0 +def _qwrite(method: str, url: str, data: bytes | None = None, headers: dict | None = None): + """A mutating request that survives an outage: routes through echo_queue.safe_request, + which enqueues the write and returns (202, ...) when the vault is unreachable (H2).""" + import echo_queue + return echo_queue.safe_request(method, url, data=data, headers=headers) + + def cmd_put(path: str, file_arg: str | None) -> int: data = read_body(file_arg) - status, body = request("PUT", vault_url(path), data=data, + status, body = _qwrite("PUT", vault_url(path), data=data, headers={"Content-Type": "text/markdown"}) + if status == 202: + print(f"queued (offline): PUT {path} — will sync on the next reachable session") + return 0 check(status, body, f"put {path}") if VERIFY: status, _ = request("GET", vault_url(path)) @@ -222,8 +237,11 @@ def cmd_put(path: str, file_arg: str | None) -> int: def cmd_post(path: str, file_arg: str | None) -> int: data = read_body(file_arg) - status, body = request("POST", vault_url(path), data=data, + status, body = _qwrite("POST", vault_url(path), data=data, headers={"Content-Type": "text/markdown"}) + if status == 202: + print(f"queued (offline): POST {path} — will sync on the next reachable session") + return 0 check(status, body, f"post {path}") print(f"ok: POST {path}") return 0 @@ -238,10 +256,15 @@ def cmd_append(path: str, line: str) -> int: if line.rstrip("\r") in existing: print(f"skip: line already present in {path}") return 0 + elif status == 0: + pass # offline: skip the idempotency read; queue the POST (flush dedups by content) elif status != 404: check(status, body, f"append(read) {path}") - status, body = request("POST", vault_url(path), data=f"{line}\n".encode(), + status, body = _qwrite("POST", vault_url(path), data=f"{line}\n".encode(), headers={"Content-Type": "text/markdown"}) + if status == 202: + print(f"queued (offline): APPEND {path} — will sync on the next reachable session") + return 0 check(status, body, f"append {path}") print(f"ok: APPEND {path}") return 0 @@ -253,7 +276,7 @@ def cmd_patch(path: str, op: str, target_type: str, target: str, file_arg: str | if target_type not in {"heading", "frontmatter", "block"}: raise EchoError("patch: target-type must be heading, frontmatter, or block", 2) data = normalize_patch_body(read_body(file_arg), op, target_type) - status, body = request( + status, body = _qwrite( "PATCH", vault_url(path), data=data, @@ -264,6 +287,9 @@ def cmd_patch(path: str, op: str, target_type: str, target: str, file_arg: str | "Content-Type": "application/json" if target_type == "frontmatter" else "text/markdown", }, ) + if status == 202: + print(f"queued (offline): PATCH {op} '{target}' -> {path} — will sync on the next reachable session") + return 0 check(status, body, f"patch {path} ({target})") print(f"ok: PATCH {op} {target_type} '{target}' -> {path}") return 0 @@ -274,7 +300,10 @@ def cmd_fm(path: str, field: str, value: str) -> int: def cmd_delete(path: str) -> int: - status, body = request("DELETE", vault_url(path)) + status, body = _qwrite("DELETE", vault_url(path)) + if status == 202: + print(f"queued (offline): DELETE {path} — will sync on the next reachable session") + return 0 check(status, body, f"delete {path}") print(f"ok: DELETE {path}") return 0 @@ -290,7 +319,7 @@ def parse_lock_time(value: str) -> int: return int(time.time()) -def cmd_lock(owner: str) -> int: +def cmd_lock(owner: str, quiet: bool = False) -> int: status, body = request("GET", vault_url(LOCK_PATH)) if status == 200 and body.strip(): current = body.decode(errors="replace").strip() @@ -310,11 +339,12 @@ def cmd_lock(owner: str) -> int: if held_owner != owner: print(f"lock race: now held by '{held_owner}', not '{owner}' — backing off", file=sys.stderr) return 75 - print(f"ok: locked by {owner}") + if not quiet: + print(f"ok: locked by {owner}") return 0 -def cmd_unlock(owner: str) -> int: +def cmd_unlock(owner: str, quiet: bool = False) -> int: status, body = request("GET", vault_url(LOCK_PATH)) if status == 200: held_owner = body.decode(errors="replace").split(" @ ", 1)[0].strip() @@ -324,7 +354,8 @@ def cmd_unlock(owner: str) -> int: status, body = request("DELETE", vault_url(LOCK_PATH)) if status != 404: check(status, body, "unlock") - print("ok: unlocked") + if not quiet: + print("ok: unlocked") return 0 @@ -403,22 +434,67 @@ def cmd_load() -> int: ("today", f"journal/daily/{today()}.md"), ("inbox", "inbox/captures/inbox.md"), ] + import echo_queue + # H2: sync any writes queued during a prior outage, best-effort and quiet on empty. + try: + synced = echo_queue.flush() + if synced: + print(f"(synced {synced} write(s) queued during a prior offline session)\n") + except Exception as exc: # noqa: BLE001 — never let queue upkeep block a load + print(f"(queue flush skipped: {exc})\n", file=sys.stderr) + marker_missing = False + heartbeat_absent = False + offline = False for label, path in targets: status, body = request("GET", vault_url(path)) - print(f"===== {label}: {path} (HTTP {status}) =====") if status == 200: + echo_queue.cache_put(path, body) # refresh last-known-good for offline fallback + print(f"===== {label}: {path} (HTTP 200) =====") text = body.decode(errors="replace") sys.stdout.write(text) if not text.endswith("\n"): sys.stdout.write("\n") elif status == 404: + print(f"===== {label}: {path} (HTTP 404) =====") print("(absent — fine)") if label == "marker": marker_missing = True + if label == "heartbeat": + heartbeat_absent = True + elif status == 0: # vault unreachable -> degrade to last-known-good cache + offline = True + cached = echo_queue.cache_get(path) + if cached is not None: + print(f"===== {label}: {path} (OFFLINE — serving stale cache) =====") + sys.stdout.write(cached.decode(errors="replace")) + if not cached.endswith(b"\n"): + sys.stdout.write("\n") + else: + print(f"===== {label}: {path} (OFFLINE — no cache) =====") + print("(unreachable and not cached)") else: + print(f"===== {label}: {path} (HTTP {status}) =====") print(f"(error HTTP {status}: {body.decode(errors='replace')[:200]})") print() + # M3: heartbeat pointer missing/stale -> fall back to the recent sessions listing + # (matches the documented loading procedure) so orientation works without the pointer. + if heartbeat_absent and not offline: + st, body = request("GET", vault_url("_agent/sessions/")) + if st == 200: + try: + files = sorted((f for f in json.loads(body).get("files", []) if f.endswith(".md")), + reverse=True) + except json.JSONDecodeError: + files = [] + if files: + print("===== recent sessions (heartbeat absent — fallback) =====") + for f in files[:5]: + print(f" _agent/sessions/{f}") + print() + if offline: + print("NOTE: vault unreachable — context above is last-known-good cache; writes this " + "session will be queued and synced when the vault returns.") if marker_missing: print("NOTE: marker absent — vault not bootstrapped. Run bootstrap.py before relying on memory.") return 0 @@ -428,6 +504,8 @@ def build_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser(description="Validated cross-platform ECHO vault client") sub = parser.add_subparsers(dest="cmd", required=True) sub.add_parser("load") + sub.add_parser("flush") # H2: replay writes queued during a prior outage + sub.add_parser("doctor") # M3: one-call readiness check sub.add_parser("get").add_argument("path") sub.add_parser("map").add_argument("path") sub.add_parser("ls").add_argument("path") @@ -443,9 +521,13 @@ def build_parser() -> argparse.ArgumentParser: sub.add_parser("delete").add_argument("path") sub.add_parser("lock").add_argument("owner") sub.add_parser("unlock").add_argument("owner") + sub.add_parser("write-key").add_argument("token") # M1: move the key to ~/.echo-memory/credentials p = sub.add_parser("scope") p.add_argument("subcommand", nargs="?", default="show", choices=["show", "set"]) p.add_argument("text", nargs="?") + p = sub.add_parser("reflect") # H5: apply a JSON proposal set (dry-run unless --apply) + p.add_argument("file", nargs="?") + p.add_argument("--apply", action="store_true") # High-level operations (entity index + cross-links + recall). See echo_ops.py. sub.add_parser("resolve").add_argument("mention", nargs="+") @@ -462,6 +544,8 @@ def build_parser() -> argparse.ArgumentParser: p.add_argument("--date") p.add_argument("--domain", default="business") p.add_argument("--no-log", action="store_true") + p.add_argument("--json", action="store_true") # M4: emit a JSON result envelope + p.add_argument("--dry-run", action="store_true") # M4: preview the plan, write nothing return parser @@ -470,6 +554,25 @@ def main(argv: list[str] | None = None) -> int: try: if args.cmd == "load": return cmd_load() + if args.cmd == "doctor": + import echo_doctor + return echo_doctor.run() + if args.cmd == "flush": + import echo_queue + n = echo_queue.flush() + remaining = len(echo_queue.pending()) + if n: + print(f"ok: flushed {n} queued write(s)" + + (f"; {remaining} still queued (vault unreachable)" if remaining else "")) + elif remaining: + print(f"ok: {remaining} write(s) still queued (vault unreachable)") + else: + print("ok: queue empty") + return 0 + if args.cmd == "write-key": + path = echo_secrets.write_key(args.token) + print(f"ok: wrote credentials to {path} (chmod 600 best-effort); ECHO_KEY env still overrides") + return 0 if args.cmd == "get": return cmd_get(args.path) if args.cmd == "map": @@ -498,6 +601,16 @@ def main(argv: list[str] | None = None) -> int: return cmd_unlock(args.owner) if args.cmd == "scope": return cmd_scope(args.subcommand, args.text) + if args.cmd == "reflect": + import echo_reflect + raw = read_body(args.file).decode("utf-8", errors="replace") + try: + proposals = json.loads(raw) + except json.JSONDecodeError as exc: + raise EchoError(f"reflect: proposals must be a JSON array ({exc})", 2) + if not isinstance(proposals, list): + raise EchoError("reflect: proposals must be a JSON array of objects", 2) + return echo_reflect.apply(proposals, confirm=args.apply) if args.cmd in ("resolve", "recall", "link", "capture"): import echo_ops # lazy: only the high-level ops pull in the index/link modules if args.cmd == "resolve": @@ -510,7 +623,8 @@ def main(argv: list[str] | None = None) -> int: args.kind, args.title, args.file, status_v=args.status, aliases=args.aliases.split(",") if args.aliases else [], sources=args.source.split(",") if args.source else [], - date=args.date, domain=args.domain, inbox=args.inbox, no_log=args.no_log) + date=args.date, domain=args.domain, inbox=args.inbox, no_log=args.no_log, + as_json=args.json, dry_run=args.dry_run) except EchoError as exc: print(f"echo.py: {exc}", file=sys.stderr) return exc.code diff --git a/echo-memory.plugin.src/skills/echo-memory/scripts/echo_concurrency.py b/echo-memory.plugin.src/skills/echo-memory/scripts/echo_concurrency.py new file mode 100644 index 0000000..de03332 --- /dev/null +++ b/echo-memory.plugin.src/skills/echo-memory/scripts/echo_concurrency.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python3 +"""echo_concurrency.py — H3: make the multi-writer story actually safe. [v1.0 SCAFFOLD — not yet wired] + +The plugin advertises a shared vault (Claude Code + CoWork), but two real races exist: + 1. The entity index is a full-file load -> mutate -> PUT (echo_index.load/save) with + NO lock. Two concurrent `capture` calls both read the old index; last PUT wins; + the other entity is silently dropped from the index (the note file survives, but + resolve/recall miss it until the next sweep). + 2. The advisory lock is MANUAL (echo.py cmd_lock) — capture / scope set never take it; + it relies on the model remembering to. + +This module provides (a) a context manager that auto-acquires/releases the existing +advisory lock around a write burst, with crash-safe release, and (b) an atomic +index-update that re-reads UNDER the lock before saving, so concurrent captures merge +instead of clobber. + +ACCEPTANCE (see ROADMAP-1.0.md > H3): + - two concurrent captures both land in the index (no lost entries) + - lock auto-released on normal AND error exit +""" +from __future__ import annotations + +import contextlib +import os +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import echo # noqa: E402 +import echo_index as idx_mod # noqa: E402 + + +def auto_owner() -> str: + """A stable-per-process owner id: client tag + pid. Avoids Math.random/time-based + ids; pid is unique enough for cooperative locking and is reproducible in tests.""" + tag = os.environ.get("ECHO_CLIENT", "cc") + return f"{tag}-{os.getpid()}" + + +@contextlib.contextmanager +def vault_lock(owner: str | None = None, required: bool = False): + """Hold the advisory lock for a write burst; release on exit (incl. exceptions). + + Yields `held: bool`. The lock is cooperative (read-back-confirmed, TTL-reclaimable), + so if another session holds it we do NOT block: with `required=False` we yield + `held=False` and let the caller proceed (advisory) or warn; with `required=True` we + raise. Release is best-effort and never masks the body's own exception. + """ + owner = owner or auto_owner() + rc = echo.cmd_lock(owner, quiet=True) + held = rc == 0 + if not held and required: + raise echo.EchoError(f"vault_lock: could not acquire lock for '{owner}' (held elsewhere)", 75) + try: + yield held + finally: + if held: + try: + echo.cmd_unlock(owner, quiet=True) + except Exception: # noqa: BLE001 — release is best-effort; TTL reclaims a leak + pass + + +def atomic_index_update(mutate, owner: str | None = None): + """Apply `mutate(index)` to the entity index without losing a concurrent writer's + entry: take the lock, **re-read the index fresh** (not a stale in-memory copy), + run the mutator, save, release. Returns the freshly-read-and-saved index. + + The fresh re-read inside the critical section is the core of the fix — even if the + advisory lock can't be taken (another session active), reading immediately before + the save collapses the read-modify-write window to two calls instead of spanning the + whole capture, so an unlocked update is still far safer than the prior code. + """ + with vault_lock(owner) as held: + index = idx_mod.load() + mutate(index) + idx_mod.save(index) + if not held: + print("echo_concurrency: entity index updated WITHOUT the advisory lock " + "(another session may be active); the fresh re-read minimizes the race.", + file=sys.stderr) + return index diff --git a/echo-memory.plugin.src/skills/echo-memory/scripts/echo_doctor.py b/echo-memory.plugin.src/skills/echo-memory/scripts/echo_doctor.py new file mode 100644 index 0000000..eceb3bf --- /dev/null +++ b/echo-memory.plugin.src/skills/echo-memory/scripts/echo_doctor.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python3 +"""echo_doctor.py — M3: one-call readiness check. [v1.0 SCAFFOLD — run() implemented] + +There is no single "is everything OK" command. doctor checks, in order, the things +that make ECHO usable this session and prints a green/red report. Intended to back an +`echo.py doctor` subcommand and the start of /echo-load when something looks wrong. + +Also tracked here (TODO): complete the `load` fallback — echo.cmd_load reads the +heartbeat pointer but never lists _agent/sessions/ when it's missing/stale, though +SKILL.md:122 documents that fallback. See patch_load_fallback() below. + +Exit: 0 all green · 1 a check is red. +""" +from __future__ import annotations + +import os +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import echo # noqa: E402 + +MIN_PY = (3, 9) + + +def run() -> int: + reds = 0 + + def line(ok: bool, label: str, detail: str = "") -> None: + nonlocal reds + reds += 0 if ok else 1 + print(f" [{'OK ' if ok else 'RED'}] {label}" + (f" — {detail}" if detail else "")) + + print(f"echo doctor — endpoint {echo.BASE}") + + # 1. interpreter + line(sys.version_info >= MIN_PY, f"python >= {MIN_PY[0]}.{MIN_PY[1]}", + f"running {sys.version_info.major}.{sys.version_info.minor}") + + # 2. reachability + auth + marker, in one GET of the bootstrap marker + status, body = echo.request("GET", echo.vault_url("_agent/echo-vault.md")) + if status == 0: + line(False, "vault reachable", body.decode(errors="replace")[:120]) + print("\ndoctor: endpoint unreachable — is Obsidian + the Local REST API running?") + return 1 + line(status not in (401, 403), "auth accepted", f"HTTP {status}" if status in (401, 403) else "") + if status == 404: + line(False, "vault bootstrapped", "marker absent — run bootstrap.py") + elif status == 200: + text = body.decode(errors="replace") + ver = next((ln.split(":", 1)[1].strip() for ln in text.splitlines() + if ln.startswith("schema_version:")), "?") + line(True, "vault bootstrapped", f"schema_version={ver}") + else: + line(status < 400, "marker fetch", f"HTTP {status}") + + # 3. key source (M1) + invariants pointer. + import echo_secrets + key_src = ("ECHO_KEY env" if os.environ.get("ECHO_KEY") + else "credentials file" if (echo_secrets._state_dir() / "credentials").exists() + else "baked-in fallback (deprecated — see API-KEY-SETUP.md)") + line(os.environ.get("ECHO_KEY") is not None or (echo_secrets._state_dir() / "credentials").exists(), + "API key source", key_src) + print(" [i] invariants — run `/echo-health` (vault_lint.py) for the full check") + + print(f"\ndoctor: {'all green' if reds == 0 else f'{reds} issue(s) — see RED above'}") + return 1 if reds else 0 + + +if __name__ == "__main__": + raise SystemExit(run()) diff --git a/echo-memory.plugin.src/skills/echo-memory/scripts/echo_ops.py b/echo-memory.plugin.src/skills/echo-memory/scripts/echo_ops.py index 7b9a219..138d955 100644 --- a/echo-memory.plugin.src/skills/echo-memory/scripts/echo_ops.py +++ b/echo-memory.plugin.src/skills/echo-memory/scripts/echo_ops.py @@ -14,7 +14,6 @@ from __future__ import annotations import json import re import sys -import urllib.parse from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parent)) @@ -54,79 +53,11 @@ def link(a_path: str, b_path: str) -> int: # ----------------------------------------------------------------- recall ----- -def _resolve_link_to_path(target: str, nmap: dict) -> str | None: - target = target.strip() - if "/" in target: - return target if target.endswith(".md") else target + ".md" - return nmap.get(idx_mod.slugify(target)) - - -def _brief(path: str) -> None: - text = links.get_text(path) - if text is None: - return - print(f"\n### {path}") - fm_type = re.search(r"(?m)^type:\s*(.+)$", text) - if fm_type: - print(f"_type: {fm_type.group(1).strip()}_") - # Prefer a Status paragraph; else the first handful of non-empty body lines. - body = text.split("\n---", 2)[-1] if text.startswith("---") else text - status = re.search(r"(?ms)^##\s+Status\s*$(.*?)(?=^##\s|\Z)", body) - if status and status.group(1).strip(): - print(status.group(1).strip()[:400]) - return - shown = 0 - for line in body.splitlines(): - if not line.strip() or line.startswith("# "): - continue - print(line[:200]) - shown += 1 - if shown >= 8: - break - - -def recall(query, limit: int = 6) -> int: - q = " ".join(query) if isinstance(query, list) else query - index = idx_mod.load() - nmap = idx_mod.name_map(index) - - status, body = echo.request("POST", f"{echo.BASE}/search/simple/?query={urllib.parse.quote_plus(q)}") - echo.check(status, body, "recall search") - try: - results = json.loads(body) - except json.JSONDecodeError: - results = [] - hits = [] - for r in results[:limit]: - fn = r.get("filename") or r.get("path") - if fn and fn not in hits: - hits.append(fn) - rslug, e = idx_mod.resolve(index, q) - if e and e.get("path") and e["path"] not in hits: - hits.insert(0, e["path"]) - - seen = set(hits) - neighbors = [] # (via, path) - for p in hits: - text = links.get_text(p) - if text is None: - continue - targets = set(links.all_wikilinks(text)) | set(links.source_notes(text)) - for t in targets: - tp = _resolve_link_to_path(t, nmap) - if tp and tp not in seen: - seen.add(tp) - neighbors.append((p, tp)) - - print(f"== recall: {q} ==") - print(f"\n# Primary hits ({len(hits)})") - for p in hits: - _brief(p) - print(f"\n# Linked context ({len(neighbors)})") - for via, p in neighbors: - print(f"\n(via {via})") - _brief(p) - return 0 +def recall(query, limit: int = 8) -> int: + """Hybrid lexical (BM25) + graph recall — implemented in echo_recall. Kept here as + the stable public entrypoint (echo.py and /echo-recall call echo_ops.recall).""" + import echo_recall # lazy: only pulls the BM25 modules when recall is actually run + return echo_recall.recall(query, limit=limit) # ----------------------------------------------------------- agent log ------- @@ -197,7 +128,30 @@ def _append_to_existing(path: str, kind: str, today_s: str, body_text: str) -> N def capture(kind: str | None, title: str, file_arg: str | None, status_v: str = "", aliases=None, sources=None, date: str | None = None, domain: str = "business", - inbox: bool = False, no_log: bool = False) -> int: + inbox: bool = False, no_log: bool = False, as_json: bool = False, + dry_run: bool = False) -> int: + import contextlib + import io + import echo_output + import echo_quality + + real_stdout = sys.stdout + + def quiet(): + # M4: in --json mode, swallow the helper "ok:" chatter so stdout is clean JSON. + return contextlib.redirect_stdout(io.StringIO()) if as_json else contextlib.nullcontext() + + def done(action: str, path: str, links: int = 0, ok: bool = True, dry: bool = False) -> int: + if as_json: + act = f"dry-run:{action}" if dry else action + env = echo_output.envelope(act, {"kind": kind, "path": path, "title": title, + "links_added": links}, ok=ok) + print(json.dumps(env, ensure_ascii=False), file=real_stdout) + elif dry: + print(f"would {action} {kind or '-'} -> {path}") + # non-dry human output is the helper "ok:" lines + the summary printed by the caller. + return 0 if ok else 1 + body_text = echo.read_body(file_arg).decode("utf-8", errors="replace") today_s = echo.today() aliases = [a.strip() for a in (aliases or []) if a.strip()] @@ -205,44 +159,74 @@ def capture(kind: str | None, title: str, file_arg: str | None, status_v: str = # Unknown home -> defer to the inbox (a single idempotent capture line). if inbox or not kind: + if dry_run: + return done("inbox", "inbox/captures/inbox.md", dry=True) line = f"- {today_s}: {title}" if body_text.strip(): line += f" — {body_text.strip().splitlines()[0]}" - return echo.cmd_append("inbox/captures/inbox.md", line) + with quiet(): + rc = echo.cmd_append("inbox/captures/inbox.md", line) + return done("inbox", "inbox/captures/inbox.md", ok=rc == 0) slug = idx_mod.slugify(title) index = idx_mod.load() _, existing = idx_mod.resolve(index, title) + existing_reachable = bool(existing and echo.request("GET", echo.vault_url(existing["path"]))[0] == 200) - if existing and echo.request("GET", echo.vault_url(existing["path"]))[0] == 200: - path = existing["path"] - kind = existing.get("kind", kind) - _append_to_existing(path, kind, today_s, body_text) - action = "updated" + if dry_run: + if existing_reachable: + return done("update", existing["path"], dry=True) + s2 = echo_quality.safe_slug(set(index.get("entities", {}).keys()), slug) + return done("create", idx_mod.derive_path(kind, s2, date=date, domain=domain), dry=True) + + with quiet(): + if existing_reachable: + path = existing["path"] + kind = existing.get("kind", kind) + _append_to_existing(path, kind, today_s, body_text) + action = "updated" + else: + if not status_v and kind == "project": + status_v = "active" + # M2: don't let a 40-char-truncated slug silently collide with a different + # entity already in the index — disambiguate to slug-2, slug-3, ... + slug = echo_quality.safe_slug(set(index.get("entities", {}).keys()), slug) + path = idx_mod.derive_path(kind, slug, date=date, domain=domain) + note = _build_note(idx_mod.KIND_TYPE.get(kind, kind), status_v, today_s, title, body_text, sources) + echo.cmd_put(path, echo.temp_file(note.encode())) + action = "created" + + # H3: the entity-index write is the race the review flagged — a bare load->save lets + # two concurrent captures clobber each other's entry. Route it through a lock-guarded, + # fresh-re-read transaction. Returns the fresh index (used by auto-link below). + import echo_concurrency + index = echo_concurrency.atomic_index_update( + lambda idx: idx_mod.upsert(idx, slug, path, kind, title, aliases)) + + # auto-link: any other known entity CONFIDENTLY named in the body (M2: is_confident_link + # rejects short/common tokens so a 3-char alias or a word like "API" can't link everywhere). + linked = 0 + for s2, e2 in index.get("entities", {}).items(): + if e2.get("path") in (None, path): + continue + names = [n for n in [e2.get("title", ""), s2, *e2.get("aliases", [])] if n] + if any(echo_quality.is_confident_link(n, body_text) for n in names): + ca, cb = links.link_bidirectional(path, e2["path"]) + linked += int(ca or cb) + + # Keep the BM25 recall index current for this note (best-effort; lock-guarded inside + # update_note, so it can't clobber a concurrent writer either). + try: + import echo_recall + echo_recall.update_note(path, links.get_text(path) or "") + except Exception as exc: # never let recall-index upkeep fail a capture + print(f"echo_ops: recall-index update skipped ({exc})", file=sys.stderr) + + if not no_log: + ensure_daily_log(f"- {today_s}: {action} {kind} [[{links.link_token(path)}]]") + + if as_json: + done(action, path, links=linked) else: - if not status_v and kind == "project": - status_v = "active" - path = idx_mod.derive_path(kind, slug, date=date, domain=domain) - note = _build_note(idx_mod.KIND_TYPE.get(kind, kind), status_v, today_s, title, body_text, sources) - echo.cmd_put(path, echo.temp_file(note.encode())) - action = "created" - - idx_mod.upsert(index, slug, path, kind, title, aliases) - idx_mod.save(index) - - # auto-link: any other known entity whose name/alias appears in the body, matched - # on word boundaries (and >=3 chars) so a short alias can't match inside a word. - linked = 0 - for s2, e2 in index.get("entities", {}).items(): - if e2.get("path") in (None, path): - continue - names = [n for n in [e2.get("title", ""), s2, *e2.get("aliases", [])] if len(n) >= 3] - if any(re.search(rf"\b{re.escape(n)}\b", body_text, re.IGNORECASE) for n in names): - ca, cb = links.link_bidirectional(path, e2["path"]) - linked += int(ca or cb) - - if not no_log: - ensure_daily_log(f"- {today_s}: {action} {kind} [[{links.link_token(path)}]]") - - print(f"ok: {action} {kind} -> {path}" + (f"; auto-linked {linked}" if linked else "")) + print(f"ok: {action} {kind} -> {path}" + (f"; auto-linked {linked}" if linked else "")) return 0 diff --git a/echo-memory.plugin.src/skills/echo-memory/scripts/echo_output.py b/echo-memory.plugin.src/skills/echo-memory/scripts/echo_output.py new file mode 100644 index 0000000..f0790d6 --- /dev/null +++ b/echo-memory.plugin.src/skills/echo-memory/scripts/echo_output.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python3 +"""echo_output.py — M4: machine-readable output + dry-run plumbing. [v1.0 SCAFFOLD — implemented] + +The high-level ops (capture/recall/resolve/scope) print prose, forcing the agent to +re-parse free text to act on a result. This provides a single JSON envelope and a +small dry-run helper so every op can emit a structured result and preview writes. + +INTEGRATION (merge step): thread a global `--json` / `--dry-run` flag through echo.py's +parser and have each cmd_* return its data dict; emit() it. In --dry-run, the write +verbs print the envelope with action="dry-run" and perform no network mutation. +""" +from __future__ import annotations + +import json +import sys + + +def envelope(action: str, data: dict | None = None, ok: bool = True, + error: str | None = None) -> dict: + """Canonical result shape for every high-level op.""" + env = {"ok": ok, "action": action} + if data: + env.update(data) + if error: + env["error"] = error + return env + + +def emit(env: dict, as_json: bool) -> None: + """Print either the JSON envelope (machine mode) or a one-line human summary.""" + if as_json: + print(json.dumps(env, ensure_ascii=False)) + return + if not env.get("ok"): + print(f"error: {env.get('error', 'failed')}", file=sys.stderr) + return + bits = [env.get("action", "ok")] + if env.get("path"): + bits.append(str(env["path"])) + if env.get("links_added"): + bits.append(f"+{env['links_added']} links") + print("ok: " + " ".join(bits)) + + +def dry_run_envelope(action: str, data: dict | None = None) -> dict: + """Result for a previewed-but-not-executed write.""" + return envelope(f"dry-run:{action}", data, ok=True) diff --git a/echo-memory.plugin.src/skills/echo-memory/scripts/echo_quality.py b/echo-memory.plugin.src/skills/echo-memory/scripts/echo_quality.py new file mode 100644 index 0000000..fb1f0be --- /dev/null +++ b/echo-memory.plugin.src/skills/echo-memory/scripts/echo_quality.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python3 +"""echo_quality.py — M2: keep the graph (which recall depends on) clean. +[v1.0 SCAFFOLD — helpers implemented; integration into echo_links/echo_index is the merge step] + +Two graph-polluting bugs today: + 1. Auto-link fires on any indexed entity whose name/alias is >=3 chars and word-matches + the body (echo_ops.py:236). A concept titled "API" or a 2-3 char alias links to every + note that happens to mention the word -> noisy, wrong edges. + 2. slugify truncates to 40 chars (echo_index.py:58) with NO collision check, so two + distinct long titles can resolve to the same slug -> same path -> silent merge. + +This module supplies the two guards. Integration: + * echo_ops auto-link loop -> gate each candidate on is_confident_link(name, body) + * echo_index.derive_path -> route the slug through safe_slug(existing_slugs, slug) +""" +from __future__ import annotations + +import re + +# Tokens too short or too common to be a confident entity reference on their own. +MIN_NAME_LEN = 4 +_COMMON = frozenset( + "api app data note task user team work code test main repo file plan goal item " + "the and for with mpm".split() +) + + +def is_confident_link(name: str, body: str) -> bool: + """True only if `name` is a strong enough signal to auto-link on its appearance in + `body`. Rejects short names, common words, and non-matches. Multi-word names are + trusted at lower length (a two-word phrase is rarely incidental).""" + n = (name or "").strip() + if not n: + return False + multiword = len(n.split()) > 1 + if not multiword: + if len(n) < MIN_NAME_LEN or n.lower() in _COMMON: + return False + return re.search(rf"\b{re.escape(n)}\b", body or "", re.IGNORECASE) is not None + + +def safe_slug(existing_slugs, slug: str, max_len: int = 40) -> str: + """Return `slug` if free, else a disambiguated `slug-2`, `slug-3`, ... that still + fits `max_len` and does not collide with anything in `existing_slugs`.""" + existing = set(existing_slugs or ()) + if slug not in existing: + return slug + n = 2 + while True: + suffix = f"-{n}" + base = slug[: max_len - len(suffix)] if len(slug) + len(suffix) > max_len else slug + cand = f"{base}{suffix}" + if cand not in existing: + return cand + n += 1 diff --git a/echo-memory.plugin.src/skills/echo-memory/scripts/echo_queue.py b/echo-memory.plugin.src/skills/echo-memory/scripts/echo_queue.py new file mode 100644 index 0000000..e57c6ff --- /dev/null +++ b/echo-memory.plugin.src/skills/echo-memory/scripts/echo_queue.py @@ -0,0 +1,191 @@ +#!/usr/bin/env python3 +"""echo_queue.py — H2: offline write-ahead queue + read cache. [v1.0 — wired] + +Today vault-unreachable => "proceed without memory" (SKILL.md), so a 502 (Obsidian not +running — the most common real failure) loses everything said that session. This module +makes explicit writes durable across an outage and lets cold-start `load` degrade to +last-known context instead of nothing. + +DESIGN: + * Outbox : append-only NDJSON of intended writes. Replayed in order on the next + reachable session. Replay is idempotent by construction — PUT/DELETE/PATCH- + replace overwrite, and POST / PATCH-append are content-deduped (skip if the + line/block is already present), the same strategy echo.py append uses. Each + record carries an idem_key so the SAME write is never queued twice. + * Cache : last-known-good GET bodies, so `load` has a fallback. Written by cmd_load. + * Location: a LOCAL state dir — it CANNOT live in the vault, because the vault is the + thing that's down. Default ~/.echo-memory/, override ECHO_STATE_DIR. + +Integration seam: `safe_request()` — the write verbs (cmd_put/post/append/patch/delete) +call it instead of echo.request; on an outage it enqueues and returns a synthesized +(202, b"queued ...") so the caller reports "queued, will sync" instead of failing. +""" +from __future__ import annotations + +import base64 +import hashlib +import json +import os +import sys +import urllib.parse +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import echo # noqa: E402 + +MUTATING = {"PUT", "POST", "PATCH", "DELETE"} + + +def state_dir() -> Path: + return Path(os.environ.get("ECHO_STATE_DIR") or (Path.home() / ".echo-memory")) + + +def outbox_path() -> Path: + return state_dir() / "outbox.ndjson" + + +def cache_dir() -> Path: + return state_dir() / "cache" + + +def _ensure_dirs() -> None: + state_dir().mkdir(parents=True, exist_ok=True) + cache_dir().mkdir(parents=True, exist_ok=True) + + +def _key(method: str, url: str, data: bytes | None) -> str: + h = hashlib.sha1() + h.update(method.encode()) + h.update(b"\0") + h.update(url.encode()) + h.update(b"\0") + h.update(data or b"") + return h.hexdigest() + + +# --------------------------------------------------------------------- outbox +def enqueue(method: str, url: str, data: bytes | None, headers: dict, idem_key: str | None = None) -> None: + """Append one intended write to the outbox (NDJSON; base64 body for binary safety). + Dedups: if a record with the same idem_key is already pending, do nothing.""" + idem_key = idem_key or _key(method, url, data) + if any(rec.get("idem_key") == idem_key for rec in pending()): + return + _ensure_dirs() + rec = { + "method": method.upper(), "url": url, + "body_b64": base64.b64encode(data).decode() if data else None, + "headers": headers or {}, "idem_key": idem_key, + # No wall-clock: replay order is file order, not a timestamp (keeps this testable). + } + with outbox_path().open("a", encoding="utf-8") as fh: + fh.write(json.dumps(rec, ensure_ascii=False) + "\n") + + +def pending() -> list[dict]: + p = outbox_path() + if not p.exists(): + return [] + return [json.loads(ln) for ln in p.read_text(encoding="utf-8").splitlines() if ln.strip()] + + +def _rewrite(records: list[dict]) -> None: + p = outbox_path() + if not records: + if p.exists(): + p.unlink() + return + _ensure_dirs() + tmp = p.with_suffix(".ndjson.tmp") + tmp.write_text("".join(json.dumps(r, ensure_ascii=False) + "\n" for r in records), encoding="utf-8") + os.replace(tmp, p) # atomic swap so a crash mid-flush can't corrupt the queue + + +def _rebase(url: str) -> str: + """Re-point a queued URL at the CURRENT echo.BASE. Writes are queued with the base + that was active when they failed; on replay the vault may be back at a different base + (e.g. an endpoint migration), so we always reconstruct the origin from echo.BASE.""" + parts = urllib.parse.urlsplit(url) + return echo.BASE + parts.path + (("?" + parts.query) if parts.query else "") + + +def _replay(rec: dict) -> str: + """Re-issue one queued write idempotently. Returns 'ok' (landed/already-present), + 'drop' (permanent 4xx — can never succeed), or 'retry' (still offline / 5xx).""" + method = rec["method"] + url = _rebase(rec["url"]) + data = base64.b64decode(rec["body_b64"]) if rec.get("body_b64") else None + headers = rec.get("headers") or {} + + # Append-style writes: skip if the content is already present (idempotency on replay). + is_append = method == "POST" or (method == "PATCH" and headers.get("Operation") == "append") + if is_append and data: + st, cur = echo.request("GET", url) + if st == 0: + return "retry" # still offline + if st == 200 and data.decode("utf-8", "replace").strip("\n") in cur.decode("utf-8", "replace"): + return "ok" # already landed (or a duplicate we must not re-add) + + st, body = echo.request(method, url, data=data, headers=headers) + if st == 0 or 500 <= st < 600: + return "retry" + if 400 <= st < 500: + print(f"echo_queue: dropping a queued {method} {url} — permanent HTTP {st} on replay", + file=sys.stderr) + return "drop" + return "ok" + + +def flush() -> int: + """Replay queued writes in order; stop at the first still-unreachable one (keeping it + and the rest). Returns the number actually replayed. Safe to call when empty.""" + items = pending() + if not items: + return 0 + replayed = 0 + remaining: list[dict] = [] + for i, rec in enumerate(items): + result = _replay(rec) + if result == "ok": + replayed += 1 + elif result == "drop": + continue # warned in _replay; remove from queue + else: # retry -> still offline; keep this and everything after, in order + remaining = items[i:] + break + _rewrite(remaining) + return replayed + + +# --------------------------------------------------------------------- cache +def cache_key(path: str) -> Path: + return cache_dir() / (path.replace("/", "__") + ".cache") + + +def cache_put(path: str, body: bytes) -> None: + _ensure_dirs() + cache_key(path).write_bytes(body) + + +def cache_get(path: str) -> bytes | None: + p = cache_key(path) + return p.read_bytes() if p.exists() else None + + +# ----------------------------------------------------------- integration seam +def safe_request(method: str, url: str, data=None, headers=None, idem_key: str | None = None): + """Network-first wrapper around echo.request with offline queueing for writes. + + - success or a client error (4xx): return (status, body) as-is — 4xx is a bug, fail loud. + - transport failure (status 0) or 5xx (after echo.request's own retry) on a MUTATING + verb: enqueue the write and return (202, b"queued (offline)") so the caller reports it + as durably queued rather than failed. + - otherwise (e.g. a failing GET): return (status, body); the read-cache fallback lives + in cmd_load, which knows which reads are worth serving stale. + """ + method = method.upper() + status, body = echo.request(method, url, data=data, headers=headers) + offline = status == 0 or 500 <= status < 600 + if offline and method in MUTATING: + enqueue(method, url, data, headers or {}, idem_key) + return 202, b"queued (offline)" + return status, body diff --git a/echo-memory.plugin.src/skills/echo-memory/scripts/echo_recall.py b/echo-memory.plugin.src/skills/echo-memory/scripts/echo_recall.py new file mode 100644 index 0000000..1d9b2d2 --- /dev/null +++ b/echo-memory.plugin.src/skills/echo-memory/scripts/echo_recall.py @@ -0,0 +1,349 @@ +#!/usr/bin/env python3 +"""echo_recall.py — H1: hybrid lexical (BM25) + graph recall. [v1.0 — wired] + +Replaces the keyword-only recall (search/simple + fixed one-hop) with a ranked, local +retriever that finds a note even when the query is phrased differently than the note +was stored, then expands along the graph with a per-hop decay so the *connected* +context is surfaced in relevance order. + +DESIGN (bounded by the project's pure-stdlib, no-deps constraint): + * Lexical layer : BM25 over note bodies. No embeddings / external API — BM25 is a + strong, dependency-free baseline that handles vocabulary mismatch + far better than substring search. + * Graph layer : from the top lexical hits, walk body wikilinks + source_notes up + to MAX_HOPS, scoring each node `parent * GRAPH_DECAY**hop` (max over + paths) so near, strongly-cued neighbours rank above distant ones. + * Persistence : the BM25 postings + doc stats are a DERIVED artifact, so they live + in the vault's machine index dir (`_agent/index/recall-index.json`), + maintained on `capture` (update_note) and fully rebuilt by `sweep.py` + — exactly how `entities.json` is maintained. Rebuildable => portable. + * Resilience : if the index can't be built (vault unreachable), recall falls back + to the server's /search/simple so it degrades instead of dying. + +Corpus = the indexable ENTITY notes (idx_mod.kind_for_path is not None): people, +companies, concepts, references, meetings, projects, areas, decisions, semantic/ +episodic memory, skills — the durable memory graph. (Sessions/journal are a future +add; tracked in ROADMAP-1.0.md.) +""" +from __future__ import annotations + +import json +import math +import re +import sys +import urllib.parse +from collections import Counter, deque +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import echo # noqa: E402 +import echo_index as idx_mod # noqa: E402 +import echo_links as links # noqa: E402 + +RECALL_INDEX_PATH = "_agent/index/recall-index.json" + +# Tuning knobs. Defaults are standard BM25 (k1/b) + a 0.6 hop decay over <=2 hops. +K1 = 1.5 +B = 0.75 +MAX_HOPS = 2 +GRAPH_DECAY = 0.6 +MIN_SCORE = 0.0 # relevance floor; raise to trim weak lexical hits + +_WORD = re.compile(r"[a-z0-9]+") +# Minimal English stoplist — keeps BM25 idf from being dominated by glue words. +_STOP = frozenset( + "the a an and or of to in is are was were be been for on at by with as it this that " + "from into over under not no yes do does did has have had will would can could".split() +) +_TEMPLATE_RE = re.compile(r"(^|/)(templates/|.*-template\.md$)") +_SKIP_BASENAMES = {"README.md"} + + +def tokenize(text: str) -> list[str]: + """Lowercase word tokens, stopworded. Frontmatter should be stripped by the caller.""" + return [t for t in _WORD.findall(text.lower()) if t not in _STOP and len(t) > 1] + + +def strip_frontmatter(text: str) -> str: + if not text.startswith("---"): + return text + end = text.find("\n---", 3) + return text[end + 4:] if end != -1 else text + + +# --------------------------------------------------------------------- BM25 core +class Bm25Index: + """A compact, serializable BM25 index. add()/remove() are idempotent per path.""" + + def __init__(self) -> None: + self.df: Counter[str] = Counter() # term -> #docs containing it + self.postings: dict[str, dict[str, int]] = {} # term -> {doc_path: tf} + self.length: dict[str, int] = {} # doc_path -> token count + self.n_docs = 0 + self.avg_len = 0.0 + + def _recompute(self) -> None: + self.n_docs = len(self.length) + self.avg_len = (sum(self.length.values()) / self.n_docs) if self.n_docs else 0.0 + + def add(self, path: str, text: str) -> None: + if path in self.length: # re-index in place (idempotent) + self.remove(path) + toks = tokenize(text) + self.length[path] = len(toks) + for term, tf in Counter(toks).items(): + self.postings.setdefault(term, {})[path] = tf + self.df[term] += 1 + self._recompute() + + def remove(self, path: str) -> None: + if path not in self.length: + return + for term in list(self.postings.keys()): + if path in self.postings[term]: + del self.postings[term][path] + self.df[term] -= 1 + if not self.postings[term]: + del self.postings[term] + del self.df[term] + del self.length[path] + self._recompute() + + def score(self, query: str, limit: int = 10) -> list[tuple[str, float]]: + scores: Counter[str] = Counter() + for term in set(tokenize(query)): + posting = self.postings.get(term) + if not posting: + continue + idf = math.log(1 + (self.n_docs - self.df[term] + 0.5) / (self.df[term] + 0.5)) + for path, tf in posting.items(): + dl = self.length.get(path, 0) + denom = tf + K1 * (1 - B + B * (dl / self.avg_len if self.avg_len else 1)) + scores[path] += idf * (tf * (K1 + 1) / denom if denom else 0) + return [(p, s) for p, s in scores.most_common(limit) if s > MIN_SCORE] + + # ---- serialization (compact; rebuildable, so the format can change freely) ---- + def to_json(self) -> dict: + return {"schema": 1, "n_docs": self.n_docs, "avg_len": self.avg_len, + "length": self.length, "postings": self.postings} + + @classmethod + def from_json(cls, d: dict) -> "Bm25Index": + ix = cls() + ix.length = d.get("length", {}) + ix.postings = d.get("postings", {}) + ix.n_docs = d.get("n_docs", len(ix.length)) + ix.avg_len = d.get("avg_len", 0.0) + ix.df = Counter({term: len(posting) for term, posting in ix.postings.items()}) + return ix + + +# ------------------------------------------------------------------- vault I/O +def _get(path: str) -> str | None: + status, body = echo.request("GET", echo.vault_url(path)) + if status == 404: + return None + echo.check(status, body, f"recall get {path}") + return body.decode(errors="replace") + + +def _list_dir(path: str): + p = "" if path in ("", "/") else (path if path.endswith("/") else path + "/") + body = _get(p) + if body is None: + return [], [] + try: + j = json.loads(body) + except json.JSONDecodeError: + return [], [] + entries = list(j.get("files", [])) + list(j.get("folders", [])) + files = [e for e in entries if not e.endswith("/")] + folders = [e[:-1] for e in entries if e.endswith("/")] + return files, folders + + +def _walk(prefix: str = ""): + files, folders = _list_dir(prefix) + for f in files: + yield prefix + f + for d in folders: + yield from _walk(f"{prefix}{d}/") + + +def _indexable(path: str) -> bool: + base = path.rsplit("/", 1)[-1] + return (path.endswith(".md") and idx_mod.kind_for_path(path) is not None + and base not in _SKIP_BASENAMES and not _TEMPLATE_RE.search(path)) + + +# ------------------------------------------------------------------- persistence +def load_index() -> Bm25Index: + status, body = echo.request("GET", echo.vault_url(RECALL_INDEX_PATH)) + if status == 404: + return Bm25Index() + echo.check(status, body, "recall-index load") + try: + return Bm25Index.from_json(json.loads(body)) + except (json.JSONDecodeError, KeyError, TypeError): + return Bm25Index() + + +def save_index(ix: Bm25Index) -> None: + body = json.dumps(ix.to_json(), ensure_ascii=False).encode("utf-8") + status, b = echo.request("PUT", echo.vault_url(RECALL_INDEX_PATH), data=body, + headers={"Content-Type": "application/json"}) + echo.check(status, b, "recall-index save") + + +def rebuild() -> Bm25Index: + """Full rebuild from every indexable entity note. Called by sweep.py and as the + cold-cache path inside recall(). Saves and returns the index.""" + ix = Bm25Index() + for path in _walk(): + if not _indexable(path): + continue + text = _get(path) + if text is not None: + ix.add(path, strip_frontmatter(text)) + save_index(ix) + return ix + + +def update_note(path: str, text: str) -> None: + """Best-effort incremental upkeep for a single note (mirrors how entities.json is + maintained on capture). Never raises into the caller. The load->save is wrapped in the + advisory lock with a fresh re-read (H3), so concurrent captures can't clobber the + recall index either.""" + try: + if not _indexable(path): + return # only entity notes are part of the recall corpus + import echo_concurrency + with echo_concurrency.vault_lock(): + ix = load_index() # fresh read inside the lock + ix.add(path, strip_frontmatter(text)) + save_index(ix) + except Exception as exc: # noqa: BLE001 — upkeep must never fail a capture + print(f"echo_recall: index update skipped ({exc})", file=sys.stderr) + + +# ------------------------------------------------------------------- graph fusion +def _resolve_target(target: str, nmap: dict) -> str | None: + target = target.strip() + if not target: + return None + if "/" in target: + return target if target.endswith(".md") else target + ".md" + return nmap.get(idx_mod.slugify(target)) + + +def expand_graph(seeds, nmap: dict, base_scores: dict, max_hops: int = MAX_HOPS): + """BFS from seeds over body wikilinks + source_notes, scoring each neighbour + `parent_score * GRAPH_DECAY**hop` (kept as the max over reaching paths). Returns a + score-sorted list of (path, (score, via)) excluding the seeds themselves.""" + score_of = dict(base_scores) + seen = set(seeds) + results: dict[str, tuple[float, str]] = {} + dq = deque((s, 0) for s in seeds) + while dq: + path, hop = dq.popleft() + if hop >= max_hops: + continue + text = links.get_text(path) + if text is None: + continue + body = strip_frontmatter(text) + targets = set(links.all_wikilinks(body)) | set(links.source_notes(text)) + parent = score_of.get(path, 1.0) + for t in targets: + tp = _resolve_target(t, nmap) + if not tp or tp in seeds: + continue + decayed = parent * (GRAPH_DECAY ** (hop + 1)) + if decayed > results.get(tp, (0.0, ""))[0]: + results[tp] = (decayed, path) + if tp not in seen: + seen.add(tp) + dq.append((tp, hop + 1)) + return sorted(results.items(), key=lambda kv: -kv[1][0]) + + +# ------------------------------------------------------------------- presentation +def _brief(path: str, score: float | None = None, via: str | None = None) -> None: + text = links.get_text(path) + if text is None: + return + head = f"\n### {path}" + if score is not None: + head += f" (score {score:.2f})" + if via: + head += f" (via {via})" + print(head) + mtype = re.search(r"(?m)^type:\s*(.+)$", text) + if mtype: + print(f"_type: {mtype.group(1).strip()}_") + body = strip_frontmatter(text) + status = re.search(r"(?ms)^##\s+Status\s*$(.*?)(?=^##\s|\Z)", body) + if status and status.group(1).strip(): + print(status.group(1).strip()[:400]) + return + shown = 0 + for line in body.splitlines(): + if not line.strip() or line.startswith("# "): + continue + print(line[:200]) + shown += 1 + if shown >= 6: + break + + +# ------------------------------------------------------------------- entrypoint +def recall(query, limit: int = 8) -> int: + q = " ".join(query) if isinstance(query, list) else query + index = idx_mod.load() + nmap = idx_mod.name_map(index) + + # --- lexical layer (BM25); rebuild the cache on a cold miss -------------- + lexical: list[tuple[str, float]] = [] + try: + ix = load_index() + if ix.n_docs == 0: + ix = rebuild() + lexical = ix.score(q, limit=limit) + except echo.EchoError as exc: + print(f"recall: BM25 index unavailable ({exc}); falling back to server search", + file=sys.stderr) + + base: dict[str, float] = {p: s for p, s in lexical} + hits: list[str] = [p for p, _ in lexical] + + # --- entity-index seed (alias-aware exact match) ------------------------- + _, e = idx_mod.resolve(index, q) + if e and e.get("path") and e["path"] not in base: + hits.insert(0, e["path"]) + base[e["path"]] = max(base.values(), default=1.0) + + # --- resilience fallback: server search when lexical found nothing ------- + if not hits: + try: + status, body = echo.request( + "POST", f"{echo.BASE}/search/simple/?query={urllib.parse.quote_plus(q)}") + if status == 200: + for r in (json.loads(body) or [])[:limit]: + fn = r.get("filename") or r.get("path") + if fn and fn not in base: + hits.append(fn) + base[fn] = 0.0 + except Exception: # noqa: BLE001 — fallback only; stay quiet + pass + + # --- graph layer ---------------------------------------------------------- + neighbours = expand_graph(hits, nmap, base, max_hops=MAX_HOPS) + + print(f"== recall: {q} ==") + print(f"\n# Primary hits ({len(hits)})") + for p in hits: + _brief(p, score=base.get(p)) + print(f"\n# Linked context ({len(neighbours)})") + for p, (sc, via) in neighbours[: 2 * limit]: + _brief(p, score=sc, via=via) + return 0 diff --git a/echo-memory.plugin.src/skills/echo-memory/scripts/echo_reflect.py b/echo-memory.plugin.src/skills/echo-memory/scripts/echo_reflect.py new file mode 100644 index 0000000..4062bce --- /dev/null +++ b/echo-memory.plugin.src/skills/echo-memory/scripts/echo_reflect.py @@ -0,0 +1,136 @@ +#!/usr/bin/env python3 +"""echo_reflect.py — H5: automatic session-reflection capture. [v1.0 — wired] + +Today memory only accrues when the agent decides to write. This makes accrual the +default: at session end the model extracts durable items from the conversation and +emits a PROPOSAL set; this module dedups them against the entity index, previews them, +and applies the accepted ones via the normal `capture` path — so nothing is written +without a go-ahead (honors the "show before large/sensitive writes" safety rule). + +DIVISION OF LABOR: + * Extraction is MODEL-side (only the LLM has the conversation). The agent produces a + JSON array of proposals matching PROPOSAL_SCHEMA. + * This script is the deterministic spine: validate -> classify (new/update/inbox vs the + entity index) -> preview -> apply on confirm. No NL understanding lives here, so it's + testable offline. + +PROPOSAL_SCHEMA (one per durable item the agent wants to remember): + { + "title": "Bob Smith", # required + "kind": "person", # required unless "inbox": true + "body": "Principal at MPM; ...", # markdown body (optional) + "aliases": ["bob", "rs"], # optional + "sources": ["_agent/sessions/..."], # optional backward links + "date": "YYYY-MM-DD", # optional (meeting/decision kinds) + "domain": "business", # optional (area kind) + "inbox": false, # route to inbox when the home is unknown + "confidence": 0.0-1.0 # agent's own confidence; gates auto-suggest + } + +CLI (echo.py reflect): dry-run previews; --apply writes — matching bootstrap/migrate/sweep. +""" +from __future__ import annotations + +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import echo # noqa: E402 +import echo_index as idx_mod # noqa: E402 + +MIN_CONFIDENCE = 0.6 + + +def validate(proposals: list[dict]) -> tuple[list[dict], list[str]]: + """Return (valid, errors). title always required; kind required unless inbox; a + confidence below the floor is routed away (the agent should send it to the inbox).""" + valid, errors = [], [] + for i, p in enumerate(proposals or []): + if not str(p.get("title", "")).strip(): + errors.append(f"proposal[{i}]: missing title") + continue + is_inbox = bool(p.get("inbox")) + if not is_inbox: + kind = str(p.get("kind", "")).strip() + if not kind: + errors.append(f"proposal[{i}] '{p['title']}': missing kind (or set inbox:true)") + continue + if kind not in idx_mod.KIND_FOLDER: + errors.append(f"proposal[{i}] '{p['title']}': unknown kind '{kind}'") + continue + if float(p.get("confidence", 1.0)) < MIN_CONFIDENCE: + errors.append(f"proposal[{i}] '{p['title']}': confidence < {MIN_CONFIDENCE} — send to inbox instead") + continue + valid.append(p) + return valid, errors + + +def classify(proposals: list[dict]) -> list[dict]: + """Annotate each proposal with `_action` (create / update / inbox) and `_path`, by + resolving its title against the entity index (alias-aware), so the preview shows + create-vs-append and dedups against what's already in memory.""" + index = idx_mod.load() + for p in proposals: + if p.get("inbox") or not p.get("kind"): + p["_action"], p["_path"] = "inbox", "inbox/captures/inbox.md" + continue + _, e = idx_mod.resolve(index, p["title"]) + if e and e.get("path"): + p["_action"], p["_path"] = "update", e["path"] + else: + try: + p["_path"] = idx_mod.derive_path( + p["kind"], idx_mod.slugify(p["title"]), + date=p.get("date"), domain=p.get("domain", "business")) + p["_action"] = "create" + except echo.EchoError as exc: + p["_action"], p["_path"] = "error", str(exc) + return proposals + + +def preview(proposals: list[dict]) -> str: + """One confirmable line per proposal: action | kind | title -> target path.""" + rows = [] + for p in proposals: + act = p.get("_action", "?") + kind = "-" if act == "inbox" else p.get("kind", "?") + rows.append(f" {act:7} | {kind:9} | {p['title']} -> {p.get('_path', '?')}") + return "\n".join(rows) + + +def apply(proposals: list[dict], confirm: bool = False) -> int: + """Validate -> classify -> preview; with confirm=True, apply each via echo_ops.capture + (which routes/indexes/links/logs, each self-lock-guarded). Without confirm it is a + dry-run that writes nothing — the preview IS the confirmation step.""" + valid, errors = validate(proposals) + for e in errors: + print(f"skip: {e}", file=sys.stderr) + if not valid: + print("reflect: no valid proposals to apply.") + return 0 + + classify(valid) + counts = {a: sum(1 for p in valid if p.get("_action") == a) for a in ("create", "update", "inbox", "error")} + print(f"reflect: {len(valid)} proposal(s) — " + f"{counts['create']} new, {counts['update']} update, {counts['inbox']} inbox" + + (f", {counts['error']} error" if counts["error"] else "")) + print(preview(valid)) + + if not confirm: + print("\nreflect: dry-run — re-run with --apply to write these to memory.") + return 0 + + import echo_ops # lazy: the apply path pulls in the capture/index/link stack + applied = 0 + for p in valid: + if p.get("_action") == "error": + continue + bodyfile = echo.temp_file((p.get("body") or "").encode()) if p.get("body") else None + rc = echo_ops.capture( + p.get("kind"), p["title"], bodyfile, + aliases=p.get("aliases") or [], sources=p.get("sources") or [], + date=p.get("date"), domain=p.get("domain", "business"), + inbox=bool(p.get("inbox")) or not p.get("kind")) + applied += 1 if rc == 0 else 0 + print(f"reflect: applied {applied}/{len(valid)} proposal(s).") + return 0 diff --git a/echo-memory.plugin.src/skills/echo-memory/scripts/echo_secrets.py b/echo-memory.plugin.src/skills/echo-memory/scripts/echo_secrets.py new file mode 100644 index 0000000..3eefa6a --- /dev/null +++ b/echo-memory.plugin.src/skills/echo-memory/scripts/echo_secrets.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python3 +"""echo_secrets.py — M1: resolve the API bearer token without shipping it in source. +[v1.0 SCAFFOLD — resolve_key() implemented; wiring into echo.py is the integration step] + +Problem: the token is hardcoded at echo.py:70 and repeated through api-reference.md / +README — committed to git and shipped inside every .plugin zip, rotatable only by a +rebuild. The plugin's own safety rule is "never store secrets". This resolves the key +from progressively less-trusted sources, with a loud warning if the legacy baked +default is ever reached. + +RESOLUTION ORDER (first hit wins): + 1. env ECHO_KEY — preferred; CI / per-session override. + 2. ECHO_STATE_DIR/credentials — local file, one `key=...` line, chmod 0600. + 3. legacy baked default (deprecated) — warns to stderr; remove before 1.0 ship. + +Pure stdlib only (no `keyring`); the local file is the cross-platform stand-in for an +OS keychain and keeps the secret OUT of the tracked tree. + +INTEGRATION (do at merge time): + echo.py: KEY = echo_secrets.resolve_key() + docs: replace the literal token with `` placeholders + rotate: document `echo write-key` (writes the credentials file) — see TODO below +""" +from __future__ import annotations + +import os +import sys +from pathlib import Path + +# Kept ONLY so a fresh install still works during the migration window; delete for 1.0. +_LEGACY_DEFAULT = "" # intentionally blank in the scaffold — do not re-introduce the literal + + +def _state_dir() -> Path: + return Path(os.environ.get("ECHO_STATE_DIR") or (Path.home() / ".echo-memory")) + + +def _from_file() -> str | None: + p = _state_dir() / "credentials" + if not p.exists(): + return None + for line in p.read_text(encoding="utf-8").splitlines(): + line = line.strip() + if line.startswith("key=") or line.startswith("ECHO_KEY="): + return line.split("=", 1)[1].strip() + return None + + +def resolve_key(legacy: str | None = None) -> str: + """Return the bearer token from the most trusted available source: + ECHO_KEY env -> credentials file -> `legacy` baked-in fallback (with a loud, once-per- + process warning that can be silenced via ECHO_KEY_LEGACY_OK=1). Raises if none exist.""" + env = os.environ.get("ECHO_KEY") + if env: + return env + from_file = _from_file() + if from_file: + return from_file + legacy = legacy or _LEGACY_DEFAULT + if legacy: + if os.environ.get("ECHO_KEY_LEGACY_OK") != "1": + print("echo_secrets: WARNING — using the deprecated baked-in API key. Run " + "`echo.py write-key ` (stores ~/.echo-memory/credentials) or set " + "ECHO_KEY; set ECHO_KEY_LEGACY_OK=1 to silence.", file=sys.stderr) + return legacy + raise RuntimeError( + "echo_secrets: no API key found. Set ECHO_KEY or create " + f"{_state_dir() / 'credentials'} with a 'key=' line." + ) + + +def write_key(token: str) -> Path: + """Persist the token to the local credentials file (one `key=...` line, chmod 0600). + Best-effort perms — Windows has no POSIX mode bits, so the chmod is advisory there.""" + d = _state_dir() + d.mkdir(parents=True, exist_ok=True) + path = d / "credentials" + path.write_text(f"key={token.strip()}\n", encoding="utf-8") + try: + os.chmod(path, 0o600) + except OSError: + pass # platform without POSIX perms (e.g. some Windows filesystems) + return path diff --git a/echo-memory.plugin.src/skills/echo-memory/scripts/migrate.py b/echo-memory.plugin.src/skills/echo-memory/scripts/migrate.py index 2329ba5..5dc0f60 100644 --- a/echo-memory.plugin.src/skills/echo-memory/scripts/migrate.py +++ b/echo-memory.plugin.src/skills/echo-memory/scripts/migrate.py @@ -25,7 +25,7 @@ sys.path.insert(0, str(SCRIPT_DIR)) import echo # noqa: E402 -CURRENT_SCHEMA = 3 +CURRENT_SCHEMA = 4 def get_text(path: str) -> str | None: @@ -141,6 +141,11 @@ def main(argv: list[str] | None = None) -> int: "# index\n\nMachine-maintained entity index. See the echo-memory plugin.\n")) print("migrate: [2->3] then run `sweep.py --apply` to build entities.json and symmetrize links.") + if start < 4: + print("migrate: [3->4] add the recall (BM25) index — hybrid lexical+graph recall") + print("migrate: [3->4] _agent/index/ already exists (schema 3); no moves needed.") + print("migrate: [3->4] then run `sweep.py --apply` to build _agent/index/recall-index.json.") + do_or_show(args.apply, f"set _agent/echo-vault.md schema_version -> {CURRENT_SCHEMA}", lambda: echo.cmd_fm("_agent/echo-vault.md", "schema_version", str(CURRENT_SCHEMA))) if args.apply: diff --git a/echo-memory.plugin.src/skills/echo-memory/scripts/sweep.py b/echo-memory.plugin.src/skills/echo-memory/scripts/sweep.py index 616297c..82baeec 100644 --- a/echo-memory.plugin.src/skills/echo-memory/scripts/sweep.py +++ b/echo-memory.plugin.src/skills/echo-memory/scripts/sweep.py @@ -33,8 +33,9 @@ sys.path.insert(0, str(SCRIPT_DIR)) import echo # noqa: E402 import echo_index as idx_mod # noqa: E402 import echo_links as links # noqa: E402 +import echo_recall # noqa: E402 -CURRENT_SCHEMA = 3 +CURRENT_SCHEMA = 4 TEMPLATE_RE = re.compile(r"(^|/)(templates/|.*-template\.md$)") SKIP_BASENAMES = {"README.md"} @@ -123,6 +124,14 @@ def main(argv: list[str] | None = None) -> int: if apply: idx_mod.save(rebuilt) + # ---- 2b. (re)build the recall (BM25) index ------------------------------- + if apply: + rix = echo_recall.rebuild() + print(f"sweep: {tag} recall index — {rix.n_docs} docs (BM25)") + else: + n_idx = sum(1 for p in all_files if echo_recall._indexable(p)) + print(f"sweep: {tag} recall index — would rebuild BM25 over {n_idx} entity notes") + # ---- 3. symmetrize existing Related links -------------------------------- fileset = set(all_files) basemap: dict[str, str] = {} diff --git a/echo-memory.plugin.src/skills/echo-memory/scripts/test_v1_scaffold.py b/echo-memory.plugin.src/skills/echo-memory/scripts/test_v1_scaffold.py new file mode 100644 index 0000000..acd393e --- /dev/null +++ b/echo-memory.plugin.src/skills/echo-memory/scripts/test_v1_scaffold.py @@ -0,0 +1,112 @@ +#!/usr/bin/env python3 +"""test_v1_scaffold.py — H4 interface guard for the v1.0 scaffolds. [offline, no creds] + +Asserts the scaffold contract holds so the skeleton can't silently rot while 1.0 is +built: every planned module imports, the parts that ARE implemented work (BM25 ranking, +queue/cache round-trip, slug helpers, secret resolution order), and the parts that are +TODO raise NotImplementedError (so a stub can't masquerade as done). + +Run: python test_v1_scaffold.py +""" +from __future__ import annotations + +import os +import sys +import tempfile +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +failures: list[str] = [] + + +def check(name: str, cond: bool, detail: str = "") -> None: + print(f"{'ok ' if cond else 'FAIL'} {name}" + (f" -- {detail}" if not cond else "")) + if not cond: + failures.append(name) + + +def main() -> int: + # H1 — BM25 core is implemented; assert it actually ranks a relevant doc first. + import echo_recall as r + ix = r.Bm25Index() + ix.add("notes/mqtt.md", "the mqtt broker config and tls certificate rotation") + ix.add("notes/payroll.md", "quarterly payroll run and employee deductions") + top = ix.score("certificate rotation broker", limit=2) + check("H1 BM25 ranks relevant doc first", bool(top) and top[0][0] == "notes/mqtt.md", str(top)) + check("H1 BM25 survives json round-trip", + r.Bm25Index.from_json(ix.to_json()).score("payroll")[0][0] == "notes/payroll.md") + ix.remove("notes/payroll.md") + check("H1 BM25 remove() drops a doc's postings", ix.score("payroll") == []) + check("H1 recall path is wired (not a stub)", + all(callable(getattr(r, n, None)) + for n in ("recall", "rebuild", "update_note", "load_index", "save_index"))) + + # H2 — cache round-trips, enqueue dedups, flush is a no-op on an empty queue (offline-safe). + import echo_queue as q + with tempfile.TemporaryDirectory() as d: + os.environ["ECHO_STATE_DIR"] = d + q.cache_put("a/b.md", b"hello") + check("H2 cache round-trip", q.cache_get("a/b.md") == b"hello") + q.enqueue("PUT", "http://x/vault/a.md", b"body", {}, idem_key="k1") + check("H2 enqueue persists a record", len(q.pending()) == 1) + q.enqueue("PUT", "http://x/vault/a.md", b"body", {}, idem_key="k1") + check("H2 enqueue dedups by idem_key", len(q.pending()) == 1) + with tempfile.TemporaryDirectory() as d2: + os.environ["ECHO_STATE_DIR"] = d2 + check("H2 flush() returns 0 on an empty queue (no network)", q.flush() == 0) + + # H3 — lock CM + atomic index update are stubs. + import echo_concurrency as c + check("H3 auto_owner is stable", c.auto_owner() == c.auto_owner()) + check("H3 vault_lock returns a context manager (no network until entered)", + hasattr(c.vault_lock(), "__enter__") and hasattr(c.vault_lock(), "__exit__")) + check("H3 atomic_index_update is wired", callable(c.atomic_index_update)) + + # H5 — proposal validation is pure (offline): keeps valid (incl. inbox), drops the rest. + import echo_reflect as rf + v, errs = rf.validate([ + {"title": "Ok", "kind": "person", "confidence": 0.9}, + {"title": "", "kind": "person"}, # missing title + {"title": "Bad", "kind": "bogus"}, # unknown kind + {"title": "Weak", "kind": "person", "confidence": 0.2}, # below confidence floor + {"title": "Note", "inbox": True}, # inbox needs no kind + ]) + check("H5 validate keeps valid (incl. inbox), drops invalid", len(v) == 2 and len(errs) == 3, (len(v), errs)) + check("H5 classify/preview/apply wired", + all(callable(getattr(rf, n, None)) for n in ("classify", "preview", "apply"))) + + # M1 — secret resolution order is implemented; env wins, then the credentials file. + import echo_secrets as s + os.environ["ECHO_KEY"] = "env-key-123" + check("M1 env ECHO_KEY wins", s.resolve_key("legacy-fallback") == "env-key-123") + with tempfile.TemporaryDirectory() as d: + os.environ["ECHO_STATE_DIR"] = d + os.environ.pop("ECHO_KEY", None) + s.write_key("file-key-xyz") + check("M1 write_key -> credentials -> resolve_key reads it", s.resolve_key() == "file-key-xyz") + os.environ["ECHO_KEY_LEGACY_OK"] = "1" # silence the warning for the legacy-path check + os.environ["ECHO_STATE_DIR"] = os.path.join(d, "empty") # an empty dir: no creds file + check("M1 legacy fallback when env+file absent", s.resolve_key("legacy-fallback") == "legacy-fallback") + os.environ["ECHO_KEY"] = "env-key-123" # restore for any later checks + + # M2 — slug collision + link-confidence helpers are implemented. + import echo_quality as qual + check("M2 safe_slug disambiguates a collision", + qual.safe_slug({"foo"}, "foo") != "foo") + check("M2 short/common tokens are not confident links", + qual.is_confident_link("API", "we built an api") is False) + + # M3 — doctor module exposes run(); M4 — output envelope is implemented. + import echo_doctor as doc + check("M3 doctor exposes run()", hasattr(doc, "run")) + import echo_output as out + env = out.envelope("created", {"path": "p.md"}) + check("M4 envelope shape", env.get("ok") is True and env.get("action") == "created") + + print(f"\n{len(failures)} failure(s)" if failures else "\nall scaffold interface checks passed") + return 1 if failures else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/eval/mock_olrapi_hifi.py b/eval/mock_olrapi_hifi.py new file mode 100644 index 0000000..37b976f --- /dev/null +++ b/eval/mock_olrapi_hifi.py @@ -0,0 +1,135 @@ +#!/usr/bin/env python3 +"""mock_olrapi_hifi.py — H4: higher-fidelity mock of the Obsidian Local REST API. + +The shipped mock's PATCH is a "naive append": it appends the body to EOF instead of +inserting/replacing under the target heading, so the 0.9 surface (capture's +_append_to_existing, scope set's replace, the daily Agent-Log insert) is only ever +validated against behavior the real API does NOT exhibit. This subclass models the +real PATCH semantics so the eval can catch heading-insert / replace regressions. + +Over the base mock (eval/mock_olrapi.py) this overrides do_PATCH only: + * heading append -> insert at the END of the target section's body (before the next + sibling/parent heading), NOT at EOF. + * heading prepend -> insert at the START of the section body (right after the heading). + * heading replace -> replace the section body, keep the heading line. + * heading missing -> 400 errorCode 40080 (the silent-write-loss trigger). [unchanged] + * frontmatter replace -> parse the YAML block and set/replace the scalar; a replace on a + MISSING field returns 400 40080 (matches the real API, per SKILL.md). + +Everything else (GET/PUT/POST/DELETE, directory listing, document-map, flaky/phantom +fault injection, debug reset) is inherited unchanged from the base handler. +""" +from __future__ import annotations + +import argparse +import json +import re +from http.server import ThreadingHTTPServer + +import mock_olrapi +from mock_olrapi import H as BaseH, STATE + + +def _heading(line: str): + m = re.match(r"^(#{1,6})\s+(.*)$", line) + return (len(m.group(1)), m.group(2).strip()) if m else (0, None) + + +def section_span(lines, target): + """(start, end, level) for the section under `target` (an "A::B" path; matched on its + leaf). Body lines are lines[start+1:end]. None if the heading is absent.""" + leaf = target.split("::")[-1].strip() + start = level = None + for i, line in enumerate(lines): + lv, txt = _heading(line) + if lv and txt == leaf: + start, level = i, lv + break + if start is None: + return None + end = len(lines) + for j in range(start + 1, len(lines)): + lv, _ = _heading(lines[j]) + if lv and lv <= level: + end = j + break + return start, end, level + + +def set_frontmatter(text: str, field: str, raw_value: str): + """Return (new_text, existed). raw_value is the JSON the client PATCHed.""" + try: + value = json.loads(raw_value) + except Exception: + value = raw_value.strip().strip('"') + rendered = value if isinstance(value, str) else json.dumps(value) + new_line = f"{field}: {rendered}" + if not text.startswith("---"): + return f"---\n{new_line}\n---\n\n{text}", False + end = text.find("\n---", 3) + head = text[4:end] if end != -1 else text + rest = text[end:] if end != -1 else "\n---\n" + out, existed = [], False + for line in head.splitlines(): + if re.match(rf"^{re.escape(field)}\s*:", line): + out.append(new_line) + existed = True + else: + out.append(line) + if not existed: + out.append(new_line) + return "---\n" + "\n".join(out) + rest, existed + + +class H(BaseH): + def do_PATCH(self): + vp = self._vpath() + body = self._read_body() + if vp is None: + return self._json(400, {"message": "bad path"}) + ttype = self.headers.get("Target-Type", "") + op = self.headers.get("Operation", "append") + target = self.headers.get("Target", "") + cur = STATE.get(vp, "") + + if ttype == "heading": + lines = cur.split("\n") + span = section_span(lines, target) + if span is None: + return self._json(400, {"errorCode": 40080, "message": "invalid-target"}) + start, end, _ = span + payload = body.strip("\n").split("\n") if body.strip("\n") else [""] + if op == "replace": + lines[start + 1:end] = payload + elif op == "prepend": + lines[start + 1:start + 1] = payload + else: # append at the tail of the section body (before the next heading) + ins = end + while ins > start + 1 and lines[ins - 1].strip() == "": + ins -= 1 + lines[ins:ins] = payload + STATE[vp] = "\n".join(lines) + return self._send(200, "") + + if ttype == "frontmatter": + new, existed = set_frontmatter(cur, target, body) + if op == "replace" and not existed: + return self._json(400, {"errorCode": 40080, "message": "invalid-target"}) + STATE[vp] = new + return self._send(200, "") + + STATE[vp] = cur + "\n" + body + return self._send(200, "") + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--port", type=int, default=8800) + a = ap.parse_args() + srv = ThreadingHTTPServer(("127.0.0.1", a.port), H) + print(f"mock-olrapi-hifi listening on http://127.0.0.1:{a.port}", flush=True) + srv.serve_forever() + + +if __name__ == "__main__": + main() diff --git a/eval/run_eval.py b/eval/run_eval.py index d323d89..9c5f342 100644 --- a/eval/run_eval.py +++ b/eval/run_eval.py @@ -33,7 +33,7 @@ from pathlib import Path HERE = Path(__file__).resolve().parent ECHO = HERE.parent / "echo-memory.plugin.src" / "skills" / "echo-memory" / "scripts" / "echo.py" -KEY = "241265fbe6830934a9a4ad3e69335f64a42153b663aa5b0017cb1ea1217b2bab" +KEY = "test-key-not-a-real-secret" # ----- tiny HTTP helpers (used for setup, ground truth, and the 0.6 model) ----- def http(method, url, body=None, headers=None): diff --git a/eval/test_features.py b/eval/test_features.py index 67dd9b7..8731b99 100644 --- a/eval/test_features.py +++ b/eval/test_features.py @@ -14,7 +14,7 @@ HERE = Path(__file__).resolve().parent SCRIPTS = HERE.parent / "echo-memory.plugin.src" / "skills" / "echo-memory" / "scripts" ECHO = SCRIPTS / "echo.py" SWEEP = SCRIPTS / "sweep.py" -KEY = "241265fbe6830934a9a4ad3e69335f64a42153b663aa5b0017cb1ea1217b2bab" +KEY = "test-key-not-a-real-secret" failures = [] @@ -102,6 +102,12 @@ def main(): r = h.echo(ECHO, "recall", "MPM") check("recall surfaces linked person", "resources/people/bob-smith.md" in r.stdout, r.stdout) + # 4b. BM25 finds a note by a BODY term. /search/simple is mocked empty, so a hit + # here can only come from the local BM25 recall index (proves H1 lexical layer). + r = h.echo(ECHO, "recall", "principal") + check("recall finds note by body term (BM25)", + "resources/companies/mpm.md" in r.stdout, r.stdout) + # 5. explicit link between two fresh notes h.echo(ECHO, "capture", "Alpha", "--kind", "concept") h.echo(ECHO, "capture", "Beta", "--kind", "concept") @@ -120,10 +126,49 @@ def main(): r = h.echo(SWEEP, "--apply") marker = h.ground("_agent/echo-vault.md") check("sweep runs clean", r.returncode == 0, r.stdout + r.stderr) - check("sweep stamps schema 3", marker and "schema_version=3" in marker.replace(": ", "="), marker) + check("sweep stamps schema 4", marker and "schema_version=4" in marker.replace(": ", "="), marker) idx2 = h.ground("_agent/index/entities.json") check("sweep rebuilt index has all entities", idx2 and all(s in idx2 for s in ["bob-smith", "mpm", "alpha", "beta"])) + rix = h.ground("_agent/index/recall-index.json") + check("sweep builds the recall (BM25) index", + rix is not None and '"postings"' in rix and "principal" in rix, rix) + + # 8. H3 — atomic_index_update re-reads fresh and MERGES, so an entry written by a + # "concurrent" session survives our write; and the advisory lock is released after. + h.http("PUT", f"{base}/vault/_agent/index/entities.json", + '{"schema":1,"updated":"2026-06-21","entities":{"ext-entity":' + '{"path":"resources/concepts/ext-entity.md","kind":"concept","title":"Ext",' + '"aliases":[],"last_seen":"2026-06-21"}}}') + env = dict(os.environ, ECHO_BASE=base, ECHO_KEY=KEY, ECHO_TODAY="2026-06-21", PYTHONPATH=str(SCRIPTS)) + snippet = ("import echo_index as ix, echo_concurrency as c; " + "c.atomic_index_update(lambda d: ix.upsert(d,'mine'," + "'resources/concepts/mine.md','concept','Mine'))") + subprocess.run([sys.executable, "-c", snippet], env=env, capture_output=True, text=True) + idx3 = h.ground("_agent/index/entities.json") + check("H3 atomic update merges a concurrent entry (no clobber)", + bool(idx3) and "ext-entity" in idx3 and "mine" in idx3, idx3) + check("H3 advisory lock released after the transaction", + h.ground("_agent/locks/vault.lock") is None) + + # 9. M4 — --dry-run previews and writes nothing; --json emits a parseable envelope. + r = h.echo(ECHO, "capture", "DryRun Co", "--kind", "company", "--dry-run", "--json") + try: + env = json.loads(r.stdout) + except Exception: + env = {} + check("M4 --dry-run emits a dry-run JSON envelope", + env.get("action", "").startswith("dry-run") and env.get("path") == "resources/companies/dryrun-co.md", r.stdout) + check("M4 --dry-run writes nothing", h.ground("resources/companies/dryrun-co.md") is None) + r = h.echo(ECHO, "capture", "Json Co", "--kind", "company", "--json") + try: + env = json.loads(r.stdout) + except Exception: + env = {} + check("M4 --json envelope reports created + path", + env.get("ok") is True and env.get("action") == "created" + and env.get("path") == "resources/companies/json-co.md", r.stdout) + check("M4 --json capture actually wrote the note", h.ground("resources/companies/json-co.md") is not None) print(f"\n{len(failures)} failure(s)" if failures else "\nall feature tests passed") return 1 if failures else 0 diff --git a/eval/test_offline_queue.py b/eval/test_offline_queue.py new file mode 100644 index 0000000..f9a864f --- /dev/null +++ b/eval/test_offline_queue.py @@ -0,0 +1,126 @@ +#!/usr/bin/env python3 +"""test_offline_queue.py — H2: writes survive an outage; cold start degrades to cache. + +Phase 1 — vault DOWN: `put` / `append` are queued (exit 0, "queued"), not lost. +Phase 2 — vault UP: `flush` replays them idempotently; the writes land. +Phase 3 — vault UP: `load` caches the orientation reads. +Phase 4 — vault DOWN: `load` serves the last-known-good cache and says OFFLINE. + +Drives the real echo.py against eval/mock_olrapi.py. No creds, no live vault. +Run: python test_offline_queue.py [--port 8840] +""" +from __future__ import annotations + +import argparse +import os +import subprocess +import sys +import tempfile +import time +import urllib.request +from pathlib import Path + +HERE = Path(__file__).resolve().parent +ECHO = HERE.parent / "echo-memory.plugin.src" / "skills" / "echo-memory" / "scripts" / "echo.py" +KEY = "test-key-not-a-real-secret" +DEAD = "http://127.0.0.1:59998" # nothing listens here -> connection refused + +failures = [] + + +def check(name, cond, detail=""): + print(f"{'ok ' if cond else 'FAIL'} {name}" + (f" -- {detail}" if not cond else "")) + if not cond: + failures.append(name) + + +def http(method, url, body=None): + data = body.encode() if isinstance(body, str) else body + req = urllib.request.Request(url, data=data, method=method, + headers={"Authorization": f"Bearer {KEY}"}) + try: + with urllib.request.urlopen(req, timeout=10) as r: + return r.status, r.read().decode("utf-8", "replace") + except Exception as e: # noqa: BLE001 + return getattr(e, "code", 0), "" + + +def tmp(content): + f = tempfile.NamedTemporaryFile("w", suffix=".md", delete=False, encoding="utf-8") + f.write(content); f.close() + return f.name + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--port", type=int, default=8840) + a = ap.parse_args() + mock_base = f"http://127.0.0.1:{a.port}" + state = tempfile.mkdtemp() + + def echo(base, *args): + env = dict(os.environ, ECHO_BASE=base, ECHO_KEY=KEY, ECHO_STATE_DIR=state, + ECHO_VERIFY="0", ECHO_TODAY="2026-06-22") + return subprocess.run([sys.executable, str(ECHO), *args], capture_output=True, text=True, env=env) + + def start_mock(): + p = subprocess.Popen([sys.executable, str(HERE / "mock_olrapi.py"), "--port", str(a.port)], + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) + for _ in range(50): + try: + urllib.request.urlopen(f"{mock_base}/__debug__reset", data=b"", timeout=1); break + except Exception: + time.sleep(0.1) + return p + + def ground(path): + _, body = http("GET", f"{mock_base}/__debug__?path={path}") + return None if body == "<>" else body + + srv = None + try: + # --- Phase 1: vault DOWN -> writes are queued, not lost ------------------ + r = echo(DEAD, "put", "_agent/sessions/2026-06-22-1200-x.md", tmp("EVALMARK-session\n")) + check("offline put exits 0 (queued)", r.returncode == 0, r.stderr) + check("offline put reports queued", "queued (offline)" in r.stdout, r.stdout) + r = echo(DEAD, "append", "inbox/captures/inbox.md", "- 2026-06-22: EVALMARK-inbox") + check("offline append reports queued", "queued (offline)" in r.stdout, r.stdout) + outbox = Path(state) / "outbox.ndjson" + check("two writes are persisted to the outbox", + outbox.exists() and len(outbox.read_text(encoding="utf-8").splitlines()) == 2) + + # --- Phase 2: vault UP -> flush replays them ----------------------------- + srv = start_mock() + r = echo(mock_base, "flush") + check("flush replays the queue", "flushed 2" in r.stdout, r.stdout + r.stderr) + check("queued PUT landed in the vault", (ground("_agent/sessions/2026-06-22-1200-x.md") or "").find("EVALMARK-session") >= 0) + check("queued APPEND landed in the vault", "EVALMARK-inbox" in (ground("inbox/captures/inbox.md") or "")) + check("outbox is empty after a clean flush", not outbox.exists() or not outbox.read_text(encoding="utf-8").strip()) + + # idempotent replay: a second flush of the same content must not duplicate. + echo(DEAD, "append", "inbox/captures/inbox.md", "- 2026-06-22: EVALMARK-inbox") # re-queue same line + echo(mock_base, "flush") + inbox = ground("inbox/captures/inbox.md") or "" + check("replay is idempotent (no duplicate line)", inbox.count("EVALMARK-inbox") == 1, inbox) + + # --- Phase 3: vault UP -> load caches the orientation reads --------------- + http("PUT", f"{mock_base}/vault/_agent/echo-vault.md", "schema_version: 4\nEVALMARK-marker\n") + r = echo(mock_base, "load") + check("online load shows marker", "EVALMARK-marker" in r.stdout, r.stdout) + check("load wrote a cache dir", (Path(state) / "cache").exists()) + + # --- Phase 4: vault DOWN -> load serves the cache ------------------------ + srv.terminate(); srv.wait(timeout=5); srv = None + r = echo(DEAD, "load") + check("offline load flags OFFLINE", "OFFLINE" in r.stdout, r.stdout) + check("offline load serves cached marker", "EVALMARK-marker" in r.stdout, r.stdout) + + print(f"\n{len(failures)} failure(s)" if failures else "\nall offline-queue tests passed") + return 1 if failures else 0 + finally: + if srv: + srv.terminate() + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/eval/test_patch_semantics.py b/eval/test_patch_semantics.py new file mode 100644 index 0000000..e9d7489 --- /dev/null +++ b/eval/test_patch_semantics.py @@ -0,0 +1,121 @@ +#!/usr/bin/env python3 +"""test_patch_semantics.py — H4: prove the hi-fi mock models REAL PATCH semantics. + +These are exactly the behaviors the shipped naive mock cannot reproduce (it appends +everything at EOF), so without this the section-replace / section-append / missing- +heading / frontmatter-replace paths are untested. Drives the real echo.py against +mock_olrapi_hifi.py. + +Run: python test_patch_semantics.py [--port 8820] +""" +from __future__ import annotations + +import argparse +import os +import subprocess +import sys +import tempfile +import time +import urllib.request +from pathlib import Path + +HERE = Path(__file__).resolve().parent +ECHO = HERE.parent / "echo-memory.plugin.src" / "skills" / "echo-memory" / "scripts" / "echo.py" +KEY = "test-key-not-a-real-secret" + +failures = [] + + +def check(name, cond, detail=""): + print(f"{'ok ' if cond else 'FAIL'} {name}" + (f" -- {detail}" if not cond else "")) + if not cond: + failures.append(name) + + +def http(method, url, body=None, headers=None): + data = body.encode() if isinstance(body, str) else body + req = urllib.request.Request(url, data=data, method=method, + headers={"Authorization": f"Bearer {KEY}", **(headers or {})}) + try: + with urllib.request.urlopen(req, timeout=10) as r: + return r.status, r.read().decode("utf-8", "replace") + except Exception as e: # noqa: BLE001 + return getattr(e, "code", 0), "" + + +def tmp(content): + f = tempfile.NamedTemporaryFile("w", suffix=".md", delete=False, encoding="utf-8") + f.write(content) + f.close() + return f.name + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--port", type=int, default=8820) + a = ap.parse_args() + base = f"http://127.0.0.1:{a.port}" + srv = subprocess.Popen([sys.executable, str(HERE / "mock_olrapi_hifi.py"), "--port", str(a.port)], + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) + + def echo(*args): + env = dict(os.environ, ECHO_BASE=base, ECHO_KEY=KEY, ECHO_VERIFY="0") + return subprocess.run([sys.executable, str(ECHO), *args], capture_output=True, text=True, env=env) + + def ground(path): + _, body = http("GET", f"{base}/__debug__?path={path}") + return None if body == "<>" else body + + try: + for _ in range(50): + try: + urllib.request.urlopen(f"{base}/__debug__reset", data=b"", timeout=1) + break + except Exception: + time.sleep(0.1) + + doc = "doc.md" + seed = ("---\ntype: note\nupdated: 2026-01-01\n---\n\n" + "# Doc\n\n## Scope\nold scope\n\n## Other\nkeep me\n") + + # replace: Scope body becomes new; Other untouched. + http("PUT", f"{base}/vault/{doc}", seed) + echo("patch", doc, "replace", "heading", "Doc::Scope", tmp("new scope")) + g = ground(doc) or "" + check("heading replace swaps section body", "new scope" in g and "old scope" not in g, g) + check("heading replace leaves sibling section intact", "## Other\nkeep me" in g, g) + + # append: lands INSIDE the section (before '## Other'), not at EOF. + echo("patch", doc, "append", "heading", "Doc::Scope", tmp("appended line")) + g = ground(doc) or "" + check("heading append inserts within the section (not EOF)", + "appended line" in g and g.index("appended line") < g.index("## Other"), g) + + # prepend: lands at the TOP of the section body. + echo("patch", doc, "prepend", "heading", "Doc::Scope", tmp("first line")) + g = ground(doc) or "" + check("heading prepend inserts at section top", + "first line" in g and g.index("first line") < g.index("new scope"), g) + + # missing heading -> 400 -> echo.py exits non-zero (the silent-loss guard). + r = echo("patch", doc, "append", "heading", "Doc::Nope", tmp("lost?")) + check("missing heading fails loud (non-zero exit)", r.returncode != 0, r.stderr) + check("missing heading does not write", "lost?" not in (ground(doc) or "")) + + # frontmatter replace on an existing field updates it. + echo("fm", doc, "updated", "2026-06-22") + g = ground(doc) or "" + check("frontmatter replace updates existing field", "updated: 2026-06-22" in g, g) + + # frontmatter replace on a MISSING field -> 400 (matches real API). + r = echo("fm", doc, "nonexistent_field", "x") + check("frontmatter replace of a missing field fails loud", r.returncode != 0, r.stderr) + + print(f"\n{len(failures)} failure(s)" if failures else "\nall PATCH-semantics tests passed") + return 1 if failures else 0 + finally: + srv.terminate() + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/eval/test_reflect.py b/eval/test_reflect.py new file mode 100644 index 0000000..6362370 --- /dev/null +++ b/eval/test_reflect.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python3 +"""test_reflect.py — H5: session-reflection capture (dry-run vs --apply). + +A dry-run previews and writes NOTHING; --apply routes each proposal through capture +(creating notes, the inbox line, and skipping low-confidence items). Drives the real +echo.py against eval/mock_olrapi.py. No creds, no live vault. + +Run: python test_reflect.py [--port 8850] +""" +from __future__ import annotations + +import argparse +import json +import os +import subprocess +import sys +import tempfile +import time +import urllib.request +from pathlib import Path + +HERE = Path(__file__).resolve().parent +ECHO = HERE.parent / "echo-memory.plugin.src" / "skills" / "echo-memory" / "scripts" / "echo.py" +KEY = "test-key-not-a-real-secret" + +failures = [] + + +def check(name, cond, detail=""): + print(f"{'ok ' if cond else 'FAIL'} {name}" + (f" -- {detail}" if not cond else "")) + if not cond: + failures.append(name) + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--port", type=int, default=8850) + a = ap.parse_args() + base = f"http://127.0.0.1:{a.port}" + srv = subprocess.Popen([sys.executable, str(HERE / "mock_olrapi.py"), "--port", str(a.port)], + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) + + def http(method, url, body=None): + data = body.encode() if isinstance(body, str) else body + req = urllib.request.Request(url, data=data, method=method, + headers={"Authorization": f"Bearer {KEY}"}) + try: + with urllib.request.urlopen(req, timeout=10) as r: + return r.status, r.read().decode("utf-8", "replace") + except Exception as e: # noqa: BLE001 + return getattr(e, "code", 0), "" + + def echo(*args, stdin=None): + env = dict(os.environ, ECHO_BASE=base, ECHO_KEY=KEY, ECHO_VERIFY="1", ECHO_TODAY="2026-06-22") + return subprocess.run([sys.executable, str(ECHO), *args], input=stdin, + capture_output=True, text=True, env=env) + + def ground(path): + _, body = http("GET", f"{base}/__debug__?path={path}") + return None if body == "<>" else body + + try: + for _ in range(50): + try: + urllib.request.urlopen(f"{base}/__debug__reset", data=b"", timeout=1); break + except Exception: + time.sleep(0.1) + http("PUT", f"{base}/vault/_agent/echo-vault.md", "---\nschema_version: 4\n---\n# marker\n") + + proposals = [ + {"title": "Acme Corp", "kind": "company", "body": "A vendor ECHO integrates with.", "confidence": 0.9}, + {"title": "Use uv not pip", "kind": "semantic", "body": "Jason standardizes on uv.", "confidence": 0.95}, + {"title": "half-formed idea", "inbox": True, "confidence": 0.9}, + {"title": "Maybe relevant", "kind": "concept", "confidence": 0.2}, # below floor -> skipped + ] + pfile = tempfile.NamedTemporaryFile("w", suffix=".json", delete=False, encoding="utf-8") + json.dump(proposals, pfile); pfile.close() + + # dry-run: previews, writes nothing + r = echo("reflect", pfile.name) + check("dry-run previews", "dry-run" in r.stdout and "Acme Corp" in r.stdout, r.stdout) + check("dry-run drops the low-confidence proposal", "Maybe relevant" not in r.stdout, r.stdout) + check("dry-run writes nothing", ground("resources/companies/acme-corp.md") is None) + + # --apply: routes each through capture + r = echo("reflect", pfile.name, "--apply") + check("apply reports applied count", "applied" in r.stdout, r.stdout + r.stderr) + check("apply creates the company note", + (ground("resources/companies/acme-corp.md") or "").find("type: company") >= 0) + check("apply creates the semantic note", + ground("_agent/memory/semantic/use-uv-not-pip.md") is not None) + check("apply routes the inbox proposal", "half-formed idea" in (ground("inbox/captures/inbox.md") or "")) + check("apply still skips the low-confidence proposal", + ground("resources/concepts/maybe-relevant.md") is None) + + # stdin path also works (proposals piped, not a file) + r = echo("reflect", "-", stdin=json.dumps([{"title": "Piped Co", "kind": "company", "confidence": 0.9}])) + check("reflect reads proposals from stdin", "Piped Co" in r.stdout, r.stdout + r.stderr) + + print(f"\n{len(failures)} failure(s)" if failures else "\nall reflect tests passed") + return 1 if failures else 0 + finally: + srv.terminate() + + +if __name__ == "__main__": + raise SystemExit(main())