diff --git a/.DS_Store b/.DS_Store deleted file mode 100644 index 6027d57..0000000 Binary files a/.DS_Store and /dev/null differ diff --git a/CHANGELOG.md b/CHANGELOG.md index a3e764a..493ef29 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,71 @@ # Changelog +## 1.5.0 + +Six improvements from the July 2026 review + the frontmatter-drift field report. + +### Fixed — capture-on-update kept only the first body line (silent data loss) + +`capture` on an existing entity appended `- : ` and **discarded the +rest of the body**. It now appends the whole body as a dated block — first line on the +bullet, continuation lines indented under it. Idempotency (bullet-per-day) unchanged. + +### Added — frontmatter completeness (prevent → tool → detect → remediate) + +A field audit found 18/71 entity notes missing `status`/`tags`; the drift was created +at write time and invisible to the tooling. Four coordinated changes, driven by one +data source (`KIND_STATUS` / `KIND_REQUIRED_FM` in `echo_index.py`): + +- **`capture`** stamps a kind-appropriate default `status` (all kinds, not just + projects) and seeds `tags` with the note's kind; new `--tags a,b` enriches it. + Reflect/triage proposals accept a `"tags"` field. +- **`echo.py fm`** is now **create-or-replace**: on `400 invalid-target` (missing key) + it surgically inserts the one line into the frontmatter block and re-PUTs, verified — + it previously failed on exactly the notes that needed repair. +- **`vault_lint.py`** gains a kind-scoped `incomplete-frontmatter` check + (missing/empty `status`/`tags` on entity notes; block-style lists handled). +- **`sweep.py --apply`** backfills flagged notes (kind-default status + baseline tag). + +### Added — pre-write duplicate gate in capture + +A strong fuzzy candidate (score ≥ 0.6, env `ECHO_DUP_GATE`) now **stops** `capture` +before it creates a near-duplicate (exit `76` + candidate list) instead of warning +after the file exists. `--merge-into ` routes the capture as an update to the +existing entity (mention learned as an alias); `--force` creates anyway. Weak +resemblances still create + warn as before. + +### Changed — recall corpus + ranking (recall-index schema 2) + +- The BM25 corpus now includes **session logs and journal notes** (down-weighted 0.5 / + 0.4 so entity notes win ties) — past discussions are findable without having been + promoted into an entity note. `echo.py put` keeps the index current for corpus paths. +- Ranking fuses BM25 with **freshness** (half-life decay on `updated:`, floored at 0.6, + env `ECHO_FRESH_HALF_LIFE`) and **status** (`active` ×1.1, `on-hold` ×0.9, + `archived` ×0.75) — a stale archived note no longer outranks the live one. +- Hits print `updated:`/`status:`; `recall --json` emits structured results. +- The recall index carries per-doc meta (schema 2); an old index is discarded and + rebuilt automatically on the next recall/sweep. + +### Added — session hooks (self-firing load & reflect) + +New `hooks/hooks.json` + two hook scripts. **SessionStart** runs `echo.py load` and +injects the output as context (skips resume/compact; NOT-CONFIGURED degrades to an +instruction, never an error). **Stop** nudges once per substantive session (≥5 user +turns, env `ECHO_REFLECT_MIN_TURNS`) when nothing was reflected or session-logged — +reflection itself still previews and asks before writing. Both use the 1.4.2 CoWork +path fallback and always exit 0. + +### Added — one-tap inbox triage + `--json` on read verbs + +New `echo.py triage`: `--list --json` parses the inbox into `{line, date, text, +age_days}`; a proposals file (reflect schema + `"line"`) is classified → previewed → +applied via `capture`, with the `inbox/processing-log/` audit line written per move +(originals never deleted). `/echo-triage` rewritten around it. Also: `--json` for +`recall`, `link`, and `scope show` (`resolve`/`capture` already had it). + +Tests: +22 feature cases (mock end-to-end incl. hooks), patch-semantics updated for +the new `fm` contract; all suites green. Docs: SKILL.md, README, command docs updated. + ## 1.4.2 ### Fixed — CoWork sandbox plugin-path resolution diff --git a/MAINTENANCE.md b/MAINTENANCE.md index 9c3e638..b8699d5 100644 --- a/MAINTENANCE.md +++ b/MAINTENANCE.md @@ -34,9 +34,13 @@ The repo root carries every historical build (`echo-memory-0.6.0.plugin` … - [ ] `eval/run_eval.py` still benchmarks **0.6 vs 0.7** — refresh to current (open H4 follow-up). - [x] README version-history has a **1.0.0** entry; title bumped to v1.0.0. - [x] Scrub the literal bearer token from docs → `$ECHO_KEY` placeholder (M1, done: - `api-reference.md` ×11, `SKILL.md`, `bootstrap.md`, eval harness). Operator key setup - documented in `API-KEY-SETUP.md`. Last remaining copy is `echo.py` `DEFAULT_KEY` - (deprecated fallback) — **remove once `ECHO_KEY` is set in every environment**. + `api-reference.md` ×11, `SKILL.md`, `bootstrap.md`, eval harness). + **Superseded in 1.3/1.4:** key resolution now lives in `echo_config.py` + (machine-local `~/.claude/echo-memory/config.json`, `ECHO_*` env overrides, and + empty-in-source `DEFAULT_*` constants filled only by `build.py --bake-key` for + per-user artifacts in gitignored `dist/`). `API-KEY-SETUP.md` and the old + `~/.echo-memory/credentials` fallback are gone; the committed tree ships zero + credentials. ## Building the `.plugin` artifact @@ -44,8 +48,10 @@ The repo root carries every historical build (`echo-memory-0.6.0.plugin` … read from the manifest) and refreshes the `echo-memory.plugin` pointer. Deterministic (sorted entries + fixed timestamp → byte-identical rebuilds). Flags: -- `--strip-key` — blank `echo.py` `DEFAULT_KEY` so **no token ships** (runtime then needs - `ECHO_KEY` or `~/.echo-memory/credentials`). Use this once the key is set everywhere. +- `--strip-key` — force-blank the `echo_config.py` `DEFAULT_*` constants for a guaranteed + token-free artifact (a clean source build is already token-free; this is the belt-and-braces flag). +- `--bake-key --from [--label ]` — per-user secret-bearing artifact in + gitignored `dist/`; deliver directly to that one user, never commit (see README). - `--no-pointer` — skip refreshing `echo-memory.plugin`. - `--outdir DIR` — write artifacts elsewhere. diff --git a/README-gretchen.md b/README-gretchen.md deleted file mode 100644 index 8b2e039..0000000 --- a/README-gretchen.md +++ /dev/null @@ -1,66 +0,0 @@ -# ECHO Memory — install guide (Gretchen) - -This is your personal build of the ECHO Memory plugin: **`echo-memory-1.4.2-gretchen.plugin`**. -Your vault owner name, endpoint, and API key are already **baked in** — you don't need a -config file or to paste a key anywhere. Just install it and go. - -- **Owner:** Gretchen Friedrich -- **Endpoint:** `https://imprintapi.mpm.to` -- **Backend:** Obsidian Local REST API (verified reachable + your key authenticates ✅) - ---- - -## Install - -### In CoWork (recommended) -1. Open the plugins panel. -2. Upload / add the file **`echo-memory-1.4.2-gretchen.plugin`**. -3. Enable **echo-memory** if it isn't on automatically. -4. Start a new session — the skill loads on its own. - -### In Claude Code (desktop / CLI) -1. Save the `.plugin` file somewhere local. -2. Add it through your plugin manager (the `.plugin` file is a standard plugin package). -3. Restart your session so the plugin is picked up. - -No `~/.claude/echo-memory/config.json` needed — the baked build carries everything. - ---- - -## Verify it's working - -Run either: - -- **`/echo-doctor`** — checks Python, your endpoint, that your key authenticates, and - whether the vault is bootstrapped. All green = you're set. -- **`/echo-load`** — loads your memory context for the session. - -If the vault hasn't been set up yet, `/echo-doctor` will say the marker is absent — that's -expected on a brand-new vault and just means a one-time bootstrap is needed. - ---- - -## Day-to-day - -| You want to… | Command | -|---|---| -| Pull in your context at the start of a session | `/echo-load` | -| Save something durable (fact, decision, preference) | `/echo-save` | -| Recall a topic + its linked notes | `/echo-recall ` | -| File a quick capture for later sorting | `/echo-triage` | -| Reflect on a session and capture takeaways | `/echo-reflect` | - -You can also just talk naturally — ask Claude to "remember" something or "what do we know -about X" and it'll route to the right place. - ---- - -## Keep in mind - -- **This file contains your vault key.** Don't forward the `.plugin` to anyone else or post - it in a shared channel — it grants access to your vault under your name. -- **Updating / rotating your key:** you can't edit a baked build. If your token changes, - ask Jason for a fresh build, then reinstall. -- **Trouble connecting?** Run `/echo-doctor` first — it pinpoints whether it's the endpoint, - the key, or the vault. If the endpoint is unreachable, confirm Obsidian + the Local REST - API plugin are running on the host serving `imprintapi.mpm.to`. diff --git a/README.md b/README.md index be0a040..fa39542 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# echo-memory — v1.4.2 +# echo-memory — v1.5.0 Persistent memory for Claude / CoWork sessions via the **ECHO** Obsidian vault, driven over the [Obsidian Local REST API](https://github.com/coddingtonbear/obsidian-local-rest-api). The skill makes direct REST calls through a bundled validated client (`scripts/echo.py`). The whole toolchain is **pure Python** (stdlib only), so it runs identically on Windows, macOS, and Linux — no bash, no platform-specific `date`. @@ -112,7 +112,7 @@ Executable logic ships under `skills/echo-memory/scripts/`; the agent prefers it | Tool | Purpose | |------|---------| | `echo.py` | The validated client + high-level CLI. The network layer is **keep-alive + connection-pooled** (one persistent connection per thread, reused across requests) with a concurrent `read_many()` bulk-GET (`ECHO_WORKERS`, default 8) — so full-vault passes that used to time out finish in under a second. Low-level verbs `get/ls/map/search/put/post/append/patch/fm/bump/delete/lock/unlock/scope/load` inject auth, **check HTTP status** (non-zero exit on ≥400), retry transient 5xx (and reconnect a stale pooled connection), read-back-verify PUT, idempotent whole-line `append`, correct `::` heading targets. High-level ops **`capture/resolve/recall/link`** do the routing/linking for you (see below). | -| `capture / resolve / recall / link` | The input-reducing layer. **`capture "" --kind <k>`** routes via the entity index, stamps frontmatter, indexes, auto-links mentioned entities, and writes the Agent-Log line in one call. **`resolve "<mention>"`** → canonical path (alias-aware). **`recall "<query>"`** → matching notes + their one-hop linked neighbourhood. **`link A B`** → reciprocal `## Related` links. | +| `capture / resolve / recall / link / triage` | The input-reducing layer. **`capture "<title>" --kind <k> [--tags a,b]`** routes via the entity index, stamps **complete** frontmatter (kind-default `status`, kind-seeded `tags`), indexes, auto-links mentioned entities, and writes the Agent-Log line in one call; on an existing entity it appends the **whole body** as a dated block; a strong fuzzy candidate **stops creation** (exit 76 — `--merge-into <slug>` / `--force` resolve it). **`resolve "<mention>"`** → canonical path (alias-aware). **`recall "<query>" [--json]`** → ranked matches + one-hop neighbourhood over entities **and sessions/journal**, fused with freshness + status priors. **`link A B`** → reciprocal `## Related` links. **`triage`** → one-tap inbox routing with an automatic processing-log audit trail. | | `echo_index.py` | The **entity index** (`_agent/index/entities.json`): slug→{path, kind, title, aliases, last_seen}. Makes routing/resolve an O(1), alias-aware lookup and supplies the name→path map for linking and recall. `resolve()` matches on slug/title/alias; a `fuzzy_candidates()` "did-you-mean" fallback (1.2) surfaces near-matches for shortened names without auto-merging. Aliases are auto-derived from titles, learned from mentions on update, stored in note frontmatter, and folded back in by `sweep.py`. Rebuilt automatically by `capture` and `sweep.py`. | | `echo_links.py` / `echo_ops.py` | Cross-link primitives (parse/add `## Related`, bidirectional linking) and the high-level ops layer (capture/recall/resolve/link/agent-log). | | `routing.json` | The **canonical machine-readable** route manifest — one regex pattern per valid destination plus retired paths. The single source of truth for "what may be written where"; `vault_lint.py` enforces it against the vault, `check_routing.py` against the docs. | @@ -124,7 +124,11 @@ Executable logic ships under `skills/echo-memory/scripts/`; the agent prefers it ### Slash commands -`/echo-load` (cold-start read), `/echo-save <text>` (route + persist via `capture`), `/echo-recall <query>` (recall a topic's neighbourhood), `/echo-triage` (drain the inbox), `/echo-health` (run the linter), `/echo-sweep` (bring the vault up to spec), `/echo-reflect` (extract→preview→apply durable items from the session), `/echo-doctor` (readiness check: Python, vault reachability, auth, bootstrap/schema, key source) — explicit, reproducible entry points to the procedures below. +`/echo-load` (cold-start read), `/echo-save <text>` (route + persist via `capture`), `/echo-recall <query>` (recall a topic's neighbourhood), `/echo-triage` (one-tap inbox routing via `echo.py triage`), `/echo-health` (run the linter), `/echo-sweep` (bring the vault up to spec), `/echo-reflect` (extract→preview→apply durable items from the session), `/echo-doctor` (readiness check: Python, vault reachability, auth, bootstrap/schema, key source) — explicit, reproducible entry points to the procedures below. + +### Session hooks (v1.5) + +The plugin ships `hooks/hooks.json`: a **SessionStart** hook runs `echo.py load` and injects the orientation reads as context (cold-start loading no longer depends on the model remembering), and a **Stop** hook nudges **once per substantive session** to run `/echo-reflect` + the session log when nothing was reflected yet (reflection still previews and asks before writing). Both are fail-safe — unconfigured/unreachable vaults degrade to a note, never an error — and use the CoWork path fallback. ### Concurrency (shared vault) @@ -247,7 +251,7 @@ The journal is one append-only time-series stream; rollups are coarser-grained e ### Vault Health (monthly) -Agent self-maintenance (not a journal entry), written to `_agent/health/YYYY-MM-vault-health.md`. Run `scripts/vault_lint.py` (or `/echo-health`) with `ECHO_TODAY` = the conversation's date so stale/aging math uses one clock. It mechanically asserts: folder↔status mismatch, duplicate slugs across lifecycle folders, wikilinks in frontmatter (swept across all folders), duplicate `## Agent Log` headings, stale active projects (`updated:` > 30 days), aging inbox items (> 14 days), **paths matching no route in `routing.json` or sitting at a retired path**, **frontmatter integrity** (missing required fields, `updated` < `created`, future dates, wikilinks leaking into `source_notes`), and **scope drift** (≥ `SCOPE_STALE_SESSIONS` session logs dated after `scope_updated`). Exit codes: `0` clean · `1` violations · `2` unreachable · `3` not bootstrapped. Findings are reported, not auto-fixed. +Agent self-maintenance (not a journal entry), written to `_agent/health/YYYY-MM-vault-health.md`. Run `scripts/vault_lint.py` (or `/echo-health`) with `ECHO_TODAY` = the conversation's date so stale/aging math uses one clock. It mechanically asserts: folder↔status mismatch, duplicate slugs across lifecycle folders, wikilinks in frontmatter (swept across all folders), duplicate `## Agent Log` headings, stale active projects (`updated:` > 30 days), aging inbox items (> 14 days), **paths matching no route in `routing.json` or sitting at a retired path**, **frontmatter integrity** (missing required fields, `updated` < `created`, future dates, wikilinks leaking into `source_notes`), **incomplete entity frontmatter** (missing/empty `status`/`tags` per the kind-scoped `KIND_REQUIRED_FM` map — `sweep.py --apply` backfills), and **scope drift** (≥ `SCOPE_STALE_SESSIONS` session logs dated after `scope_updated`). Exit codes: `0` clean · `1` violations · `2` unreachable · `3` not bootstrapped. Findings are reported, not auto-fixed. --- @@ -380,6 +384,7 @@ If the API returns a connection error, timeout, or `502` (usually Obsidian / the | Version | Highlights | |---------|-----------| +| **1.5.0** | **Six-improvement pass: retention, routing, and self-firing memory.** (1) **Capture-update keeps the whole body** — the update path previously appended only the first line of a multi-line body (silent data loss); now the full body rides under the dated bullet. (2) **Frontmatter completeness** — capture stamps a kind-default `status` and kind-seeded `tags` (`--tags` enriches); `fm` is create-or-replace (missing keys are surgically inserted instead of 400-failing); `vault_lint` gains a kind-scoped `incomplete-frontmatter` check; `sweep --apply` backfills (fixes the 18/71-notes field-audit drift, one data source: `KIND_STATUS`/`KIND_REQUIRED_FM`). (3) **Pre-write duplicate gate** — a strong fuzzy candidate stops `capture` (exit 76 + candidates) before the duplicate exists; `--merge-into <slug>` updates the canonical entity, `--force` overrides. (4) **Recall corpus + ranking** — sessions and journal notes join the BM25 corpus (down-weighted); scores fuse freshness (half-life on `updated:`) and status (`active` boosted, `archived` demoted); hits show `updated:`/`status:`; `recall --json`; recall-index schema 2 (auto-rebuilds). (5) **Session hooks** — SessionStart auto-loads memory into context, Stop nudges reflection once per substantive session; fail-safe, CoWork-fallback-aware. (6) **One-tap triage** — `echo.py triage` lists the inbox structured, then classifies → previews → routes accepted items via `capture` with automatic processing-log audit lines; `--json` also lands on `link`/`scope show`. +22 mock end-to-end tests; all suites green. | | **1.4.2** | **CoWork sandbox path resilience.** In a remote CoWork sandbox `${CLAUDE_PLUGIN_ROOT}` can point at a host path the sandbox can't reach (`/var/folders/…`) while the plugin is mounted under `…/mnt/.remote-plugins/…`, so script invocations failed until the agent found the real path by hand. SKILL.md and all eight slash commands now resolve the scripts dir with a fallback — prefer `${CLAUDE_PLUGIN_ROOT}`, else locate the mounted copy under `/sessions/*/mnt/.remote-plugins/*/…` — reused via `$ECHO`/`$LINT`/`$SWEEP`/`$SDIR`. On a normal host the primary path always wins (no behaviour change). `echo-health`/`echo-sweep` `allowed-tools` broadened to match the resolved invocation. (The scripts never hardcoded a path — root cause is the harness env var — but the plugin now self-heals.) | | **1.4.1** | **Baked credentials now win unconditionally.** A per-user baked artifact could still be prompted for credentials on install: the baked `DEFAULT_*` tier was lowest priority, so a stale `ECHO_*` env var or a leftover/placeholder `~/.claude/echo-memory/config.json` shadowed the baked key — even an unedited `"<placeholder>"` value beat it, flipping `is_configured()` to false and triggering the first-run "ask the operator" flow despite valid baked creds. `echo_config.load()` now treats a **complete** baked set (endpoint + key both present) as authoritative — it wins over env vars and the config file and cannot be shadowed; the generic (unbaked) plugin keeps its env → file → first-run flow. New `baked_complete()`; `source()` reports `baked-in (plugin)`. Per-user artifacts rebaked; manifest → 1.4.1. | | **1.4.0** | **Per-user baked-key builds — zero-touch CoWork install.** The CoWork sandbox doesn't bridge your real `~/.claude`, but the plugin itself is mounted every session — so credentials can travel *inside* the artifact. `echo_config` gains a baked tier (`DEFAULT_OWNER/BASE/KEY`, **empty in source** so the committed tree still ships zero credentials). New `build.py --bake-key --from <config.json> [--label <name>]` substitutes one user's owner/endpoint/key into those constants, producing a per-user artifact that needs **no config file and no per-session paste** (values may also come from `--owner/--endpoint/--key` or `ECHO_*` env). Baked builds default to `dist/` (gitignored) and never touch the shared `echo-memory.plugin` pointer; `--strip-key` force-blanks the constants for a guaranteed token-free build. `config show`/`doctor` report the baked source. | diff --git a/echo-improvements-prompt.md b/echo-improvements-prompt.md new file mode 100644 index 0000000..0d20f96 --- /dev/null +++ b/echo-improvements-prompt.md @@ -0,0 +1,38 @@ +# ECHO plugin — improvements & fixes (frontmatter completeness) + +## Context +While doing routine vault edits I found that partially-populated entity notes accumulate silently. An audit of `resources/people|companies|references` found **18 of 71 notes** missing `status` and/or with empty `tags` (`tags: []` or no key). Neither `echo doctor` nor `vault_lint.py` (`/echo-health`) catches this. I hand-fixed the vault, but the tooling should prevent the drift and make it easy to remediate. Three changes, root-cause first. + +## Root cause — `capture` writes incomplete canonical frontmatter +New entity notes created via `capture` land with `tags: []` and **no `status` key** (verified on freshly-captured `reference` and `company` notes). Every such note is then "incomplete" by convention (complete siblings like `resources/references/obsidian-local-rest-api.md` carry `status: active` + real tags). So the drift is created at write time. + +**Fix (in `capture` / `_build_note`, `echo_ops.py`):** +- Stamp a kind-appropriate default `status` (e.g. `active` for person/company/reference/project) instead of omitting it. +- Replace the empty `tags: []` scaffold with either (a) an auto-seeded baseline tag (the note's `kind`, e.g. `- company`) so the key is never empty, or (b) a required `--tags` argument for entity kinds, or (c) a `needs-tags` marker tag that the completeness check (below) flags. Pick one; my preference is (a)+(b): seed `- <kind>` and accept `--tags a,b,c` to enrich at capture time. +- Acceptance: a note created by `capture "X" --kind company` passes the completeness check below with zero follow-up edits. + +## Fix 2 — `fm` cannot create a missing key (only replace) +`echo.py: cmd_fm()` is `cmd_patch(path, "replace", "frontmatter", field, ...)`. If the field is absent, the Obsidian REST API returns `400 invalid-target` ("The patch you provided could not be applied"). This means `echo fm note.md status active` **fails on exactly the notes that need it** (status missing), forcing agents into GET-modify-PUT, which is error-prone (I clobbered `tags` on 2 notes with a naive rewrite before catching it). + +**Fix:** make `fm` create-or-replace. On `invalid-target`, fall back to inserting the key into the frontmatter block (after `type:`, preserving all other lines) and re-PUT — or add an explicit `fm --create` / `set` op. Must be surgical: never rewrite unrelated frontmatter or body. +- Acceptance: `echo fm note.md status active` succeeds whether or not `status` already exists, changing nothing else in the file. + +## Fix 3 — `vault_lint.py`: add an `incomplete-frontmatter` check +Today `REQUIRED_FM = ("type", "created")` only, and `status` is validated solely for projects (folder↔status match). Nothing requires `status`/`tags` on entity notes. + +**Fix:** add a check that, for entity kinds (`person`, `company`, `reference`, and any others that should be classified — `area`, `skill`), flags: +- missing or empty `status` +- missing `tags` key, or `tags: []`, or a `tags:` key with no list items + +Emit as a new check id `incomplete-frontmatter` with a message like `"{path}: missing status"` / `"{path}: empty tags"`. Keep it kind-scoped so episodic/journal/session notes (which legitimately omit these) are not flagged. Consider a per-kind required-field map rather than the single global `REQUIRED_FM` tuple, so the rule is data-driven. +- Acceptance: running `vault_lint.py` on a vault with a `tags: []` company note reports one `incomplete-frontmatter` violation; a fully-populated note reports none. + +## Nice-to-have +- A `--fix` / remediation mode (or a `sweep` sub-pass) that backfills `status: <default>` and a `- <kind>` baseline tag on flagged notes, so cleanup is one command instead of a hand-rolled script. +- Note the zsh gotcha for any shipped helper loops: unquoted scalar `for x in $var` does NOT word-split in zsh — use arrays or `while read`. + +## Evidence / where I looked +- `echo.py` `cmd_fm` (~L405), arg parser (~L673), `cmd_patch`. +- `vault_lint.py` `REQUIRED_FM` (L42), per-note loop (~L161–173), check-label registry (~L301+), `walk()`/`list_dir()` enumeration. +- `echo_ops.py` `_build_note` (~L103) — has a `status_v` param already, so capture is positioned to stamp status. +- Entity index `_agent/index/entities.json` stores `path/kind/title/aliases/last_seen` — not `tags`/`status` (so a completeness check must read notes, as `vault_lint` already does). diff --git a/echo-memory-1.5.0.plugin b/echo-memory-1.5.0.plugin new file mode 100644 index 0000000..81fd54a Binary files /dev/null and b/echo-memory-1.5.0.plugin differ diff --git a/echo-memory.plugin b/echo-memory.plugin index 51e1908..81fd54a 100644 Binary files a/echo-memory.plugin and b/echo-memory.plugin differ diff --git a/echo-memory.plugin.src/.claude-plugin/plugin.json b/echo-memory.plugin.src/.claude-plugin/plugin.json index eb617ec..86ac2cf 100644 --- a/echo-memory.plugin.src/.claude-plugin/plugin.json +++ b/echo-memory.plugin.src/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "echo-memory", - "version": "1.4.2", - "description": "Persistent memory via the ECHO Obsidian vault over the Obsidian Local REST API. A cross-platform, connection-pooled Python client: one-call capture/resolve/recall/link over an entity index, hybrid BM25 + graph recall, auto-linking, an offline write-ahead queue, and lock-guarded concurrency. Linter-enforced routing manifest plus /echo-load|save|recall|triage|health|sweep|reflect|doctor commands. Crosslinks notes across Claude and CoWork sessions.", + "version": "1.5.0", + "description": "Persistent memory via the ECHO Obsidian vault over the Obsidian Local REST API. Cross-platform Python client: one-call capture/resolve/recall/link/triage over an entity index, hybrid BM25 + graph recall spanning entities + sessions/journal (recency/status-aware), a pre-write duplicate gate, complete-frontmatter capture, session hooks that self-fire load/reflect, offline write-ahead queue, lock-guarded concurrency, linter-enforced routing, and /echo-* commands.", "author": { "name": "Jason Stedwell" }, diff --git a/echo-memory.plugin.src/README.md b/echo-memory.plugin.src/README.md index decd96b..6e462a3 100644 --- a/echo-memory.plugin.src/README.md +++ b/echo-memory.plugin.src/README.md @@ -11,11 +11,12 @@ Architected by **Jason Stedwell**. ## What it does - Loads operator preferences, current context, and relevant project notes at the start of substantive conversations -- **One-call `capture`** routes a memory to its canonical home, stamps frontmatter, indexes it, auto-links any entity it mentions, and logs it — driven by a machine-maintained **entity index** so routing is an alias-aware lookup, not a fuzzy search -- **`recall`** returns a topic's matching notes *and* their one-hop linked neighbourhood (Related links + `source_notes`) for comprehensive context; **`resolve`** maps any mention to its canonical path; **`link`** adds reciprocal cross-links +- **One-call `capture`** routes a memory to its canonical home, stamps **complete** frontmatter (kind-default `status`, kind-seeded `tags`), indexes it, auto-links any entity it mentions, and logs it — driven by a machine-maintained **entity index** so routing is an alias-aware lookup, not a fuzzy search. Updates keep the **whole body**; a title that strongly resembles an existing entity **stops at a duplicate gate** (exit 76 — `--merge-into <slug>` / `--force`) instead of spawning a near-duplicate +- **`recall`** returns a topic's matching notes *and* their one-hop linked neighbourhood (Related links + `source_notes`) — the corpus spans the entity graph **plus session logs and journal notes** (down-weighted), ranked by BM25 × freshness × status so live notes outrank stale ones; **`resolve`** maps any mention to its canonical path; **`link`** adds reciprocal cross-links; **`triage`** routes aging inbox captures with an automatic processing-log audit trail - Logs working sessions so future conversations can pick up where they left off - **Bootstraps an empty vault from the bundled `scaffold/`** (folders, templates, anchor seeds, marker), repairs/migrates an existing one via `scripts/bootstrap.py` / `scripts/migrate.py`, and brings an upgraded vault up to spec (index + cross-links) via `scripts/sweep.py` — see `skills/echo-memory/references/bootstrap.md` - Exposes `/echo-load`, `/echo-save`, `/echo-recall`, `/echo-triage`, `/echo-health`, `/echo-sweep`, `/echo-reflect`, `/echo-doctor` slash commands as explicit entry points, and coordinates concurrent Claude/CoWork sessions via a cooperative advisory lock +- **Ships session hooks** (`hooks/hooks.json`): SessionStart auto-runs the cold-start load into context, and Stop nudges `/echo-reflect` once per substantive session — the load/reflect discipline fires deterministically instead of depending on the model remembering (reflection still previews before writing) ## Configuration diff --git a/echo-memory.plugin.src/commands/echo-recall.md b/echo-memory.plugin.src/commands/echo-recall.md index 3f93505..ca81433 100644 --- a/echo-memory.plugin.src/commands/echo-recall.md +++ b/echo-memory.plugin.src/commands/echo-recall.md @@ -16,4 +16,6 @@ python3 "$ECHO" recall "$ARGUMENTS" # Windows: use `python` or `py -3` if `python3` is not on PATH. ``` +The corpus spans the entity graph **plus session logs and journal notes**, ranked by relevance × freshness × status — each hit shows its `updated:`/`status:`, so prefer fresh/active hits and treat stale ones as history. Add `--json` for structured hits (`path/score/type/updated/status/excerpt`) when you need to act on the results programmatically. + Then answer from the assembled context. If you only need the canonical path for one entity, use `echo.py resolve "<title>"` instead. Don't narrate the retrieval. diff --git a/echo-memory.plugin.src/commands/echo-reflect.md b/echo-memory.plugin.src/commands/echo-reflect.md index daef919..a56207d 100644 --- a/echo-memory.plugin.src/commands/echo-reflect.md +++ b/echo-memory.plugin.src/commands/echo-reflect.md @@ -10,7 +10,7 @@ things worth remembering across sessions, then propose them — don't write blin commitments, people/companies/projects introduced, and anything the operator said to remember. Skip the ephemeral. Anchor relative dates on the conversation's `currentDate`. 2. **Emit a JSON array** of proposals (one per item), each: - `{"title","kind","body","aliases","sources","confidence"}` — `kind` ∈ + `{"title","kind","body","aliases","tags","sources","confidence"}` — `kind` ∈ `person, company, concept, reference, meeting, project, area, semantic, episodic, working, skill, decision`; set `"inbox": true` when the home is genuinely unknown; `confidence` 0–1 (items below 0.6 are dropped — send those to the inbox instead). diff --git a/echo-memory.plugin.src/commands/echo-save.md b/echo-memory.plugin.src/commands/echo-save.md index 5b4e10c..01b1944 100644 --- a/echo-memory.plugin.src/commands/echo-save.md +++ b/echo-memory.plugin.src/commands/echo-save.md @@ -7,15 +7,17 @@ Use the **echo-memory** skill to persist this to the ECHO vault: > $ARGUMENTS -Prefer the one-call **`capture`** — it routes (via the entity index), derives the canonical path, stamps frontmatter, indexes, auto-links mentioned entities, and writes the Agent-Log line. Pick the `--kind` from the content (`person, company, concept, reference, meeting, project, area, semantic, episodic, working, skill, decision`); use `--inbox` only when the home is genuinely unknown. Write multi-line bodies to a file with the Write tool (cross-platform, no heredoc). +Prefer the one-call **`capture`** — it routes (via the entity index), derives the canonical path, stamps complete frontmatter (kind-default `status`, the kind seeded into `tags`), indexes, auto-links mentioned entities, and writes the Agent-Log line. Pick the `--kind` from the content (`person, company, concept, reference, meeting, project, area, semantic, episodic, working, skill, decision`); add `--tags` when obvious; use `--inbox` only when the home is genuinely unknown. Write multi-line bodies to a file with the Write tool (cross-platform, no heredoc) — updates to an existing entity preserve the whole body, so don't pre-trim it. ```bash ECHO="${CLAUDE_PLUGIN_ROOT}/skills/echo-memory/scripts/echo.py" # run with: python3 "$ECHO" ... (Windows: python / py -3) [ -f "$ECHO" ] || ECHO=$(ls /sessions/*/mnt/.remote-plugins/*/skills/echo-memory/scripts/echo.py 2>/dev/null | head -1) # CoWork sandbox fallback -python3 "$ECHO" capture "<title>" <bodyfile> --kind <kind> [--aliases a,b] [--source p1,p2] +python3 "$ECHO" capture "<title>" <bodyfile> --kind <kind> [--aliases a,b] [--tags t1,t2] [--source p1,p2] python3 "$ECHO" capture "<title>" --inbox # unknown home -> idempotent inbox line ``` +**If capture exits `76` (duplicate gate):** the title strongly resembles an existing entity — do NOT retry blindly. Show the operator the printed candidates and either update the existing note (`capture ... --merge-into <slug>`) or, only after they confirm it's genuinely distinct, re-run with `--force`. + Drop to the low-level verbs only for shapes `capture` doesn't model (a project `## Status` replace, an ADR mirror, a daily-note edit): ```bash diff --git a/echo-memory.plugin.src/commands/echo-triage.md b/echo-memory.plugin.src/commands/echo-triage.md index 7324602..9f3a37a 100644 --- a/echo-memory.plugin.src/commands/echo-triage.md +++ b/echo-memory.plugin.src/commands/echo-triage.md @@ -2,13 +2,22 @@ description: Triage the ECHO inbox — route aging captures to their canonical homes and log the moves --- -Use the **echo-memory** skill to run **Inbox Triage**. GET `inbox/captures/inbox.md`, list captures older than ~7 days that were never routed, and offer to route them. +Use the **echo-memory** skill to run **one-tap Inbox Triage** via the `triage` verb (it reuses the reflect pipeline: classify against the entity index → preview → apply via `capture`, and writes the audit log for you). ```bash ECHO="${CLAUDE_PLUGIN_ROOT}/skills/echo-memory/scripts/echo.py" [ -f "$ECHO" ] || ECHO=$(ls /sessions/*/mnt/.remote-plugins/*/skills/echo-memory/scripts/echo.py 2>/dev/null | head -1) # CoWork sandbox fallback -python3 "$ECHO" get inbox/captures/inbox.md +python3 "$ECHO" triage --list --json # Windows: use `python` or `py -3` if `python3` is not on PATH. ``` -For each capture the operator accepts: send it to its proper home per the routing map (preference → `operator-preferences.md::Observations`; project idea → `projects/incubating/`; durable fact → `_agent/memory/semantic/`; person → `resources/people/`), then record the move in `inbox/processing-log/YYYY-MM-DD.md` (`- <original> → <destination>`). Do **not** delete the original capture unless the operator explicitly asks — the processing log is the audit trail. +1. **List** — the JSON gives each capture as `{line, date, text, age_days}`. Surface the aging ones (≥ ~7 days) to the operator and ask which to route. +2. **Propose** — for accepted items, write a proposals JSON with the Write tool (reflect schema: `{"title","kind","body",...}` **plus `"line"`: the original inbox line**, echoed into the audit log). Route per the map: preference/pattern → `semantic` (or a direct PATCH to `operator-preferences.md::Observations`); project idea → `project` with `--status incubating`; durable fact → `semantic`; person fact → `person`. +3. **Preview, then apply** with the operator's go-ahead: + +```bash +python3 "$ECHO" triage proposals.json # dry-run: classify + preview, writes nothing +python3 "$ECHO" triage proposals.json --apply # route via capture + log each move +``` + +`--apply` records every move in `inbox/processing-log/YYYY-MM-DD.md` (`- <original> → <destination>`) automatically. Do **not** delete the original captures unless the operator explicitly asks — the processing log is the audit trail. A proposal that stops at the duplicate gate (exit note in the summary) should be re-proposed with the existing entity's title. diff --git a/echo-memory.plugin.src/hooks/hooks.json b/echo-memory.plugin.src/hooks/hooks.json new file mode 100644 index 0000000..b2ffcaa --- /dev/null +++ b/echo-memory.plugin.src/hooks/hooks.json @@ -0,0 +1,27 @@ +{ + "hooks": { + "SessionStart": [ + { + "matcher": "startup|clear", + "hooks": [ + { + "type": "command", + "command": "S=\"${CLAUDE_PLUGIN_ROOT}/skills/echo-memory/scripts/echo_hook_session_start.py\"; [ -f \"$S\" ] || S=$(ls /sessions/*/mnt/.remote-plugins/*/skills/echo-memory/scripts/echo_hook_session_start.py 2>/dev/null | head -1); [ -n \"$S\" ] || exit 0; python3 \"$S\" || python \"$S\"", + "timeout": 60 + } + ] + } + ], + "Stop": [ + { + "hooks": [ + { + "type": "command", + "command": "S=\"${CLAUDE_PLUGIN_ROOT}/skills/echo-memory/scripts/echo_hook_stop.py\"; [ -f \"$S\" ] || S=$(ls /sessions/*/mnt/.remote-plugins/*/skills/echo-memory/scripts/echo_hook_stop.py 2>/dev/null | head -1); [ -n \"$S\" ] || exit 0; python3 \"$S\" || python \"$S\"", + "timeout": 30 + } + ] + } + ] + } +} diff --git a/echo-memory.plugin.src/skills/echo-memory/SKILL.md b/echo-memory.plugin.src/skills/echo-memory/SKILL.md index dfebbfd..edab8f8 100644 --- a/echo-memory.plugin.src/skills/echo-memory/SKILL.md +++ b/echo-memory.plugin.src/skills/echo-memory/SKILL.md @@ -80,16 +80,18 @@ python3 "$ECHO" lock <session-id> ; python3 "$ECHO" unlock <session-id> These collapse the multi-step write/recall discipline into one call each, backed by the **entity index** (`_agent/index/entities.json`, a slug→{path,kind,title,aliases} registry). **Default to them** — they are how this skill reduces the input needed to route and cross-link memory: ```bash -python3 "$ECHO" capture "<title>" <bodyfile> --kind <kind> [--aliases a,b] [--source p1,p2] [--status s] [--date YYYY-MM-DD] [--domain business] +python3 "$ECHO" capture "<title>" <bodyfile> --kind <kind> [--aliases a,b] [--tags t1,t2] [--source p1,p2] [--status s] [--date YYYY-MM-DD] [--domain business] python3 "$ECHO" capture "<title>" --inbox # unknown home -> idempotent inbox line python3 "$ECHO" resolve "<mention>" # mention -> canonical path (or where to create) -python3 "$ECHO" recall "<query>" # search + 1-hop link/source expansion -> connected context -python3 "$ECHO" link <pathA> <pathB> # add reciprocal [[Related]] links A<->B +python3 "$ECHO" recall "<query>" [--json] # ranked search + 1-hop expansion -> connected context +python3 "$ECHO" link <pathA> <pathB> [--json] # add reciprocal [[Related]] links A<->B +python3 "$ECHO" triage --list --json # structured inbox listing (see Inbox Triage) ``` -- **`capture` is the default write.** Given a title + `--kind`, it: resolves the entity via the index (create-vs-update, no manual search-first), derives the canonical path, stamps canonical frontmatter (`type/created/updated/aliases/agent_written/source_notes`), updates the index, **auto-links** any known entity mentioned in the body (bidirectionally), and appends today's Agent Log line — one call instead of search→put→bump→log. It also **auto-derives aliases** from the title, **learns the mention as an alias** when updating an entity under a different name, and **warns if the new note resembles an existing entity** (so a shortened name can't silently spawn a duplicate). `--kind` ∈ `person, company, concept, reference, meeting, project, area, semantic, episodic, working, skill, decision`. Use `--inbox` (or omit `--kind`) only when the home is genuinely unknown. +- **`capture` is the default write.** Given a title + `--kind`, it: resolves the entity via the index (create-vs-update, no manual search-first), derives the canonical path, stamps **complete** canonical frontmatter (`type/status/created/updated/tags/aliases/agent_written/source_notes` — a kind-appropriate default `status` and the kind seeded as a baseline tag, enriched by `--tags`), updates the index, **auto-links** any known entity mentioned in the body (bidirectionally), and appends today's Agent Log line — one call instead of search→put→bump→log. It also **auto-derives aliases** from the title and **learns the mention as an alias** when updating an entity under a different name. Updating an existing entity appends the **whole body** as a dated block (first line on the bullet, the rest indented under it) — nothing is truncated. `--kind` ∈ `person, company, concept, reference, meeting, project, area, semantic, episodic, working, skill, decision`. Use `--inbox` (or omit `--kind`) only when the home is genuinely unknown. +- **Duplicate gate (create path).** When a new title strongly resembles an existing entity (fuzzy score ≥ 0.6, env `ECHO_DUP_GATE`), `capture` **stops before writing** and exits `76` with the candidate list, instead of creating a near-duplicate and warning afterwards. Resolve it deliberately: `--merge-into <slug>` routes the capture as an update to the existing entity (the new name is learned as an alias), or `--force` creates anyway after you've confirmed they're genuinely distinct. Weak resemblances still create + warn as before. - **`resolve` before constructing any path by hand** — it returns the canonical note (matching slug/title/alias, so *Bob/Robert/RS* converge) or, when nothing matches exactly, a ranked list of **`candidates`** (entities sharing a distinctive token — e.g. "echo memory" surfaces the project `echo`). Always check candidates before creating a new note; a shortened name not matching exactly is the classic way a duplicate gets made. This replaces the two-search "search-first" dance for slug-addressed notes. -- **`recall` for "what do we know about X"** — it returns the matching notes *and* their one-hop neighbourhood (Related links, `source_notes`), so recall surfaces the web around a topic, not an isolated note. +- **`recall` for "what do we know about X"** — it returns the matching notes *and* their one-hop neighbourhood (Related links, `source_notes`), so recall surfaces the web around a topic, not an isolated note. The corpus covers the entity graph **plus session logs and journal notes** (down-weighted, so entity notes win ties) — past discussions and decisions are findable even when nobody promoted them into an entity note. Ranking fuses BM25 with **freshness** (`updated:` half-life decay) and **status** (`active` boosted, `archived` demoted), and each hit prints its `updated:`/`status:` so staleness is visible. `--json` emits structured hits (`path/score/type/updated/status/excerpt`) instead of prose. - **`link`** when two existing notes are related but not yet connected; `capture` already auto-links what it detects. The index is maintained automatically by `capture`; `sweep.py` rebuilds it and back-fills links for an existing vault (see **Vault Maintenance**). @@ -112,6 +114,15 @@ Use a stable `<session-id>` for both calls (any unique string for this session). `/echo-load` (cold-start read), `/echo-save <text>` (route + persist via `capture`), `/echo-recall <query>` (recall a topic's neighbourhood), `/echo-triage` (drain the inbox), `/echo-health` (run the linter), `/echo-sweep` (bring the vault up to current spec), `/echo-reflect` (extract→preview→apply durable items from the session), `/echo-doctor` (readiness check: config, reachability, auth, bootstrap/schema). These are explicit entry points to the procedures below. +### Session hooks (self-firing load & reflect) + +The plugin ships `hooks/hooks.json`, so the two most drift-prone behaviors no longer depend on remembering them: + +- **SessionStart** runs `echo.py load` and injects the output as context — cold-start loading is deterministic. Still do the **reconcile** (inbox depth, scope confirm) on that injected context before working. +- **Stop** fires **once per substantive session** when nothing has been reflected or session-logged yet: it blocks with a reminder to run the `/echo-reflect` flow and write the session log + heartbeat. Reflection still previews and asks the operator before writing — the hook only ensures the question gets asked. If nothing durable emerged, just finish; never invent memories to satisfy the nudge. + +If the hooks are disabled or unavailable, the manual procedures below are unchanged and still authoritative. + ## Operating Contract & Safety The vault is the **system of record** for long-term memory, not a scratchpad. Default to **additive updates, explicit status changes, and traceable summaries**. Keep agent-managed content (`agent_written: true` + `source_notes`) separable from human-authored content. Non-negotiable safety rules: @@ -159,22 +170,25 @@ python3 "$ECHO" recall "<topic or title>" # the matching notes AND their one-h Do NOT narrate this loading. Reading memory is expected behavior. -## Inbox Triage +## Inbox Triage (one-tap: `echo.py triage`) `inbox/captures/inbox.md` is the catch-all for "save this somewhere — figure out where later." Without triage it grows forever and durable facts get stranded there. -At the start of a substantive session, GET `inbox/captures/inbox.md`. If it contains lines older than ~7 days that haven't been routed elsewhere, surface them once: +At the start of a substantive session, check the inbox. If it holds lines older than ~7 days that haven't been routed elsewhere, surface them once: > "Three captures from last week are still in the inbox — want to route them now or leave them?" -When routing accepted items, send each to its proper home: +**The triage verb does the mechanical work** — it reuses the reflect pipeline (classify against the entity index → preview → apply via `capture`) and writes the processing-log audit line for every move: -- Preference / pattern → PATCH-append under `operator-preferences.md::Observations` -- Project idea → PUT `projects/incubating/<slug>.md` -- Durable fact → PUT `_agent/memory/semantic/<slug>.md` -- Person fact → PUT/PATCH `resources/people/<name>.md` +```bash +python3 "$ECHO" triage --list --json # structured captures: {line, date, text, age_days} +# decide a kind/title per accepted item, write a proposals JSON (reflect schema + +# optional "line": the original inbox line, echoed into the audit log), then: +python3 "$ECHO" triage proposals.json # dry-run: classify + preview, writes nothing +python3 "$ECHO" triage proposals.json --apply # route via capture + log each move +``` -Then record the move in `inbox/processing-log/YYYY-MM-DD.md` (one line per item: `- <original line> → <destination path>`). Don't delete the original capture unless the operator explicitly asks — the processing log is the audit trail. +Routing targets are the usual homes (preference/pattern → `operator-preferences.md::Observations` via a `semantic` proposal or a direct PATCH; project idea → `project` with `--status incubating`; durable fact → `semantic`; person fact → `person`). `--apply` records each move in `inbox/processing-log/YYYY-MM-DD.md` (`- <original line> → <destination path>`) automatically. Originals are **never deleted** unless the operator explicitly asks — the processing log is the audit trail. ## Project Lifecycle @@ -247,6 +261,9 @@ python3 "$ECHO" put projects/active/<slug>.md <bodyfile> # frontmatter freshness after a SUBSTANTIVE change (status/decision/scope) — NOT routine log appends ECHO_TODAY=<currentDate> python3 "$ECHO" bump projects/active/<slug>.md # sets updated: to today +# fm is CREATE-or-replace: a key the note lacks is inserted surgically (nothing else +# in the file is touched), so repairing a note missing `status:` is one call: +python3 "$ECHO" fm <path> status active # search (run before creating any slug-addressed note — see the search-first rule above) python3 "$ECHO" search <terms...> @@ -311,10 +328,11 @@ Exit codes: `0` clean · `1` violations · `2` unreachable · `3` not bootstrapp 7. **Unknown / retired paths** — any vault file that matches no route in `scripts/routing.json` (or sits at a retired path). 8. **Frontmatter integrity** — missing required fields, `updated` < `created`, future `updated`, and `[[wikilinks]]` leaking into `source_notes`. 9. **Graph health** — **broken wikilinks** (`[[X]]` resolving to no note), **orphans** (an entity note with no inbound links and an empty `## Related`), and **index drift** (entity-index entries whose file is gone, or indexable notes missing from `entities.json` — fix with `sweep.py`). +10. **Incomplete entity frontmatter** — entity notes missing or empty `status:`/`tags:` per the kind-scoped requirement map (`KIND_REQUIRED_FM` in `echo_index.py`; journal/session notes are exempt). `capture` now stamps these at write time; `sweep.py --apply` backfills older notes (kind-default status + the kind as a baseline tag). ## Vault Maintenance — `sweep.py` (bring the vault up to spec) -After a plugin upgrade, run the sweep once to back-fill what older vaults lack — it (1) creates `_agent/index/`, (2) **rebuilds the entity index** from the notes already present, (3) **symmetrizes cross-links** (for every `## Related` A→B, adds the missing B→A; it never invents new links from body text), and (4) stamps the marker `schema_version` to current. Dry-run by default: +After a plugin upgrade, run the sweep once to back-fill what older vaults lack — it (1) creates `_agent/index/`, (2) **rebuilds the entity index** from the notes already present (and the recall index, now spanning entities + sessions/journal), (3) **backfills incomplete entity frontmatter** (kind-default `status`, the kind as a baseline tag — what `/echo-health` flags as `incomplete-frontmatter`), (4) **symmetrizes cross-links** (for every `## Related` A→B, adds the missing B→A; it never invents new links from body text), and (5) stamps the marker `schema_version` to current. Dry-run by default: ```bash SWEEP="${CLAUDE_PLUGIN_ROOT}/skills/echo-memory/scripts/sweep.py" diff --git a/echo-memory.plugin.src/skills/echo-memory/references/api-reference.md b/echo-memory.plugin.src/skills/echo-memory/references/api-reference.md index 3be08d1..85b78bb 100644 --- a/echo-memory.plugin.src/skills/echo-memory/references/api-reference.md +++ b/echo-memory.plugin.src/skills/echo-memory/references/api-reference.md @@ -156,6 +156,12 @@ Same call with `Operation: prepend`. ### Patch a frontmatter field +> A `PATCH` `Target-Type: frontmatter` **replace** on a key the note does not have returns +> `400 invalid-target` — the raw API cannot create a key. `echo.py fm` (v1.5+) handles this: +> on that 400 it surgically inserts the one `field: value` line into the frontmatter block +> and re-PUTs (verified), leaving every other line untouched. Raw-curl users must GET-edit-PUT +> themselves — carefully (a naive rewrite is how frontmatter gets clobbered). + ```bash curl -s -X PATCH \ -H "Authorization: Bearer $ECHO_KEY" \ diff --git a/echo-memory.plugin.src/skills/echo-memory/references/vault-layout.md b/echo-memory.plugin.src/skills/echo-memory/references/vault-layout.md index 322ee94..fac66c9 100644 --- a/echo-memory.plugin.src/skills/echo-memory/references/vault-layout.md +++ b/echo-memory.plugin.src/skills/echo-memory/references/vault-layout.md @@ -61,15 +61,15 @@ ## Canonical Frontmatter -Every note starts with this block. Fill what applies; leave the rest empty rather than guessing. +Every note starts with this block. Fill what applies; leave the rest empty rather than guessing — **except `status:` and `tags:` on entity notes** (person/company/concept/reference/project/area/skill, plus `tags` on meeting/decision), which must be populated: `capture` stamps a kind-appropriate default `status` and seeds `tags` with the note's kind, `/echo-health` flags missing/empty ones (`incomplete-frontmatter`), and `sweep.py --apply` backfills older notes. The per-kind requirement map is `KIND_REQUIRED_FM` / `KIND_STATUS` in `scripts/echo_index.py`. ```yaml --- type: # see Note Types below -status: # active | draft | done | archived | complete +status: # active | draft | done | archived | complete — REQUIRED on entity notes (kind default stamped by capture) created: # YYYY-MM-DD (or YYYY-MM-DDTHH:mm for sessions) updated: # YYYY-MM-DD -tags: [] +tags: [] # REQUIRED (non-empty) on entity notes — capture seeds the kind (e.g. [company]); enrich via --tags aliases: [] # optional — other names this entity is called (Obsidian-native). Folded # into the entity index so a shortened/expanded name resolves to this note. agent_written: false diff --git a/echo-memory.plugin.src/skills/echo-memory/scripts/echo.py b/echo-memory.plugin.src/skills/echo-memory/scripts/echo.py index 9e20015..0f3b1ec 100644 --- a/echo-memory.plugin.src/skills/echo-memory/scripts/echo.py +++ b/echo-memory.plugin.src/skills/echo-memory/scripts/echo.py @@ -40,8 +40,9 @@ Usage: echo.py post <path> [file] # raw append (NON-idempotent; prefer `append`) echo.py append <path> <line> # idempotent append: skips only on an exact whole-line match echo.py patch <path> <append|prepend|replace> <heading|frontmatter|block> <target> [file] - echo.py fm <path> <field> <json-value> # PATCH a frontmatter scalar + echo.py fm <path> <field> <json-value> # create-or-replace a frontmatter scalar echo.py bump <path> [YYYY-MM-DD] # set frontmatter updated: to today (or given date) + echo.py triage [--list --json | <proposals.json> [--apply]] # one-tap inbox triage + audit log echo.py delete <path> # DELETE (destructive; explicit use only) echo.py lock <owner-id> # acquire advisory lock (exit 75 if held & fresh, or lost the race) echo.py unlock <owner-id> # release advisory lock if owned by <owner-id> @@ -49,7 +50,8 @@ Usage: echo.py scope set "<text>" # switch scope atomically (history + replace + stamp scope_updated) echo.py config [show|init|set|import] # machine-local owner/endpoint/key (config import <file> adopts a provided file) -Exit codes: 0 ok · 44 not-found(404) · 75 lock-held/lost · 78 config-required · 2 usage · 1 other HTTP/transport error. +Exit codes: 0 ok · 44 not-found(404) · 75 lock-held/lost · 76 duplicate-gate (capture) · + 78 config-required · 2 usage · 1 other HTTP/transport error. """ from __future__ import annotations @@ -60,6 +62,7 @@ import http.client import json import locale import os +import re import sys import tempfile import threading @@ -339,6 +342,14 @@ def cmd_put(path: str, file_arg: str | None) -> int: if status != 200: raise EchoError(f"put {path}: write did not verify (GET returned {status})") print(f"ok: PUT {path}") + # Keep the recall (BM25) index current when a corpus note is written directly — + # session logs, journal notes, hand-authored entity notes. update_note early-returns + # for non-corpus paths and never raises, so this can't fail or slow a normal put. + try: + import echo_recall + echo_recall.update_note(path, data.decode("utf-8", errors="replace")) + except Exception as exc: # noqa: BLE001 — index upkeep must never fail a put + print(f"echo.py: recall-index update skipped ({exc})", file=sys.stderr) return 0 @@ -402,8 +413,64 @@ def cmd_patch(path: str, op: str, target_type: str, target: str, file_arg: str | return 0 +def _yaml_flow(value) -> str: + """Render a JSON-decoded value as a YAML flow scalar/list for one frontmatter line.""" + if isinstance(value, list): + return "[" + ", ".join(_yaml_flow(v) for v in value) + "]" + if isinstance(value, bool): + return "true" if value else "false" + if value is None: + return "" + s = str(value) + return s if re.match(r"^[A-Za-z0-9][A-Za-z0-9 ._/-]*$", s) else json.dumps(s) + + +def _fm_upsert_line(text: str, field: str, rendered: str) -> str: + """Surgically set `field: rendered` inside the YAML frontmatter block, creating the + key (or the whole block) if absent. Every other line is preserved verbatim — this is + the safety property that makes it OK to fall back from a failed PATCH.""" + line = f"{field}: {rendered}".rstrip() + lines = text.splitlines() + if lines and lines[0].strip() == "---": + for end in range(1, len(lines)): + if lines[end].strip() == "---": + for i in range(1, end): # key exists -> replace that one line + if re.match(rf"^{re.escape(field)}\s*:", lines[i]): + lines[i] = line + break + else: + lines.insert(end, line) # else append just before the closing --- + return "\n".join(lines) + ("\n" if text.endswith("\n") else "") + return f"---\n{line}\n---\n{text}" # no frontmatter at all -> prepend a minimal block + + def cmd_fm(path: str, field: str, value: str) -> int: - return cmd_patch(path, "replace", "frontmatter", field, temp_file(normalize_json_scalar(value))) + """Create-or-replace a frontmatter scalar. PATCH replace handles the common case; + a missing key returns 400 invalid-target (PATCH cannot create), which used to make + `fm` fail on exactly the notes that needed repair — now it falls back to a surgical + GET -> insert-one-line -> verified PUT.""" + data = normalize_json_scalar(value) + try: + return cmd_patch(path, "replace", "frontmatter", field, temp_file(data)) + except EchoError as exc: + if "HTTP 400" not in str(exc): + raise + status, body = request("GET", vault_url(path)) + check(status, body, f"fm {path}") + new = _fm_upsert_line(body.decode(errors="replace"), field, + _yaml_flow(json.loads(data.decode("utf-8")))) + status, body = _qwrite("PUT", vault_url(path), data=new.encode("utf-8"), + headers={"Content-Type": "text/markdown"}) + if status == 202: + print(f"queued (offline): FM {field} -> {path} — will sync on the next reachable session") + return 0 + check(status, body, f"fm {path}") + if VERIFY: + status, body = request("GET", vault_url(path)) + if status != 200 or not re.search(rf"(?m)^{re.escape(field)}\s*:", body.decode(errors="replace")): + raise EchoError(f"fm {path}: created key '{field}' did not verify on read-back") + print(f"ok: FM created '{field}' in {path}") + return 0 def cmd_delete(path: str) -> int: @@ -482,31 +549,39 @@ def extract_heading(markdown: str, heading: str) -> str: return "\n".join(out).strip() -def cmd_scope(subcommand: str, text: str | None = None) -> int: +def cmd_scope(subcommand: str, text: str | None = None, as_json: bool = False) -> int: path = "_agent/context/current-context.md" status, body = request("GET", vault_url(path)) check(status, body, f"scope {subcommand}") current = body.decode(errors="replace") if subcommand == "show": - print("-- Active scope --") - print(extract_heading(current, "Scope")) + scope_text = extract_heading(current, "Scope") scope_updated = "" for line in current.splitlines(): if line.startswith("scope_updated:"): scope_updated = line.split(":", 1)[1].strip().strip('"').strip("'") break - print(f"scope_updated: {scope_updated or '<missing — drift cannot be detected; run scope set or repair>'}") + sessions_since = None # Count session logs dated after scope_updated (the drift signal). if scope_updated: status, body = request("GET", vault_url("_agent/sessions/")) if status == 200: try: files = json.loads(body).get("files", []) - n = sum(1 for f in files if f.endswith(".md") and f[:10] > scope_updated) - print(f"sessions logged since: {n}") + sessions_since = sum(1 for f in files if f.endswith(".md") and f[:10] > scope_updated) except json.JSONDecodeError: pass + if as_json: + print(json.dumps({"ok": True, "action": "scope-show", "scope": scope_text, + "scope_updated": scope_updated or None, + "sessions_since": sessions_since}, ensure_ascii=False)) + return 0 + print("-- Active scope --") + print(scope_text) + print(f"scope_updated: {scope_updated or '<missing — drift cannot be detected; run scope set or repair>'}") + if sessions_since is not None: + print(f"sessions logged since: {sessions_since}") return 0 if subcommand != "set": @@ -682,14 +757,22 @@ def build_parser() -> argparse.ArgumentParser: p = sub.add_parser("scope") p.add_argument("subcommand", nargs="?", default="show", choices=["show", "set"]) p.add_argument("text", nargs="?") + p.add_argument("--json", action="store_true") # structured scope + freshness p = sub.add_parser("reflect") # H5: apply a JSON proposal set (dry-run unless --apply) p.add_argument("file", nargs="?") p.add_argument("--apply", action="store_true") + p = sub.add_parser("triage") # one-tap inbox triage (reflect pipeline + audit log) + p.add_argument("file", nargs="?") # proposals JSON; omit to list + p.add_argument("--list", action="store_true", dest="list_inbox") + p.add_argument("--json", action="store_true") # structured inbox listing + p.add_argument("--apply", action="store_true") # route + write processing log # High-level operations (entity index + cross-links + recall). See echo_ops.py. sub.add_parser("resolve").add_argument("mention", nargs="+") p = sub.add_parser("recall"); p.add_argument("query", nargs="+"); p.add_argument("--limit", type=int, default=6) + p.add_argument("--json", action="store_true") # structured hits + neighbourhood p = sub.add_parser("link"); p.add_argument("a"); p.add_argument("b") + p.add_argument("--json", action="store_true") p = sub.add_parser("capture") p.add_argument("title") p.add_argument("file", nargs="?") @@ -698,11 +781,14 @@ def build_parser() -> argparse.ArgumentParser: p.add_argument("--status", default="") p.add_argument("--aliases", default="") p.add_argument("--source", default="") + p.add_argument("--tags", default="") # extra tags; the kind is always seeded p.add_argument("--date") p.add_argument("--domain", default="business") p.add_argument("--no-log", action="store_true") p.add_argument("--json", action="store_true") # M4: emit a JSON result envelope p.add_argument("--dry-run", action="store_true") # M4: preview the plan, write nothing + p.add_argument("--force", action="store_true") # create despite a duplicate-gate hit + p.add_argument("--merge-into", metavar="SLUG") # route as an update to this entity return parser @@ -755,7 +841,7 @@ def main(argv: list[str] | None = None) -> int: if args.cmd == "unlock": return cmd_unlock(args.owner) if args.cmd == "scope": - return cmd_scope(args.subcommand, args.text) + return cmd_scope(args.subcommand, args.text, as_json=args.json) if args.cmd == "reflect": import echo_reflect raw = read_body(args.file).decode("utf-8", errors="replace") @@ -766,20 +852,34 @@ def main(argv: list[str] | None = None) -> int: if not isinstance(proposals, list): raise EchoError("reflect: proposals must be a JSON array of objects", 2) return echo_reflect.apply(proposals, confirm=args.apply) + if args.cmd == "triage": + import echo_triage + if args.list_inbox or not args.file: + return echo_triage.list_inbox(as_json=args.json) + raw = read_body(args.file).decode("utf-8", errors="replace") + try: + proposals = json.loads(raw) + except json.JSONDecodeError as exc: + raise EchoError(f"triage: proposals must be a JSON array ({exc})", 2) + if not isinstance(proposals, list): + raise EchoError("triage: proposals must be a JSON array of objects", 2) + return echo_triage.apply(proposals, confirm=args.apply) if args.cmd in ("resolve", "recall", "link", "capture"): import echo_ops # lazy: only the high-level ops pull in the index/link modules if args.cmd == "resolve": return echo_ops.resolve(" ".join(args.mention)) if args.cmd == "recall": - return echo_ops.recall(args.query, limit=args.limit) + return echo_ops.recall(args.query, limit=args.limit, as_json=args.json) if args.cmd == "link": - return echo_ops.link(args.a, args.b) + return echo_ops.link(args.a, args.b, as_json=args.json) return echo_ops.capture( args.kind, args.title, args.file, status_v=args.status, aliases=args.aliases.split(",") if args.aliases else [], sources=args.source.split(",") if args.source else [], + tags=args.tags.split(",") if args.tags else [], date=args.date, domain=args.domain, inbox=args.inbox, no_log=args.no_log, - as_json=args.json, dry_run=args.dry_run) + as_json=args.json, dry_run=args.dry_run, + force=args.force, merge_into=args.merge_into) except EchoError as exc: print(f"echo.py: {exc}", file=sys.stderr) return exc.code diff --git a/echo-memory.plugin.src/skills/echo-memory/scripts/echo_hook_session_start.py b/echo-memory.plugin.src/skills/echo-memory/scripts/echo_hook_session_start.py new file mode 100644 index 0000000..688cf68 --- /dev/null +++ b/echo-memory.plugin.src/skills/echo-memory/scripts/echo_hook_session_start.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python3 +"""echo_hook_session_start.py — SessionStart hook: self-firing cold-start load. + +The two behaviours the skill cares most about (load at session start, reflect at +session end) used to depend entirely on the model remembering to do them — scope +drift and the inbox backlog are the residue of that discipline failing. This hook +makes loading deterministic: on session start it runs the canonical `echo.py load` +and hands the output to the model as additionalContext. + +Contract (a hook must NEVER break a session): + * always exits 0 — any failure degrades to a one-line note or to silence; + * fires only on `startup` / `clear` (resume & compact already carry context); + * NOT CONFIGURED (exit 78) becomes a short instruction to run the first-run flow, + not an error; + * output is capped so a huge vault can't blow up the context window. +""" +from __future__ import annotations + +import contextlib +import io +import json +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +MAX_CONTEXT = 16000 # chars of load output forwarded into the session context + + +def main() -> int: + try: + payload = json.load(sys.stdin) + except Exception: # noqa: BLE001 + payload = {} + if payload.get("source", "startup") not in ("startup", "clear"): + return 0 + + header = ("[echo-memory] Cold-start memory load (SessionStart hook). Reconcile per " + "the echo-memory skill: check inbox depth and confirm the scope still " + "matches this session's request before working.\n\n") + buf = io.StringIO() + try: + import echo + with contextlib.redirect_stdout(buf): + rc = echo.cmd_load() + text = buf.getvalue() + if rc == 78: + context = ("[echo-memory] ECHO is NOT CONFIGURED on this machine. Before " + "doing memory work, ask the operator for their echo-memory config " + "(owner/endpoint/key) and install it via `echo.py config import " + "<file>` — see the skill's First-run section.") + elif rc == 0 and text.strip(): + if len(text) > MAX_CONTEXT: + text = text[:MAX_CONTEXT] + "\n[... load output truncated by the hook ...]" + context = header + text + else: + context = ("[echo-memory] `echo.py load` returned nothing usable " + f"(exit {rc}) — run /echo-doctor if memory seems unavailable.") + except Exception as exc: # noqa: BLE001 — never break session start + context = (f"[echo-memory] SessionStart load skipped ({exc}). The vault may be " + "unreachable; proceed without memory and mention it once if relevant.") + + print(json.dumps({"hookSpecificOutput": { + "hookEventName": "SessionStart", + "additionalContext": context, + }}, ensure_ascii=False)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/echo-memory.plugin.src/skills/echo-memory/scripts/echo_hook_stop.py b/echo-memory.plugin.src/skills/echo-memory/scripts/echo_hook_stop.py new file mode 100644 index 0000000..dc20a35 --- /dev/null +++ b/echo-memory.plugin.src/skills/echo-memory/scripts/echo_hook_stop.py @@ -0,0 +1,118 @@ +#!/usr/bin/env python3 +"""echo_hook_stop.py — Stop hook: one-shot session-end reflection nudge. + +Reflection (/echo-reflect) only captures memory if someone remembers to run it. +This hook watches the first Stop of a *substantive* session and, when nothing has +been reflected or session-logged yet, blocks once with a reminder so the model +runs the reflect flow (which still previews and asks the operator before writing +— the safety gate lives in reflect, not here). + +Guards (a hook must NEVER trap a session in a loop or nag): + * `stop_hook_active` -> exit immediately (loop guard); + * fires at most ONCE per session (marker file under ECHO_STATE_DIR/hook-state); + * skips quiet sessions (< ECHO_REFLECT_MIN_TURNS user turns, default 5); + * skips when the transcript shows reflection / a session log already happened; + * skips when ECHO isn't configured on this machine; + * always exits 0. +""" +from __future__ import annotations + +import json +import os +import re +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +MIN_TURNS = int(os.environ.get("ECHO_REFLECT_MIN_TURNS", "5")) + +REASON = ( + "[echo-memory] Session-end reflection has not run yet. If this session produced " + "durable facts, decisions, commitments, or artifacts worth remembering: run the " + "/echo-reflect flow (extract -> preview -> apply only with the operator's confirm) " + "and write the session log + heartbeat per the echo-memory skill. If nothing " + "durable emerged, simply finish — do not invent memories. " + "(This reminder fires once per session.)" +) + + +def state_marker(session_id: str) -> Path: + base = Path(os.environ.get("ECHO_STATE_DIR") or (Path.home() / ".echo-memory")) + return base / "hook-state" / f"{session_id}.reflect-nudged" + + +def count_user_turns(raw: str) -> int: + turns = 0 + for line in raw.splitlines(): + try: + rec = json.loads(line) + except Exception: # noqa: BLE001 + continue + if rec.get("type") != "user": + continue + content = (rec.get("message") or {}).get("content") + if isinstance(content, str) and content.strip(): + turns += 1 + elif isinstance(content, list) and any( + isinstance(b, dict) and b.get("type") == "text" for b in content): + turns += 1 + return turns + + +def already_reflected(raw: str) -> bool: + """Transcript evidence that reflection or session logging already happened.""" + return ("/echo-reflect" in raw + or re.search(r'echo\.py"?\s+reflect\b', raw) is not None + or "put _agent/sessions/" in raw + or "put _agent/heartbeat/last-session.md" in raw) + + +def main() -> int: + try: + payload = json.load(sys.stdin) + except Exception: # noqa: BLE001 + return 0 + if payload.get("stop_hook_active"): + return 0 + session_id = str(payload.get("session_id") or "unknown") + marker = state_marker(session_id) + if marker.exists(): + return 0 + + try: + import echo_config + if not echo_config.is_configured(echo_config.load()): + return 0 # nothing to reflect into — stay silent + except Exception: # noqa: BLE001 + return 0 + + raw = "" + tp = payload.get("transcript_path") + if tp: + try: + raw = Path(tp).read_text(encoding="utf-8", errors="replace") + except Exception: # noqa: BLE001 + raw = "" + + if already_reflected(raw): + try: # done for this session — never fire + marker.parent.mkdir(parents=True, exist_ok=True) + marker.write_text("reflected\n", encoding="utf-8") + except Exception: # noqa: BLE001 + pass + return 0 + if count_user_turns(raw) < MIN_TURNS: + return 0 # quiet so far; a later stop may qualify (no marker written) + + try: + marker.parent.mkdir(parents=True, exist_ok=True) + marker.write_text("nudged\n", encoding="utf-8") + except Exception: # noqa: BLE001 + pass # if the marker can't be written, still nudge; stop_hook_active guards loops + print(json.dumps({"decision": "block", "reason": REASON}, ensure_ascii=False)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/echo-memory.plugin.src/skills/echo-memory/scripts/echo_index.py b/echo-memory.plugin.src/skills/echo-memory/scripts/echo_index.py index 7d817bb..4dc03e4 100644 --- a/echo-memory.plugin.src/skills/echo-memory/scripts/echo_index.py +++ b/echo-memory.plugin.src/skills/echo-memory/scripts/echo_index.py @@ -55,6 +55,31 @@ KIND_TYPE = { DATE_KINDS = {"meeting", "decision"} # filename carries a YYYY-MM-DD prefix AREA_DOMAINS = ("business", "personal", "learning", "systems") +# kind -> default `status:` stamped by capture when none is given. Every entity note is +# born with a status — a missing key is the #1 source of frontmatter drift (18/71 notes +# in the field audit). Records of past events default to done; living notes to active. +KIND_STATUS = { + "person": "active", "company": "active", "concept": "active", + "reference": "active", "meeting": "done", "project": "active", + "area": "active", "semantic": "active", "episodic": "done", + "working": "active", "skill": "active", "decision": "done", +} + +# kind -> frontmatter fields that must be present AND non-empty on that kind's notes. +# Data-driven so vault_lint's incomplete-frontmatter check and sweep's backfill stay in +# lockstep with what capture stamps; add a (kind, field) pair here and all three follow. +KIND_REQUIRED_FM = { + "person": ("status", "tags"), + "company": ("status", "tags"), + "concept": ("status", "tags"), + "reference": ("status", "tags"), + "project": ("status", "tags"), + "area": ("status", "tags"), + "skill": ("status", "tags"), + "meeting": ("tags",), + "decision": ("tags",), +} + def slugify(text: str) -> str: text = str(text).strip().lower() diff --git a/echo-memory.plugin.src/skills/echo-memory/scripts/echo_ops.py b/echo-memory.plugin.src/skills/echo-memory/scripts/echo_ops.py index ddf1554..987015c 100644 --- a/echo-memory.plugin.src/skills/echo-memory/scripts/echo_ops.py +++ b/echo-memory.plugin.src/skills/echo-memory/scripts/echo_ops.py @@ -12,6 +12,7 @@ Network goes through echo.py; routing/aliases through echo_index; links through from __future__ import annotations import json +import os import re import sys from pathlib import Path @@ -21,6 +22,11 @@ import echo # noqa: E402 import echo_index as idx_mod # noqa: E402 import echo_links as links # noqa: E402 +# A fuzzy candidate scoring at/above this blocks `capture` from creating what is very +# likely a duplicate (require --force to create anyway, or --merge-into to update the +# existing entity). Below the gate, candidates are surfaced as a warning only. +DUP_GATE = float(os.environ.get("ECHO_DUP_GATE", "0.6")) + # Existing-entity capture appends a dated bullet under this per-kind heading. LOG_HEADING = { "project": "Session History", "person": "Log", "company": "Log", @@ -53,19 +59,25 @@ def resolve(mention: str) -> int: # ------------------------------------------------------------------- link ----- -def link(a_path: str, b_path: str) -> int: +def link(a_path: str, b_path: str, as_json: bool = False) -> int: a_changed, b_changed = links.link_bidirectional(a_path, b_path) + if as_json: + import echo_output + env = echo_output.envelope("link", {"a": a_path, "b": b_path, + "a_changed": a_changed, "b_changed": b_changed}) + print(json.dumps(env, ensure_ascii=False)) + return 0 print(f"ok: linked {a_path} <-> {b_path} " f"(added: {'A' if a_changed else '-'}{'B' if b_changed else '-'})") return 0 # ----------------------------------------------------------------- recall ----- -def recall(query, limit: int = 8) -> int: +def recall(query, limit: int = 8, as_json: bool = False) -> int: """Hybrid lexical (BM25) + graph recall — implemented in echo_recall. Kept here as the stable public entrypoint (echo.py and /echo-recall call echo_ops.recall).""" import echo_recall # lazy: only pulls the BM25 modules when recall is actually run - return echo_recall.recall(query, limit=limit) + return echo_recall.recall(query, limit=limit, as_json=as_json) # ----------------------------------------------------------- agent log ------- @@ -101,12 +113,16 @@ def ensure_daily_log(line: str) -> None: # ----------------------------------------------------------------- capture --- def _build_note(fm_type: str, status_v: str, today_s: str, title: str, - body_text: str, sources: list[str], aliases: list[str] | None = None) -> str: + body_text: str, sources: list[str], aliases: list[str] | None = None, + tags: list[str] | None = None) -> str: src = "[" + ", ".join(json.dumps(s) for s in sources) + "]" fm = ["---", f"type: {fm_type}"] if status_v: fm.append(f"status: {status_v}") - fm += [f"created: {today_s}", f"updated: {today_s}", "tags: []"] + # Never scaffold an empty `tags: []` — an entity note is born complete (the caller + # seeds the kind as a baseline tag), so the incomplete-frontmatter lint stays quiet. + tags_v = "[" + ", ".join(tags or []) + "]" + fm += [f"created: {today_s}", f"updated: {today_s}", f"tags: {tags_v}"] if aliases: # Obsidian-native, durable home for aliases; sweep folds these back into the index. fm.append("aliases: [" + ", ".join(json.dumps(a) for a in aliases) + "]") @@ -118,6 +134,19 @@ def _build_note(fm_type: str, status_v: str, today_s: str, title: str, return out +def _dated_block(today_s: str, body_text: str) -> tuple[str, str]: + """Render a capture body as a dated log entry that PRESERVES the whole body: + the first line rides on the bullet, every further line is indented under it as + a markdown continuation. Returns (bullet, full_block) — the bullet alone is the + idempotency key (same first-line-per-day semantics as before).""" + lines = [ln.rstrip() for ln in body_text.strip().splitlines()] + bullet = f"- {today_s}: {lines[0] if lines else '(update)'}" + block = bullet + for ln in lines[1:]: + block += "\n" + (f" {ln}" if ln else "") + return bullet, block + + def _append_to_existing(path: str, kind: str, today_s: str, body_text: str) -> None: text = links.get_text(path) or "" h1 = links.first_h1(text) or path.rsplit("/", 1)[-1][:-3] @@ -125,22 +154,22 @@ def _append_to_existing(path: str, kind: str, today_s: str, body_text: str) -> N if not re.search(rf"(?m)^##\s+{re.escape(heading)}\s*$", text): echo.request("POST", echo.vault_url(path), data=f"\n\n## {heading}\n".encode(), headers={"Content-Type": "text/markdown"}) - firstline = body_text.strip().splitlines()[0] if body_text.strip() else "(update)" - bullet = f"- {today_s}: {firstline}" + bullet, block = _dated_block(today_s, body_text) status, body = echo.request("GET", echo.vault_url(path)) if status == 200 and bullet in body.decode(errors="replace"): return echo.request("PATCH", echo.vault_url(path), - data=echo.normalize_patch_body((bullet + "\n").encode(), "append", "heading"), + data=echo.normalize_patch_body((block + "\n").encode(), "append", "heading"), headers={"Operation": "append", "Target-Type": "heading", "Target": f"{h1}::{heading}", "Content-Type": "text/markdown"}) echo.cmd_fm(path, "updated", json.dumps(today_s)) def capture(kind: str | None, title: str, file_arg: str | None, status_v: str = "", - aliases=None, sources=None, date: str | None = None, domain: str = "business", - inbox: bool = False, no_log: bool = False, as_json: bool = False, - dry_run: bool = False) -> int: + aliases=None, sources=None, tags=None, date: str | None = None, + domain: str = "business", inbox: bool = False, no_log: bool = False, + as_json: bool = False, dry_run: bool = False, force: bool = False, + merge_into: str | None = None) -> int: import contextlib import io import echo_output @@ -170,6 +199,7 @@ def capture(kind: str | None, title: str, file_arg: str | None, status_v: str = today_s = echo.today() aliases = [a.strip() for a in (aliases or []) if a.strip()] sources = [s.strip() for s in (sources or []) if s.strip()] + tags = [t.strip() for t in (tags or []) if t.strip()] # Unknown home -> defer to the inbox (a single idempotent capture line). if inbox or not kind: @@ -185,15 +215,55 @@ def capture(kind: str | None, title: str, file_arg: str | None, status_v: str = slug = idx_mod.slugify(title) index = idx_mod.load() match_slug, existing = idx_mod.resolve(index, title) + # --merge-into: the operator has already identified the canonical entity (e.g. after + # a duplicate-gate stop) — route this capture as an UPDATE to it, whatever the title. + if merge_into and not existing: + match_slug, existing = idx_mod.resolve(index, merge_into) + if not existing: + print(f"echo_ops: --merge-into '{merge_into}' matches no entity in the index " + f"(try `resolve` first)", file=sys.stderr) + return 2 existing_reachable = bool(existing and echo.request("GET", echo.vault_url(existing["path"]))[0] == 200) + # Pre-write duplicate gate: a strong fuzzy candidate means this title is very likely + # an existing entity under a different name. STOP before creating the duplicate — + # after the fact, the warning arrives too late (the parallel note already exists). + gate_hits = [] + if not existing_reachable and not force and not dry_run: + gate_hits = [{"slug": s, "path": c.get("path"), "title": c.get("title"), + "kind": c.get("kind"), "score": sc} + for s, c, sc in idx_mod.fuzzy_candidates(index, title) + if sc >= DUP_GATE] + if gate_hits: + if as_json: + env = echo_output.envelope("duplicate-gate", { + "kind": kind, "title": title, "candidates": gate_hits, + "note": "likely duplicate — re-run with --merge-into <slug> to update the " + "existing entity, or --force to create anyway"}, ok=False) + print(json.dumps(env, ensure_ascii=False), file=real_stdout) + else: + print(f"STOP: '{title}' likely already exists — not creating a duplicate.", + file=real_stdout) + for c in gate_hits: + print(f" candidate: {c['slug']} -> {c['path']} (score {c['score']})", + file=real_stdout) + print(" re-run with --merge-into <slug> to update the existing entity, " + "or --force to create anyway.", file=real_stdout) + return 76 # distinct exit: duplicate gate (cf. 75 lock, 78 config) + if dry_run: if existing_reachable: return done("update", existing["path"], dry=True) s2 = echo_quality.safe_slug(set(index.get("entities", {}).keys()), slug) - near = [c.get("path") for _, c, _ in idx_mod.fuzzy_candidates(index, title)][:3] - return done("create", idx_mod.derive_path(kind, s2, date=date, domain=domain), + cands = idx_mod.fuzzy_candidates(index, title) + near = [c.get("path") for _, c, _ in cands][:3] + plan = done("create", idx_mod.derive_path(kind, s2, date=date, domain=domain), dry=True, near=near) + if not as_json and not force and any(sc >= DUP_GATE for _, _, sc in cands): + # (in --json mode the near_duplicates field carries this; keep stdout clean) + print("note: a real run would STOP at the duplicate gate (candidate score " + f">= {DUP_GATE}) — use --merge-into or --force.", file=real_stdout) + return plan near_dupes: list[str] = [] index_title = title # title to record in the index entry @@ -213,18 +283,24 @@ def capture(kind: str | None, title: str, file_arg: str | None, status_v: str = _append_to_existing(path, kind, today_s, body_text) action = "updated" else: - if not status_v and kind == "project": - status_v = "active" + if not status_v: + # Every entity note is born with a status — a kind-appropriate default + # (KIND_STATUS) instead of the old project-only special case. Missing + # status was the root cause of the frontmatter-completeness drift. + status_v = idx_mod.KIND_STATUS.get(kind, "active") # Surface (don't silently create alongside) an existing entity this resembles — - # the duplication trap when a shortened name didn't resolve exactly. + # the duplication trap when a shortened name didn't resolve exactly. (Strong + # candidates already stopped at the duplicate gate above; these are the weak + # ones, surfaced as a warning.) near_dupes = [c.get("path") for _, c, _ in idx_mod.fuzzy_candidates(index, title)][:3] # M2: don't let a 40-char-truncated slug silently collide with a different # entity already in the index — disambiguate to slug-2, slug-3, ... slug = echo_quality.safe_slug(set(index.get("entities", {}).keys()), slug) path = idx_mod.derive_path(kind, slug, date=date, domain=domain) note_aliases = sorted((set(index_aliases) | set(idx_mod.derive_aliases(title))) - {slug}) + note_tags = sorted({kind, *tags}) # baseline `- <kind>` tag; --tags enriches note = _build_note(idx_mod.KIND_TYPE.get(kind, kind), status_v, today_s, title, - body_text, sources, note_aliases) + body_text, sources, note_aliases, tags=note_tags) echo.cmd_put(path, echo.temp_file(note.encode())) action = "created" diff --git a/echo-memory.plugin.src/skills/echo-memory/scripts/echo_recall.py b/echo-memory.plugin.src/skills/echo-memory/scripts/echo_recall.py index 93bbed8..13ecfba 100644 --- a/echo-memory.plugin.src/skills/echo-memory/scripts/echo_recall.py +++ b/echo-memory.plugin.src/skills/echo-memory/scripts/echo_recall.py @@ -20,15 +20,28 @@ DESIGN (bounded by the project's pure-stdlib, no-deps constraint): * Resilience : if the index can't be built (vault unreachable), recall falls back to the server's /search/simple so it degrades instead of dying. -Corpus = the indexable ENTITY notes (idx_mod.kind_for_path is not None): people, +Corpus = the indexable ENTITY notes (idx_mod.kind_for_path is not None) — people, companies, concepts, references, meetings, projects, areas, decisions, semantic/ -episodic memory, skills — the durable memory graph. (Sessions/journal are a future -add; tracked in ROADMAP-1.0.md.) +episodic memory, skills — PLUS the time-series notes where conversation-derived +context actually lives: session logs (`_agent/sessions/`) and journal notes +(`journal/daily|weekly|monthly|quarterly|annual/`). Time-series docs carry a +per-kind DOWN-WEIGHT so entity notes still rank first for the same lexical match. + +RANKING (v1.5): final score = BM25 * kind_weight * freshness * status_factor — + * kind_weight : 1.0 entity · 0.5 session · 0.4 journal (KIND_WEIGHT) + * freshness : half-life decay on `updated:` (or the filename date), floored at + FRESH_FLOOR so old notes are demoted, never buried + * status : `active` boosted, `archived` demoted (STATUS_FACTOR) +so a stale archived project no longer outranks the live one on raw term frequency. +Doc metadata rides in the index (schema 2); a schema-1 index is discarded and +rebuilt on the next recall/sweep. """ from __future__ import annotations +import datetime as _dt import json import math +import os import re import sys import urllib.parse @@ -41,6 +54,7 @@ import echo_index as idx_mod # noqa: E402 import echo_links as links # noqa: E402 RECALL_INDEX_PATH = "_agent/index/recall-index.json" +INDEX_SCHEMA = 2 # bumped: schema 2 adds per-doc meta (weight/updated/status) # Tuning knobs. Defaults are standard BM25 (k1/b) + a 0.6 hop decay over <=2 hops. K1 = 1.5 @@ -49,6 +63,63 @@ MAX_HOPS = 2 GRAPH_DECAY = 0.6 MIN_SCORE = 0.0 # relevance floor; raise to trim weak lexical hits +# --- rank-fusion priors ------------------------------------------------------- +# Time-series notes are in the corpus but down-weighted: they are where "what did we +# decide three weeks ago" lives, yet the entity note should win a tie on the same terms. +_TIME_SERIES = [ + (re.compile(r"^_agent/sessions/"), "session"), + (re.compile(r"^journal/daily/"), "daily"), + (re.compile(r"^journal/(weekly|monthly|quarterly|annual)/"), "rollup"), +] +KIND_WEIGHT = {"session": 0.5, "daily": 0.4, "rollup": 0.4} +STATUS_FACTOR = {"active": 1.1, "on-hold": 0.9, "archived": 0.75} +FRESH_HALF_LIFE = float(os.environ.get("ECHO_FRESH_HALF_LIFE", "90")) # days +FRESH_FLOOR = 0.6 # freshness multiplier never drops below this — demote, don't bury + +_FM_UPDATED = re.compile(r"(?m)^updated:\s*[\"']?(\d{4}-\d{2}-\d{2})") +_FM_STATUS = re.compile(r"(?m)^status:\s*[\"']?([A-Za-z-]+)") +_PATH_DATE = re.compile(r"(\d{4}-\d{2}-\d{2})") + + +def _series_kind(path: str) -> str | None: + for rx, kind in _TIME_SERIES: + if rx.match(path): + return kind + return None + + +def doc_meta(path: str, raw_text: str) -> dict: + """Per-doc ranking priors, extracted once at index time: kind weight, last-updated + date (frontmatter `updated:`, else the date in the filename), and `status:`.""" + kind = _series_kind(path) + w = KIND_WEIGHT.get(kind, 1.0) if kind else 1.0 + head = raw_text[:2000] + mu = _FM_UPDATED.search(head) + updated = mu.group(1) if mu else None + if updated is None: + mp = _PATH_DATE.search(path.rsplit("/", 1)[-1]) + updated = mp.group(1) if mp else None + ms = _FM_STATUS.search(head) + meta = {"w": w} + if updated: + meta["u"] = updated + if ms: + meta["s"] = ms.group(1) + return meta + + +def freshness(updated: str | None, today_s: str) -> float: + """Half-life decay on document age, floored. Unknown age is neutral (1.0).""" + if not updated: + return 1.0 + try: + age = (_dt.date.fromisoformat(today_s) - _dt.date.fromisoformat(updated)).days + except ValueError: + return 1.0 + if age <= 0: + return 1.0 + return FRESH_FLOOR + (1.0 - FRESH_FLOOR) * (0.5 ** (age / FRESH_HALF_LIFE)) + _WORD = re.compile(r"[a-z0-9]+") # Minimal English stoplist — keeps BM25 idf from being dominated by glue words. _STOP = frozenset( @@ -73,12 +144,15 @@ def strip_frontmatter(text: str) -> str: # --------------------------------------------------------------------- BM25 core class Bm25Index: - """A compact, serializable BM25 index. add()/remove() are idempotent per path.""" + """A compact, serializable BM25 index with per-doc ranking meta. + add()/remove() are idempotent per path. add() takes the RAW note text — it strips + frontmatter itself and extracts the doc meta (weight/updated/status) in one pass.""" def __init__(self) -> None: self.df: Counter[str] = Counter() # term -> #docs containing it self.postings: dict[str, dict[str, int]] = {} # term -> {doc_path: tf} self.length: dict[str, int] = {} # doc_path -> token count + self.meta: dict[str, dict] = {} # doc_path -> {w, u, s} priors self.n_docs = 0 self.avg_len = 0.0 @@ -86,11 +160,12 @@ class Bm25Index: self.n_docs = len(self.length) self.avg_len = (sum(self.length.values()) / self.n_docs) if self.n_docs else 0.0 - def add(self, path: str, text: str) -> None: + def add(self, path: str, raw_text: str) -> None: if path in self.length: # re-index in place (idempotent) self.remove(path) - toks = tokenize(text) + toks = tokenize(strip_frontmatter(raw_text)) self.length[path] = len(toks) + self.meta[path] = doc_meta(path, raw_text) for term, tf in Counter(toks).items(): self.postings.setdefault(term, {})[path] = tf self.df[term] += 1 @@ -107,9 +182,19 @@ class Bm25Index: del self.postings[term] del self.df[term] del self.length[path] + self.meta.pop(path, None) self._recompute() - def score(self, query: str, limit: int = 10) -> list[tuple[str, float]]: + def _doc_factor(self, path: str, today_s: str) -> float: + """The rank-fusion prior: kind weight x freshness x status factor.""" + m = self.meta.get(path) or {} + return (float(m.get("w", 1.0)) + * freshness(m.get("u"), today_s) + * STATUS_FACTOR.get(m.get("s", ""), 1.0)) + + def score(self, query: str, limit: int = 10, + today_s: str | None = None) -> list[tuple[str, float]]: + today_s = today_s or echo.today() scores: Counter[str] = Counter() for term in set(tokenize(query)): posting = self.postings.get(term) @@ -120,18 +205,23 @@ class Bm25Index: dl = self.length.get(path, 0) denom = tf + K1 * (1 - B + B * (dl / self.avg_len if self.avg_len else 1)) scores[path] += idf * (tf * (K1 + 1) / denom if denom else 0) - return [(p, s) for p, s in scores.most_common(limit) if s > MIN_SCORE] + fused = {p: s * self._doc_factor(p, today_s) for p, s in scores.items()} + ranked = sorted(fused.items(), key=lambda kv: -kv[1])[:limit] + return [(p, s) for p, s in ranked if s > MIN_SCORE] # ---- serialization (compact; rebuildable, so the format can change freely) ---- def to_json(self) -> dict: - return {"schema": 1, "n_docs": self.n_docs, "avg_len": self.avg_len, - "length": self.length, "postings": self.postings} + return {"schema": INDEX_SCHEMA, "n_docs": self.n_docs, "avg_len": self.avg_len, + "length": self.length, "postings": self.postings, "meta": self.meta} @classmethod def from_json(cls, d: dict) -> "Bm25Index": + if d.get("schema") != INDEX_SCHEMA: + return cls() # older/newer format -> empty; recall's cold path rebuilds ix = cls() ix.length = d.get("length", {}) ix.postings = d.get("postings", {}) + ix.meta = d.get("meta", {}) ix.n_docs = d.get("n_docs", len(ix.length)) ix.avg_len = d.get("avg_len", 0.0) ix.df = Counter({term: len(posting) for term, posting in ix.postings.items()}) @@ -172,8 +262,12 @@ def _walk(prefix: str = ""): def _indexable(path: str) -> bool: base = path.rsplit("/", 1)[-1] - return (path.endswith(".md") and idx_mod.kind_for_path(path) is not None - and base not in _SKIP_BASENAMES and not _TEMPLATE_RE.search(path)) + if (not path.endswith(".md") or base in _SKIP_BASENAMES + or _TEMPLATE_RE.search(path)): + return False + # Entity notes (the durable graph) + time-series notes (sessions, journal) — the + # latter join the corpus down-weighted so past decisions/discussions are findable. + return idx_mod.kind_for_path(path) is not None or _series_kind(path) is not None # ------------------------------------------------------------------- persistence @@ -208,7 +302,7 @@ def rebuild(prefetched: dict | None = None) -> Bm25Index: items = ((p, _get(p)) for p in _walk() if _indexable(p)) for path, text in items: if text is not None: - ix.add(path, strip_frontmatter(text)) + ix.add(path, text) # add() strips frontmatter + extracts doc meta itself save_index(ix) return ix @@ -224,7 +318,7 @@ def update_note(path: str, text: str) -> None: import echo_concurrency with echo_concurrency.vault_lock(): ix = load_index() # fresh read inside the lock - ix.add(path, strip_frontmatter(text)) + ix.add(path, text) # raw text: add() strips + extracts meta save_index(ix) except Exception as exc: # noqa: BLE001 — upkeep must never fail a capture print(f"echo_recall: index update skipped ({exc})", file=sys.stderr) @@ -279,37 +373,57 @@ def expand_graph(seeds, nmap: dict, base_scores: dict, max_hops: int = MAX_HOPS) # ------------------------------------------------------------------- presentation +def _doc_summary(path: str, text: str) -> dict: + """Structured per-note summary shared by the human brief and --json output.""" + out: dict = {"path": path} + mtype = re.search(r"(?m)^type:\s*(.+)$", text) + if mtype: + out["type"] = mtype.group(1).strip() + mu = _FM_UPDATED.search(text[:2000]) + if mu: + out["updated"] = mu.group(1) + ms = _FM_STATUS.search(text[:2000]) + if ms: + out["status"] = ms.group(1) + body = strip_frontmatter(text) + status = re.search(r"(?ms)^##\s+Status\s*$(.*?)(?=^##\s|\Z)", body) + if status and status.group(1).strip(): + out["excerpt"] = status.group(1).strip()[:400] + else: + kept = [ln[:200] for ln in body.splitlines() + if ln.strip() and not ln.startswith("# ")][:6] + out["excerpt"] = "\n".join(kept) + return out + + def _brief(path: str, score: float | None = None, via: str | None = None) -> None: text = links.get_text(path) if text is None: return + info = _doc_summary(path, text) head = f"\n### {path}" if score is not None: head += f" (score {score:.2f})" if via: head += f" (via {via})" print(head) - mtype = re.search(r"(?m)^type:\s*(.+)$", text) - if mtype: - print(f"_type: {mtype.group(1).strip()}_") - body = strip_frontmatter(text) - status = re.search(r"(?ms)^##\s+Status\s*$(.*?)(?=^##\s|\Z)", body) - if status and status.group(1).strip(): - print(status.group(1).strip()[:400]) - return - shown = 0 - for line in body.splitlines(): - if not line.strip() or line.startswith("# "): - continue - print(line[:200]) - shown += 1 - if shown >= 6: - break + stamps = [] + if info.get("type"): + stamps.append(f"type: {info['type']}") + if info.get("updated"): + stamps.append(f"updated: {info['updated']}") + if info.get("status"): + stamps.append(f"status: {info['status']}") + if stamps: + print("_" + " · ".join(stamps) + "_") # staleness is visible, not guessed + if info.get("excerpt"): + print(info["excerpt"]) # ------------------------------------------------------------------- entrypoint -def recall(query, limit: int = 8) -> int: +def recall(query, limit: int = 8, as_json: bool = False) -> int: q = " ".join(query) if isinstance(query, list) else query + today_s = echo.today() index = idx_mod.load() nmap = idx_mod.name_map(index) @@ -319,7 +433,7 @@ def recall(query, limit: int = 8) -> int: ix = load_index() if ix.n_docs == 0: ix = rebuild() - lexical = ix.score(q, limit=limit) + lexical = ix.score(q, limit=limit, today_s=today_s) except echo.EchoError as exc: print(f"recall: BM25 index unavailable ({exc}); falling back to server search", file=sys.stderr) @@ -350,6 +464,26 @@ def recall(query, limit: int = 8) -> int: # --- graph layer ---------------------------------------------------------- neighbours = expand_graph(hits, nmap, base, max_hops=MAX_HOPS) + if as_json: + import echo_output + texts = echo.read_many(hits + [p for p, _ in neighbours[: 2 * limit]]) + primary = [] + for p in hits: + if texts.get(p) is None: + continue + primary.append({**_doc_summary(p, texts[p]), + "score": round(base.get(p, 0.0), 3)}) + linked = [] + for p, (sc, via) in neighbours[: 2 * limit]: + if texts.get(p) is None: + continue + linked.append({**_doc_summary(p, texts[p]), + "score": round(sc, 3), "via": via}) + env = echo_output.envelope("recall", {"query": q, "primary": primary, + "linked": linked}) + print(json.dumps(env, ensure_ascii=False)) + return 0 + print(f"== recall: {q} ==") print(f"\n# Primary hits ({len(hits)})") for p in hits: diff --git a/echo-memory.plugin.src/skills/echo-memory/scripts/echo_reflect.py b/echo-memory.plugin.src/skills/echo-memory/scripts/echo_reflect.py index 380a241..f7ffbc0 100644 --- a/echo-memory.plugin.src/skills/echo-memory/scripts/echo_reflect.py +++ b/echo-memory.plugin.src/skills/echo-memory/scripts/echo_reflect.py @@ -20,6 +20,7 @@ PROPOSAL_SCHEMA (one per durable item the agent wants to remember): "kind": "person", # required unless "inbox": true "body": "Principal at Acme; ...", # markdown body (optional) "aliases": ["bob", "rs"], # optional + "tags": ["client"], # optional; the kind is always seeded as a tag "sources": ["_agent/sessions/..."], # optional backward links "date": "YYYY-MM-DD", # optional (meeting/decision kinds) "domain": "business", # optional (area kind) @@ -121,7 +122,7 @@ def apply(proposals: list[dict], confirm: bool = False) -> int: return 0 import echo_ops # lazy: the apply path pulls in the capture/index/link stack - applied = 0 + applied = gated = 0 for p in valid: if p.get("_action") == "error": continue @@ -129,8 +130,12 @@ def apply(proposals: list[dict], confirm: bool = False) -> int: rc = echo_ops.capture( p.get("kind"), p["title"], bodyfile, aliases=p.get("aliases") or [], sources=p.get("sources") or [], + tags=p.get("tags") or [], date=p.get("date"), domain=p.get("domain", "business"), inbox=bool(p.get("inbox")) or not p.get("kind")) applied += 1 if rc == 0 else 0 - print(f"reflect: applied {applied}/{len(valid)} proposal(s).") + gated += 1 if rc == 76 else 0 + print(f"reflect: applied {applied}/{len(valid)} proposal(s)." + + (f" {gated} stopped at the duplicate gate — re-propose with the existing " + f"entity's title (or capture --merge-into <slug>)." if gated else "")) return 0 diff --git a/echo-memory.plugin.src/skills/echo-memory/scripts/echo_triage.py b/echo-memory.plugin.src/skills/echo-memory/scripts/echo_triage.py new file mode 100644 index 0000000..3f5d652 --- /dev/null +++ b/echo-memory.plugin.src/skills/echo-memory/scripts/echo_triage.py @@ -0,0 +1,133 @@ +#!/usr/bin/env python3 +"""echo_triage.py — one-tap inbox triage, built on the reflect pipeline. + +Triage used to be pure prose: the agent hand-routed each inbox line with individual +PATCH/PUT calls and hand-wrote the processing log — which is why the inbox rots. +But the machinery it needs already exists in echo_reflect (validate -> classify +against the entity index -> preview -> apply via capture). This module reuses it +and adds the two triage-specific pieces: + + * `list_inbox` — parse `inbox/captures/inbox.md` into structured capture lines + (date, text, age in days) so the model builds proposals from + data, not by re-reading prose; + * `apply` — same contract as reflect (dry-run unless confirm), plus an + audit line in `inbox/processing-log/YYYY-MM-DD.md` for every + routed item. Originals are NEVER deleted (operating contract: + the processing log is the audit trail, deletion is explicit). + +Proposals are reflect's PROPOSAL_SCHEMA plus an optional `"line"` — the original +inbox line, echoed into the processing log so the audit trail maps 1:1. +""" +from __future__ import annotations + +import json +import re +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import echo # noqa: E402 + +INBOX_PATH = "inbox/captures/inbox.md" +LOG_DIR = "inbox/processing-log" + +_CAPTURE_LINE = re.compile(r"^\s*-\s*(\d{4}-\d{2}-\d{2})\s*:?\s*(.*\S)\s*$") + + +def parse_inbox(text: str) -> list[dict]: + """Dated capture bullets -> [{line, date, text, age_days}] (age from ECHO_TODAY).""" + import datetime as dt + try: + today = dt.date.fromisoformat(echo.today()) + except ValueError: + today = dt.date.today() + items = [] + for raw_line in text.splitlines(): + m = _CAPTURE_LINE.match(raw_line) + if not m: + continue + try: + age = (today - dt.date.fromisoformat(m.group(1))).days + except ValueError: + age = None + items.append({"line": raw_line.strip(), "date": m.group(1), + "text": m.group(2), "age_days": age}) + return items + + +def list_inbox(as_json: bool = False) -> int: + status, body = echo.request("GET", echo.vault_url(INBOX_PATH)) + if status == 404: + items = [] + else: + echo.check(status, body, f"triage list {INBOX_PATH}") + items = parse_inbox(body.decode(errors="replace")) + if as_json: + import echo_output + env = echo_output.envelope("triage-list", {"path": INBOX_PATH, "items": items, + "count": len(items)}) + print(json.dumps(env, ensure_ascii=False)) + return 0 + if not items: + print("triage: inbox is empty — nothing to route.") + return 0 + print(f"triage: {len(items)} capture(s) in {INBOX_PATH}") + for it in items: + age = f"{it['age_days']}d" if it["age_days"] is not None else "?" + print(f" [{age:>4}] {it['date']}: {it['text']}") + print("\nBuild a proposals JSON (reflect schema + optional \"line\") and run " + "`echo.py triage <file>` to preview, `--apply` to route.") + return 0 + + +def apply(proposals: list[dict], confirm: bool = False) -> int: + """Validate -> classify -> preview; with confirm, route each via capture AND write + the processing-log audit line. Mirrors echo_reflect.apply's contract exactly.""" + import echo_reflect + valid, errors = echo_reflect.validate(proposals) + for e in errors: + print(f"skip: {e}", file=sys.stderr) + if not valid: + print("triage: no valid proposals to route.") + return 0 + + echo_reflect.classify(valid) + counts = {a: sum(1 for p in valid if p.get("_action") == a) + for a in ("create", "update", "inbox", "error")} + print(f"triage: {len(valid)} proposal(s) — " + f"{counts['create']} new, {counts['update']} update, {counts['inbox']} stay-in-inbox" + + (f", {counts['error']} error" if counts["error"] else "")) + print(echo_reflect.preview(valid)) + + if not confirm: + print("\ntriage: dry-run — re-run with --apply to route these and log the moves.") + return 0 + + import echo_ops + today_s = echo.today() + applied = gated = 0 + for p in valid: + if p.get("_action") == "error": + continue + if p.get("_action") == "inbox": + continue # routing an inbox line back to the inbox is a no-op, not a move + bodyfile = echo.temp_file((p.get("body") or "").encode()) if p.get("body") else None + rc = echo_ops.capture( + p.get("kind"), p["title"], bodyfile, + aliases=p.get("aliases") or [], sources=p.get("sources") or [], + tags=p.get("tags") or [], + date=p.get("date"), domain=p.get("domain", "business")) + if rc == 76: + gated += 1 + continue + if rc != 0: + continue + applied += 1 + original = (p.get("line") or p["title"]).strip() + echo.cmd_append(f"{LOG_DIR}/{today_s}.md", + f"- {original} → {p.get('_path', '?')}") + print(f"triage: routed {applied}/{len(valid)} item(s); audit in {LOG_DIR}/{today_s}.md. " + "Originals kept in the inbox (deletion is explicit-only)." + + (f" {gated} stopped at the duplicate gate — re-propose with the existing " + f"entity's title or use capture --merge-into." if gated else "")) + return 0 diff --git a/echo-memory.plugin.src/skills/echo-memory/scripts/sweep.py b/echo-memory.plugin.src/skills/echo-memory/scripts/sweep.py index c953826..a645790 100644 --- a/echo-memory.plugin.src/skills/echo-memory/scripts/sweep.py +++ b/echo-memory.plugin.src/skills/echo-memory/scripts/sweep.py @@ -8,10 +8,14 @@ backfill what older vaults don't have yet: 2. **(re)build the entity index** (`_agent/index/entities.json`) from the notes already in the vault (people, companies, projects, concepts, references, meetings, areas, decisions, semantic/episodic memory, skills), - 3. **symmetrize cross-links** — for every `## Related` link A -> B, ensure the + 3. **backfill incomplete entity frontmatter** — fill a missing/empty `status:` with + the kind default and seed a missing/empty `tags:` with the note's kind, per the + KIND_REQUIRED_FM/KIND_STATUS maps in echo_index (what `/echo-health` flags as + `incomplete-frontmatter`), + 4. **symmetrize cross-links** — for every `## Related` link A -> B, ensure the reciprocal B -> A exists (it only adds the missing direction; it never invents new links from body text), and - 4. stamp the marker `schema_version` to the current schema (4). + 5. stamp the marker `schema_version` to the current schema (4). Dry-run by default; pass --apply to write. READ-ONLY without --apply. Cross-platform: pure Python via echo.py. @@ -43,6 +47,30 @@ SKIP_BASENAMES = {"README.md"} kind_for = idx_mod.kind_for_path +def fm_block(text: str) -> str: + if not text.startswith("---"): + return "" + end = text.find("\n---", 3) + return text[:end] if end != -1 else text + + +def fm_populated(text: str, field: str) -> bool: + """True when the frontmatter carries a real value for `field` (flow list `[x]`, + block list items, or a non-empty scalar). Mirrors vault_lint.fm_field_populated.""" + fm = fm_block(text) + m = re.search(rf"(?m)^{re.escape(field)}:[ \t]*(.*)$", fm) + if not m: + return False + val = m.group(1).strip() + if val == "[]": + return False + if val: + return True + rest = fm[m.end():].lstrip("\n") + first = rest.splitlines()[0] if rest else "" + return bool(re.match(r"^\s+-\s*\S", first)) + + def get(path: str) -> str | None: status, body = echo.request("GET", echo.vault_url(path)) if status == 404: @@ -139,9 +167,36 @@ def main(argv: list[str] | None = None) -> int: print(f"sweep: {tag} recall index — {rix.n_docs} docs (BM25)") else: n_idx = sum(1 for p in all_files if echo_recall._indexable(p)) - print(f"sweep: {tag} recall index — would rebuild BM25 over {n_idx} entity notes") + print(f"sweep: {tag} recall index — would rebuild BM25 over {n_idx} corpus notes " + f"(entities + sessions/journal)") - # ---- 3. symmetrize existing Related links -------------------------------- + # ---- 3. backfill incomplete entity frontmatter ---------------------------- + # What /echo-health flags as incomplete-frontmatter, filled mechanically: a missing + # or empty status gets the kind default; missing/empty tags get the kind as a + # baseline tag. cmd_fm is create-or-replace, so absent keys are inserted surgically. + fixes: list[tuple[str, str, str]] = [] + for path in all_files: + kind = kind_for(path) + if kind is None: + continue + text = texts.get(path) or "" + if not text.startswith("---"): + continue # no frontmatter block at all — lint flags it; not a mechanical fill + for field in idx_mod.KIND_REQUIRED_FM.get(kind, ()): + if not fm_populated(text, field): + value = (idx_mod.KIND_STATUS.get(kind, "active") if field == "status" + else json.dumps([kind])) + fixes.append((path, field, value)) + print(f"sweep: {tag} frontmatter backfill — {len(fixes)} field(s) to fill") + for path, field, value in fixes[:40]: + print(f" {path}: {field} -> {value}") + if len(fixes) > 40: + print(f" ... and {len(fixes) - 40} more") + if apply: + for path, field, value in fixes: + echo.cmd_fm(path, field, value) + + # ---- 4. symmetrize existing Related links -------------------------------- fileset = set(all_files) basemap: dict[str, str] = {} for p in all_files: @@ -176,7 +231,7 @@ def main(argv: list[str] | None = None) -> int: for tp, sp in missing: links.add_one(tp, sp) - # ---- 4. stamp schema ------------------------------------------------------ + # ---- 5. stamp schema ------------------------------------------------------ marker = get("_agent/echo-vault.md") or "" cur = next((int(m) for m in re.findall(r"(?m)^schema_version:\s*(\d+)", marker)), 0) if cur < CURRENT_SCHEMA: diff --git a/echo-memory.plugin.src/skills/echo-memory/scripts/vault_lint.py b/echo-memory.plugin.src/skills/echo-memory/scripts/vault_lint.py index 97daa6c..67c67c0 100644 --- a/echo-memory.plugin.src/skills/echo-memory/scripts/vault_lint.py +++ b/echo-memory.plugin.src/skills/echo-memory/scripts/vault_lint.py @@ -120,6 +120,22 @@ def as_list(value: object) -> list[object]: return value if isinstance(value, list) else [value] +def fm_field_populated(raw: str, fields: dict[str, object], field: str) -> bool: + """True when `field` exists AND holds a real value. parse_frontmatter is line-based, + so a block-style list (`tags:\n - x`) parses as "" — check the raw text for items + before calling the field empty.""" + value = fields.get(field) + if isinstance(value, list): + return bool(value) + if str(value or "").strip(): + return True + lines = raw.splitlines() + for index, line in enumerate(lines): + if re.match(rf"^{re.escape(field)}:\s*$", line): + return index + 1 < len(lines) and bool(re.match(r"^\s+-\s*\S", lines[index + 1])) + return False + + def route_matchers(): routing = json.loads((SCRIPT_DIR / "routing.json").read_text(encoding="utf-8")) routes = [(item["id"], re.compile(item["pattern"])) for item in routing.get("routes", [])] @@ -171,6 +187,15 @@ def main() -> int: missing = [k for k in REQUIRED_FM if not str(fm.get(k, "")).strip()] if fm and missing: flag("missing-frontmatter", f"{path}: missing {', '.join(missing)}") + # Kind-scoped completeness: entity notes must classify (status + tags). The + # per-kind requirement map lives in echo_index (KIND_REQUIRED_FM) so capture's + # defaults, this check, and sweep's backfill all read the same source of truth. + kind = idx_mod.kind_for_path(path) + if fm and kind: + for field in idx_mod.KIND_REQUIRED_FM.get(kind, ()): + if not fm_field_populated(raw, fm, field): + what = f"missing {field}" if field not in fm else f"empty {field}" + flag("incomplete-frontmatter", f"{path}: {what}") created, updated = parse_date(fm.get("created")), parse_date(fm.get("updated")) if created and updated and updated < created: flag("date-order", f"{path}: updated {updated} is before created {created}") @@ -307,6 +332,7 @@ def main() -> int: "unknown-path": "Path matches no route in routing.json", "retired-path": "Write to a retired/dead path", "missing-frontmatter": "Missing required frontmatter field", + "incomplete-frontmatter": "Incomplete entity frontmatter (status/tags) — sweep.py --apply backfills", "date-order": "updated earlier than created", "future-date": "updated date is in the future", "source-notes-wikilink": "Wikilink in source_notes (must be plain paths)", diff --git a/eval/README.md b/eval/README.md index bf6e868..d0ccab1 100644 --- a/eval/README.md +++ b/eval/README.md @@ -5,7 +5,20 @@ recipes from `SKILL.md`) and **after** (the shipped `scripts/echo.py` client) th hardening work. It quantifies the claims in the comparative analysis: token cost of the I/O layer, and the rate of **silent write failures**. -## Run it +## Current test suites (run these for any change) + +The A/B harness below is the historical 0.6-vs-0.7 comparison. The **current-version** +suites live alongside it and are what CI gates on: + +```bash +python3 test_features.py # end-to-end features vs the mock (capture/recall/gate/triage/hooks/…) +python3 test_reflect.py # reflection pipeline +python3 test_offline_queue.py # H2 outage queue + read cache +python3 test_patch_semantics.py # real PATCH semantics vs mock_olrapi_hifi.py (incl. fm create-or-replace) +# plus, in the plugin tree: scripts/test_echo_client.py (offline unit + routing-sync guard) +``` + +## Run it (historical A/B) ```bash cd eval diff --git a/eval/test_features.py b/eval/test_features.py index 8731b99..143dde4 100644 --- a/eval/test_features.py +++ b/eval/test_features.py @@ -170,6 +170,158 @@ def main(): and env.get("path") == "resources/companies/json-co.md", r.stdout) check("M4 --json capture actually wrote the note", h.ground("resources/companies/json-co.md") is not None) + # ---------------- v1.5 features ---------------------------------------- + # step 8 clobbered entities.json on purpose; rebuild the index from the notes + # so the v1.5 tests run against a coherent vault. + h.echo(SWEEP, "--apply") + + # 10. frontmatter completeness at write time: entity notes are born classified. + bob2 = h.ground("resources/people/bob-smith.md") or "" + check("v1.5 capture stamps a default status", "status: active" in bob2, bob2) + check("v1.5 capture seeds tags with the kind", "tags: [person]" in bob2, bob2) + + # 11. update-capture preserves the FULL body (old code kept only line 1). + r = subprocess.run([sys.executable, str(ECHO), "capture", "Bob Smith", "-", "--kind", "person"], + input="met at expo\nsecond line survives\nthird line too\n", + capture_output=True, text=True, + env=dict(os.environ, ECHO_BASE=base, ECHO_KEY=KEY, ECHO_TODAY="2026-06-21")) + bob3 = h.ground("resources/people/bob-smith.md") or "" + check("v1.5 update keeps the dated bullet", "- 2026-06-21: met at expo" in bob3, r.stdout + r.stderr) + check("v1.5 update keeps EVERY body line", + " second line survives" in bob3 and " third line too" in bob3, bob3) + + # 12. duplicate gate: a strong fuzzy candidate STOPS creation (exit 76)... + r = h.echo(ECHO, "capture", "Smith Consulting", "--kind", "company") + check("v1.5 duplicate gate stops the create (exit 76)", r.returncode == 76, str(r.returncode)) + check("v1.5 gate wrote nothing", h.ground("resources/companies/smith-consulting.md") is None) + # ... --force overrides ... + r = h.echo(ECHO, "capture", "Smith Consulting", "--kind", "company", "--force") + check("v1.5 --force creates past the gate", + h.ground("resources/companies/smith-consulting.md") is not None, r.stdout + r.stderr) + # ... and --merge-into routes the capture as an update to the canonical entity. + r = h.echo(ECHO, "capture", "Bobby The Builder", "--kind", "person", "--merge-into", "bob-smith") + idx4 = h.ground("_agent/index/entities.json") or "" + check("v1.5 --merge-into updates the existing entity (mention learned as alias)", + r.returncode == 0 and "Bobby The Builder" in idx4, r.stdout + r.stderr) + check("v1.5 --merge-into did not spawn a new note", + h.ground("resources/people/bobby-the-builder.md") is None) + + # 13. recall corpus now includes session logs (down-weighted, but findable). + h.seed("_agent/sessions/2026-06-15-1200-mpm-review.md", + "---\ntype: session-log\ncreated: 2026-06-15\nupdated: 2026-06-15\n---\n" + "# MPM review\n\nDiscussed the kerfuffle about invoicing.\n") + h.echo(SWEEP, "--apply") + r = h.echo(ECHO, "recall", "kerfuffle") + check("v1.5 recall finds a session log by body term", + "_agent/sessions/2026-06-15-1200-mpm-review.md" in r.stdout, r.stdout) + rix2 = h.ground("_agent/index/recall-index.json") or "" + check("v1.5 recall index carries doc meta (schema 2)", + '"schema": 2' in rix2 or '"schema":2' in rix2, rix2[:200]) + + # 14. recency-aware ranking: same term, fresher note wins. + h.seed("resources/concepts/old-idea.md", + "---\ntype: concept\nstatus: active\ncreated: 2025-01-01\nupdated: 2025-01-01\n" + "tags: [concept]\n---\n# Old Idea\n\nzeppelin research notes\n") + h.seed("resources/concepts/new-idea.md", + "---\ntype: concept\nstatus: active\ncreated: 2026-06-20\nupdated: 2026-06-20\n" + "tags: [concept]\n---\n# New Idea\n\nzeppelin research notes\n") + h.echo(SWEEP, "--apply") + r = h.echo(ECHO, "recall", "zeppelin", "--json") + try: + env2 = json.loads(r.stdout) + except Exception: + env2 = {} + primary = env2.get("primary") or [] + check("v1.5 recall --json emits structured hits", + env2.get("ok") is True and len(primary) >= 2, r.stdout[:300]) + check("v1.5 fresher note outranks stale twin", + primary and primary[0].get("path") == "resources/concepts/new-idea.md", + json.dumps(primary[:2])) + check("v1.5 recall hits carry updated/status metadata", + primary and primary[0].get("updated") == "2026-06-20" + and primary[0].get("status") == "active", json.dumps(primary[:1])) + + # 15. one-tap triage: structured list, then route with an audit trail. + h.seed("inbox/captures/inbox.md", + "- 2026-06-01: prefers uv over pip\n- 2026-06-20: try the new espresso place\n") + r = h.echo(ECHO, "triage", "--list", "--json") + try: + tenv = json.loads(r.stdout) + except Exception: + tenv = {} + items = tenv.get("items") or [] + check("v1.5 triage --list parses dated captures with ages", + len(items) == 2 and items[0].get("age_days") == 20, r.stdout[:300]) + props = [{"title": "Prefers uv over pip", "kind": "semantic", + "body": "The operator prefers uv over pip for Python tooling.", + "line": "- 2026-06-01: prefers uv over pip", "confidence": 0.9}] + pfile = HERE / "_triage_props.json" + pfile.write_text(json.dumps(props), encoding="utf-8") + r = h.echo(ECHO, "triage", str(pfile), "--apply") + pfile.unlink() + routed = h.ground("_agent/memory/semantic/prefers-uv-over-pip.md") + plog = h.ground("inbox/processing-log/2026-06-21.md") or "" + check("v1.5 triage routes the item via capture", routed is not None, r.stdout + r.stderr) + check("v1.5 triage writes the processing-log audit line", + "prefers uv over pip" in plog and "_agent/memory/semantic/prefers-uv-over-pip.md" in plog, plog) + check("v1.5 triage keeps the original capture", + "prefers uv over pip" in (h.ground("inbox/captures/inbox.md") or "")) + + # 16. lint flags incomplete entity frontmatter; sweep backfills it. + h.seed("resources/companies/driftco.md", + "---\ntype: company\ncreated: 2026-06-01\nupdated: 2026-06-01\ntags: []\n---\n# DriftCo\n") + LINT = SCRIPTS / "vault_lint.py" + r = h.echo(LINT) + check("v1.5 lint flags missing status", "driftco.md: missing status" in r.stdout, r.stdout[-800:]) + check("v1.5 lint flags empty tags", "driftco.md: empty tags" in r.stdout, r.stdout[-800:]) + h.echo(SWEEP, "--apply") + drift = h.ground("resources/companies/driftco.md") or "" + # (the naive mock records frontmatter PATCHes as <fm:...> markers instead of + # editing the YAML; the hi-fi suite proves the real fm semantics) + check("v1.5 sweep backfills status", + "status: active" in drift or '<fm:status="active">' in drift, drift) + check("v1.5 sweep backfills the kind tag", + "tags: [company]" in drift or '<fm:tags=["company"]>' in drift, drift) + + # 17. hooks: stop-hook nudges once on a substantive session, then stays quiet. + import tempfile + tdir = Path(tempfile.mkdtemp()) + transcript = tdir / "t.jsonl" + turns = [json.dumps({"type": "user", "message": {"content": f"user turn {i}"}}) for i in range(6)] + transcript.write_text("\n".join(turns), encoding="utf-8") + hook_env = dict(os.environ, ECHO_BASE=base, ECHO_KEY=KEY, ECHO_STATE_DIR=str(tdir)) + payload = json.dumps({"session_id": "s1", "transcript_path": str(transcript), + "stop_hook_active": False}) + HOOK_STOP = SCRIPTS / "echo_hook_stop.py" + r = subprocess.run([sys.executable, str(HOOK_STOP)], input=payload, + capture_output=True, text=True, env=hook_env) + try: + henv = json.loads(r.stdout) + except Exception: + henv = {} + check("v1.5 stop hook nudges a substantive session", + r.returncode == 0 and henv.get("decision") == "block", r.stdout + r.stderr) + r = subprocess.run([sys.executable, str(HOOK_STOP)], input=payload, + capture_output=True, text=True, env=hook_env) + check("v1.5 stop hook fires only once per session", + r.returncode == 0 and not r.stdout.strip(), r.stdout) + HOOK_START = SCRIPTS / "echo_hook_session_start.py" + r = subprocess.run([sys.executable, str(HOOK_START)], + input=json.dumps({"source": "startup"}), + capture_output=True, text=True, env=hook_env) + try: + senv = json.loads(r.stdout) + except Exception: + senv = {} + ctx = (senv.get("hookSpecificOutput") or {}).get("additionalContext", "") + check("v1.5 session-start hook injects the load output", + r.returncode == 0 and "marker" in ctx and "echo-vault.md" in ctx, r.stdout[:300]) + r = subprocess.run([sys.executable, str(HOOK_START)], + input=json.dumps({"source": "resume"}), + capture_output=True, text=True, env=hook_env) + check("v1.5 session-start hook stays quiet on resume", + r.returncode == 0 and not r.stdout.strip(), r.stdout) + print(f"\n{len(failures)} failure(s)" if failures else "\nall feature tests passed") return 1 if failures else 0 finally: diff --git a/eval/test_patch_semantics.py b/eval/test_patch_semantics.py index e9d7489..e57d336 100644 --- a/eval/test_patch_semantics.py +++ b/eval/test_patch_semantics.py @@ -107,9 +107,22 @@ def main(): g = ground(doc) or "" check("frontmatter replace updates existing field", "updated: 2026-06-22" in g, g) - # frontmatter replace on a MISSING field -> 400 (matches real API). + # fm on a MISSING field: the server still 400s the raw PATCH (matches real API), + # but cmd_fm (v1.5) falls back to a surgical create — the key is inserted and + # every other line is preserved. (The old contract failed loud on exactly the + # notes that needed repair.) + before = ground(doc) or "" r = echo("fm", doc, "nonexistent_field", "x") - check("frontmatter replace of a missing field fails loud", r.returncode != 0, r.stderr) + check("fm creates a missing field (create-or-replace)", r.returncode == 0, + r.stderr or r.stdout) + g = ground(doc) or "" + check("fm-created key holds the value", "nonexistent_field: x" in g, g) + check("fm create preserves every existing line", + all(line in g for line in before.splitlines() if line.strip()), g) + + # raw `patch replace frontmatter` (no fallback) still fails loud on a missing key. + r = echo("patch", doc, "replace", "frontmatter", "still_missing", tmp('"x"')) + check("raw frontmatter PATCH of a missing field fails loud", r.returncode != 0, r.stderr) print(f"\n{len(failures)} failure(s)" if failures else "\nall PATCH-semantics tests passed") return 1 if failures else 0