testing and documentation
This commit is contained in:
Jason Stedwell
@@ -0,0 +1,931 @@
|
||||
{
|
||||
"env": {
|
||||
"date": "2026-06-23T20:17:09",
|
||||
"endpoint": "https://echoapi.alwisp.com",
|
||||
"commit": "af16598",
|
||||
"echo_workers": 8,
|
||||
"echo_timeout": 30,
|
||||
"python": "3.10.12",
|
||||
"iterations": 4,
|
||||
"count_k": 50
|
||||
},
|
||||
"metrics": {
|
||||
"bulk-append": {
|
||||
"result": {
|
||||
"k": 30,
|
||||
"append_write": {
|
||||
"n": 30,
|
||||
"min_ms": 87.11,
|
||||
"median_ms": 104.08,
|
||||
"p90_ms": 190.63,
|
||||
"p95_ms": 198.89,
|
||||
"max_ms": 241.91,
|
||||
"mean_ms": 117.22
|
||||
},
|
||||
"append_skip": {
|
||||
"n": 30,
|
||||
"min_ms": 47.57,
|
||||
"median_ms": 51.42,
|
||||
"p90_ms": 59.82,
|
||||
"p95_ms": 61.57,
|
||||
"max_ms": 73.49,
|
||||
"mean_ms": 53.22
|
||||
},
|
||||
"skips_confirmed": 30,
|
||||
"all_skipped_second_pass": true
|
||||
},
|
||||
"gates": [
|
||||
{
|
||||
"rule": {
|
||||
"field": "all_skipped_second_pass",
|
||||
"op": ">=",
|
||||
"min": 1,
|
||||
"why": "second pass of identical lines must be 100% idempotent skips (boolean true coerced to 1)"
|
||||
},
|
||||
"value": true,
|
||||
"pass": true
|
||||
}
|
||||
]
|
||||
},
|
||||
"bulk-get": {
|
||||
"result": {
|
||||
"k": 30,
|
||||
"workers": 8,
|
||||
"serial": {
|
||||
"n": 5,
|
||||
"min_ms": 1520.47,
|
||||
"median_ms": 1606.86,
|
||||
"p90_ms": 1805.72,
|
||||
"p95_ms": 1835.83,
|
||||
"max_ms": 1865.94,
|
||||
"mean_ms": 1658.6,
|
||||
"warmup": {
|
||||
"n": 1,
|
||||
"min_ms": 1601.2,
|
||||
"median_ms": 1601.2,
|
||||
"p90_ms": 1601.2,
|
||||
"p95_ms": 1601.2,
|
||||
"max_ms": 1601.2,
|
||||
"mean_ms": 1601.2
|
||||
}
|
||||
},
|
||||
"concurrent": {
|
||||
"n": 5,
|
||||
"min_ms": 340.31,
|
||||
"median_ms": 356.48,
|
||||
"p90_ms": 358.82,
|
||||
"p95_ms": 359.32,
|
||||
"max_ms": 359.83,
|
||||
"mean_ms": 351.42,
|
||||
"warmup": {
|
||||
"n": 2,
|
||||
"min_ms": 353.43,
|
||||
"median_ms": 434.94,
|
||||
"p90_ms": 500.15,
|
||||
"p95_ms": 508.3,
|
||||
"max_ms": 516.45,
|
||||
"mean_ms": 434.94
|
||||
}
|
||||
},
|
||||
"speedup_ratio": 4.51
|
||||
},
|
||||
"gates": [
|
||||
{
|
||||
"rule": {
|
||||
"field": "speedup_ratio",
|
||||
"op": ">=",
|
||||
"min": 1.5,
|
||||
"why": "controlled-K concurrent vs serial; lower bar than full-vault since K is small"
|
||||
},
|
||||
"value": 4.51,
|
||||
"pass": true
|
||||
}
|
||||
]
|
||||
},
|
||||
"bulk-put": {
|
||||
"result": {
|
||||
"k": 30,
|
||||
"per_put": {
|
||||
"n": 30,
|
||||
"min_ms": 100.09,
|
||||
"median_ms": 114.98,
|
||||
"p90_ms": 212.46,
|
||||
"p95_ms": 236.18,
|
||||
"max_ms": 330.77,
|
||||
"mean_ms": 143.25
|
||||
},
|
||||
"total_ms": 4297.5,
|
||||
"puts_per_sec": 7.0
|
||||
},
|
||||
"gates": []
|
||||
},
|
||||
"cache": {
|
||||
"result": {
|
||||
"requested": 60,
|
||||
"unique": 20,
|
||||
"returned": 20,
|
||||
"deduped": true,
|
||||
"elapsed_ms": 307.13
|
||||
},
|
||||
"gates": [
|
||||
{
|
||||
"rule": {
|
||||
"field": "deduped",
|
||||
"op": ">=",
|
||||
"min": 1,
|
||||
"why": "read_many must collapse a 3x-duplicated path list to the unique set"
|
||||
},
|
||||
"value": true,
|
||||
"pass": true
|
||||
}
|
||||
]
|
||||
},
|
||||
"expand-graph": {
|
||||
"result": {
|
||||
"subjects": [
|
||||
{
|
||||
"label": "hub-echo",
|
||||
"query": "echo",
|
||||
"neighbours": 120,
|
||||
"serial_ms": 2227.67,
|
||||
"concurrent_ms": 642.2,
|
||||
"speedup": 3.47,
|
||||
"ranking_identical": true,
|
||||
"scores_identical": true
|
||||
},
|
||||
{
|
||||
"label": "hub-prefs",
|
||||
"query": "operator preferences",
|
||||
"neighbours": 126,
|
||||
"serial_ms": 2910.46,
|
||||
"concurrent_ms": 698.8,
|
||||
"speedup": 4.16,
|
||||
"ranking_identical": true,
|
||||
"scores_identical": true
|
||||
}
|
||||
],
|
||||
"min_speedup": 3.47,
|
||||
"all_ranking_identical": true,
|
||||
"all_scores_identical": true
|
||||
},
|
||||
"gates": [
|
||||
{
|
||||
"rule": {
|
||||
"field": "min_speedup",
|
||||
"op": ">=",
|
||||
"min": 2.0,
|
||||
"why": "recall()'s graph layer must fetch each BFS hop concurrently (read_many), not serially per node; <2x means expand_graph regressed to the pre-fix serial walk"
|
||||
},
|
||||
"value": 3.47,
|
||||
"pass": true
|
||||
},
|
||||
{
|
||||
"rule": {
|
||||
"field": "all_ranking_identical",
|
||||
"op": ">=",
|
||||
"min": 1,
|
||||
"why": "the concurrent expansion must return the same ranked neighbours as the serial reference"
|
||||
},
|
||||
"value": true,
|
||||
"pass": true
|
||||
},
|
||||
{
|
||||
"rule": {
|
||||
"field": "all_scores_identical",
|
||||
"op": ">=",
|
||||
"min": 1,
|
||||
"why": "decayed scores must match the serial reference to within float tolerance"
|
||||
},
|
||||
"value": true,
|
||||
"pass": true
|
||||
}
|
||||
]
|
||||
},
|
||||
"index": {
|
||||
"result": {
|
||||
"script": "sweep.py",
|
||||
"elapsed_ms": 4629.0,
|
||||
"exit_code": 0,
|
||||
"ok": true,
|
||||
"stderr_tail": []
|
||||
},
|
||||
"gates": []
|
||||
},
|
||||
"lint": {
|
||||
"result": {
|
||||
"script": "vault_lint.py",
|
||||
"elapsed_ms": 5126.4,
|
||||
"exit_code": 1,
|
||||
"ok": true,
|
||||
"stderr_tail": []
|
||||
},
|
||||
"gates": []
|
||||
},
|
||||
"lock": {
|
||||
"result": {
|
||||
"acquire_free_ms": 159.18,
|
||||
"acquire_free_rc": 0,
|
||||
"contended_ms": 57.89,
|
||||
"contended_rc_expected_75": 75,
|
||||
"release_ms": 132.92,
|
||||
"ok": true
|
||||
},
|
||||
"gates": [
|
||||
{
|
||||
"rule": {
|
||||
"field": "ok",
|
||||
"op": ">=",
|
||||
"min": 1,
|
||||
"why": "free acquire returns 0 and a contended acquire fast-fails with exit 75"
|
||||
},
|
||||
"value": true,
|
||||
"pass": true
|
||||
}
|
||||
]
|
||||
},
|
||||
"pool-warmup": {
|
||||
"result": {
|
||||
"probe": "_agent/echo-vault.md",
|
||||
"cold_ms": 163.25,
|
||||
"warm": {
|
||||
"n": 12,
|
||||
"min_ms": 45.64,
|
||||
"median_ms": 50.29,
|
||||
"p90_ms": 52.51,
|
||||
"p95_ms": 53.79,
|
||||
"max_ms": 55.28,
|
||||
"mean_ms": 50.26,
|
||||
"warmup": {
|
||||
"n": 1,
|
||||
"min_ms": 51.68,
|
||||
"median_ms": 51.68,
|
||||
"p90_ms": 51.68,
|
||||
"p95_ms": 51.68,
|
||||
"max_ms": 51.68,
|
||||
"mean_ms": 51.68
|
||||
}
|
||||
},
|
||||
"cold_over_warm_ratio": 3.25
|
||||
},
|
||||
"gates": [
|
||||
{
|
||||
"rule": {
|
||||
"field": "cold_over_warm_ratio",
|
||||
"op": ">=",
|
||||
"min": 1.5,
|
||||
"why": "cold request must be meaningfully slower than warm \u2014 proves keep-alive is reusing the connection; a ratio near 1.0 means every request is re-handshaking (the pre-1.1.0 bug)"
|
||||
},
|
||||
"value": 3.25,
|
||||
"pass": true
|
||||
}
|
||||
]
|
||||
},
|
||||
"queue": {
|
||||
"result": {
|
||||
"k": 10,
|
||||
"enqueue": {
|
||||
"n": 10,
|
||||
"min_ms": 0.07,
|
||||
"median_ms": 0.08,
|
||||
"p90_ms": 0.18,
|
||||
"p95_ms": 0.2,
|
||||
"max_ms": 0.22,
|
||||
"mean_ms": 0.1
|
||||
},
|
||||
"pending_after_enqueue": 10,
|
||||
"note": "flush replay intentionally not run against the live vault; enqueue path timed only. See README queue caveat."
|
||||
},
|
||||
"gates": []
|
||||
},
|
||||
"read-full": {
|
||||
"result": {
|
||||
"note_count": 197,
|
||||
"workers": 8,
|
||||
"serial": {
|
||||
"n": 1,
|
||||
"min_ms": 11376.08,
|
||||
"median_ms": 11376.08,
|
||||
"p90_ms": 11376.08,
|
||||
"p95_ms": 11376.08,
|
||||
"max_ms": 11376.08,
|
||||
"mean_ms": 11376.08
|
||||
},
|
||||
"concurrent": {
|
||||
"n": 4,
|
||||
"min_ms": 1559.79,
|
||||
"median_ms": 1595.31,
|
||||
"p90_ms": 1740.92,
|
||||
"p95_ms": 1768.05,
|
||||
"max_ms": 1795.17,
|
||||
"mean_ms": 1636.4,
|
||||
"warmup": {
|
||||
"n": 2,
|
||||
"min_ms": 1641.35,
|
||||
"median_ms": 1652.91,
|
||||
"p90_ms": 1662.17,
|
||||
"p95_ms": 1663.32,
|
||||
"max_ms": 1664.48,
|
||||
"mean_ms": 1652.91
|
||||
}
|
||||
},
|
||||
"speedup_ratio": 7.13,
|
||||
"notes_per_sec_concurrent": 123.5
|
||||
},
|
||||
"gates": [
|
||||
{
|
||||
"rule": {
|
||||
"field": "speedup_ratio",
|
||||
"op": ">=",
|
||||
"min": 1.8,
|
||||
"why": "concurrent read_many must beat serial by the 1.1.0 concurrency margin; <1.8 means concurrency regressed"
|
||||
},
|
||||
"value": 7.13,
|
||||
"pass": true
|
||||
}
|
||||
]
|
||||
},
|
||||
"resolve": {
|
||||
"result": {
|
||||
"cases": [
|
||||
{
|
||||
"mention": "echo",
|
||||
"expect": "exact",
|
||||
"expect_slug": "echo",
|
||||
"resolved_slug": "echo",
|
||||
"candidate_slugs": [
|
||||
"2026-06-19-goldbrain-full-echo-architect",
|
||||
"echo",
|
||||
"echo-memory-codex-plugin",
|
||||
"echo-plugin-build",
|
||||
"echo-skill-improvements"
|
||||
],
|
||||
"verdict": "PASS",
|
||||
"resolve_timing": {
|
||||
"n": 6,
|
||||
"min_ms": 0.0,
|
||||
"median_ms": 0.0,
|
||||
"p90_ms": 0.0,
|
||||
"p95_ms": 0.0,
|
||||
"max_ms": 0.0,
|
||||
"mean_ms": 0.0,
|
||||
"warmup": {
|
||||
"n": 2,
|
||||
"min_ms": 0.01,
|
||||
"median_ms": 0.1,
|
||||
"p90_ms": 0.17,
|
||||
"p95_ms": 0.18,
|
||||
"max_ms": 0.19,
|
||||
"mean_ms": 0.1
|
||||
}
|
||||
},
|
||||
"fuzzy_timing": {
|
||||
"n": 6,
|
||||
"min_ms": 1.35,
|
||||
"median_ms": 1.49,
|
||||
"p90_ms": 1.67,
|
||||
"p95_ms": 1.68,
|
||||
"max_ms": 1.68,
|
||||
"mean_ms": 1.51,
|
||||
"warmup": {
|
||||
"n": 2,
|
||||
"min_ms": 1.82,
|
||||
"median_ms": 1.83,
|
||||
"p90_ms": 1.84,
|
||||
"p95_ms": 1.84,
|
||||
"max_ms": 1.85,
|
||||
"mean_ms": 1.83
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"mention": "echo memory",
|
||||
"expect": "candidates",
|
||||
"expect_slug": "echo",
|
||||
"resolved_slug": "echo",
|
||||
"candidate_slugs": [
|
||||
"echo",
|
||||
"echo-memory-codex-plugin",
|
||||
"echo-plugin-build",
|
||||
"echo-skill-improvements",
|
||||
"2026-06-19-goldbrain-full-echo-architect"
|
||||
],
|
||||
"verdict": "INFO",
|
||||
"resolve_timing": {
|
||||
"n": 6,
|
||||
"min_ms": 0.55,
|
||||
"median_ms": 0.56,
|
||||
"p90_ms": 0.57,
|
||||
"p95_ms": 0.58,
|
||||
"max_ms": 0.58,
|
||||
"mean_ms": 0.56,
|
||||
"warmup": {
|
||||
"n": 2,
|
||||
"min_ms": 0.61,
|
||||
"median_ms": 0.61,
|
||||
"p90_ms": 0.61,
|
||||
"p95_ms": 0.61,
|
||||
"max_ms": 0.61,
|
||||
"mean_ms": 0.61
|
||||
}
|
||||
},
|
||||
"fuzzy_timing": {
|
||||
"n": 6,
|
||||
"min_ms": 0.97,
|
||||
"median_ms": 0.99,
|
||||
"p90_ms": 1.06,
|
||||
"p95_ms": 1.09,
|
||||
"max_ms": 1.11,
|
||||
"mean_ms": 1.01,
|
||||
"warmup": {
|
||||
"n": 2,
|
||||
"min_ms": 1.15,
|
||||
"median_ms": 1.18,
|
||||
"p90_ms": 1.2,
|
||||
"p95_ms": 1.21,
|
||||
"max_ms": 1.21,
|
||||
"mean_ms": 1.18
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"mention": "ECHO plugin",
|
||||
"expect": "candidates",
|
||||
"expect_slug": "echo",
|
||||
"resolved_slug": "echo",
|
||||
"candidate_slugs": [
|
||||
"echo",
|
||||
"echo-memory-codex-plugin",
|
||||
"echo-plugin-build",
|
||||
"2026-06-19-goldbrain-full-echo-architect",
|
||||
"alabama-wisp-brand-docs"
|
||||
],
|
||||
"verdict": "INFO",
|
||||
"resolve_timing": {
|
||||
"n": 6,
|
||||
"min_ms": 0.41,
|
||||
"median_ms": 0.41,
|
||||
"p90_ms": 0.42,
|
||||
"p95_ms": 0.42,
|
||||
"max_ms": 0.42,
|
||||
"mean_ms": 0.41,
|
||||
"warmup": {
|
||||
"n": 2,
|
||||
"min_ms": 0.42,
|
||||
"median_ms": 0.45,
|
||||
"p90_ms": 0.47,
|
||||
"p95_ms": 0.48,
|
||||
"max_ms": 0.48,
|
||||
"mean_ms": 0.45
|
||||
}
|
||||
},
|
||||
"fuzzy_timing": {
|
||||
"n": 6,
|
||||
"min_ms": 0.79,
|
||||
"median_ms": 0.8,
|
||||
"p90_ms": 0.88,
|
||||
"p95_ms": 0.9,
|
||||
"max_ms": 0.92,
|
||||
"mean_ms": 0.83,
|
||||
"warmup": {
|
||||
"n": 2,
|
||||
"min_ms": 0.89,
|
||||
"median_ms": 0.89,
|
||||
"p90_ms": 0.9,
|
||||
"p95_ms": 0.9,
|
||||
"max_ms": 0.9,
|
||||
"mean_ms": 0.89
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"mention": "goldbrain",
|
||||
"expect": "exact",
|
||||
"expect_slug": "goldbrain",
|
||||
"resolved_slug": "goldbrain",
|
||||
"candidate_slugs": [
|
||||
"2026-06-19-goldbrain-full-echo-architect",
|
||||
"goldbrain"
|
||||
],
|
||||
"verdict": "PASS",
|
||||
"resolve_timing": {
|
||||
"n": 6,
|
||||
"min_ms": 0.0,
|
||||
"median_ms": 0.0,
|
||||
"p90_ms": 0.0,
|
||||
"p95_ms": 0.0,
|
||||
"max_ms": 0.0,
|
||||
"mean_ms": 0.0,
|
||||
"warmup": {
|
||||
"n": 2,
|
||||
"min_ms": 0.0,
|
||||
"median_ms": 0.0,
|
||||
"p90_ms": 0.0,
|
||||
"p95_ms": 0.0,
|
||||
"max_ms": 0.0,
|
||||
"mean_ms": 0.0
|
||||
}
|
||||
},
|
||||
"fuzzy_timing": {
|
||||
"n": 6,
|
||||
"min_ms": 0.73,
|
||||
"median_ms": 0.75,
|
||||
"p90_ms": 0.85,
|
||||
"p95_ms": 0.88,
|
||||
"max_ms": 0.91,
|
||||
"mean_ms": 0.78,
|
||||
"warmup": {
|
||||
"n": 2,
|
||||
"min_ms": 0.8,
|
||||
"median_ms": 0.81,
|
||||
"p90_ms": 0.82,
|
||||
"p95_ms": 0.82,
|
||||
"max_ms": 0.82,
|
||||
"mean_ms": 0.81
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"mention": "jason stedwell",
|
||||
"expect": "exact",
|
||||
"expect_slug": "jason-stedwell",
|
||||
"resolved_slug": "jason-stedwell",
|
||||
"candidate_slugs": [
|
||||
"jason-stedwell",
|
||||
"jason-mcp-gateway"
|
||||
],
|
||||
"verdict": "PASS",
|
||||
"resolve_timing": {
|
||||
"n": 6,
|
||||
"min_ms": 0.0,
|
||||
"median_ms": 0.0,
|
||||
"p90_ms": 0.0,
|
||||
"p95_ms": 0.0,
|
||||
"max_ms": 0.0,
|
||||
"mean_ms": 0.0,
|
||||
"warmup": {
|
||||
"n": 2,
|
||||
"min_ms": 0.0,
|
||||
"median_ms": 0.0,
|
||||
"p90_ms": 0.0,
|
||||
"p95_ms": 0.0,
|
||||
"max_ms": 0.0,
|
||||
"mean_ms": 0.0
|
||||
}
|
||||
},
|
||||
"fuzzy_timing": {
|
||||
"n": 6,
|
||||
"min_ms": 0.69,
|
||||
"median_ms": 0.72,
|
||||
"p90_ms": 0.75,
|
||||
"p95_ms": 0.75,
|
||||
"max_ms": 0.75,
|
||||
"mean_ms": 0.72,
|
||||
"warmup": {
|
||||
"n": 2,
|
||||
"min_ms": 0.74,
|
||||
"median_ms": 0.74,
|
||||
"p90_ms": 0.75,
|
||||
"p95_ms": 0.75,
|
||||
"max_ms": 0.75,
|
||||
"mean_ms": 0.74
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"mention": "jason",
|
||||
"expect": "candidates",
|
||||
"expect_slug": "jason-stedwell",
|
||||
"resolved_slug": "jason-stedwell",
|
||||
"candidate_slugs": [
|
||||
"jason-mcp-gateway",
|
||||
"jason-stedwell"
|
||||
],
|
||||
"verdict": "INFO",
|
||||
"resolve_timing": {
|
||||
"n": 6,
|
||||
"min_ms": 0.61,
|
||||
"median_ms": 0.65,
|
||||
"p90_ms": 0.66,
|
||||
"p95_ms": 0.67,
|
||||
"max_ms": 0.67,
|
||||
"mean_ms": 0.64,
|
||||
"warmup": {
|
||||
"n": 2,
|
||||
"min_ms": 0.66,
|
||||
"median_ms": 0.68,
|
||||
"p90_ms": 0.69,
|
||||
"p95_ms": 0.69,
|
||||
"max_ms": 0.69,
|
||||
"mean_ms": 0.68
|
||||
}
|
||||
},
|
||||
"fuzzy_timing": {
|
||||
"n": 6,
|
||||
"min_ms": 0.63,
|
||||
"median_ms": 0.63,
|
||||
"p90_ms": 0.65,
|
||||
"p95_ms": 0.65,
|
||||
"max_ms": 0.66,
|
||||
"mean_ms": 0.64,
|
||||
"warmup": {
|
||||
"n": 2,
|
||||
"min_ms": 0.68,
|
||||
"median_ms": 0.69,
|
||||
"p90_ms": 0.7,
|
||||
"p95_ms": 0.7,
|
||||
"max_ms": 0.7,
|
||||
"mean_ms": 0.69
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"mention": "zzqx nonexistent entity",
|
||||
"expect": "miss",
|
||||
"expect_slug": null,
|
||||
"resolved_slug": null,
|
||||
"candidate_slugs": [],
|
||||
"verdict": "PASS",
|
||||
"resolve_timing": {
|
||||
"n": 6,
|
||||
"min_ms": 0.62,
|
||||
"median_ms": 0.63,
|
||||
"p90_ms": 0.67,
|
||||
"p95_ms": 0.67,
|
||||
"max_ms": 0.68,
|
||||
"mean_ms": 0.64,
|
||||
"warmup": {
|
||||
"n": 2,
|
||||
"min_ms": 0.66,
|
||||
"median_ms": 0.67,
|
||||
"p90_ms": 0.67,
|
||||
"p95_ms": 0.67,
|
||||
"max_ms": 0.67,
|
||||
"mean_ms": 0.67
|
||||
}
|
||||
},
|
||||
"fuzzy_timing": {
|
||||
"n": 6,
|
||||
"min_ms": 0.61,
|
||||
"median_ms": 0.62,
|
||||
"p90_ms": 0.65,
|
||||
"p95_ms": 0.65,
|
||||
"max_ms": 0.66,
|
||||
"mean_ms": 0.62,
|
||||
"warmup": {
|
||||
"n": 2,
|
||||
"min_ms": 0.62,
|
||||
"median_ms": 0.63,
|
||||
"p90_ms": 0.64,
|
||||
"p95_ms": 0.64,
|
||||
"max_ms": 0.64,
|
||||
"mean_ms": 0.63
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"passes": 4,
|
||||
"needs_tuning": 3,
|
||||
"note": "verdicts are PASS/INFO only; promote to FAIL once the table is confirmed against the live vault"
|
||||
},
|
||||
"gates": []
|
||||
},
|
||||
"soak": {
|
||||
"result": {
|
||||
"seconds": 22,
|
||||
"iterations": 15,
|
||||
"errors": 0,
|
||||
"per_pass": {
|
||||
"n": 15,
|
||||
"min_ms": 1472.88,
|
||||
"median_ms": 1551.67,
|
||||
"p90_ms": 1648.45,
|
||||
"p95_ms": 1702.66,
|
||||
"max_ms": 1820.58,
|
||||
"mean_ms": 1569.81
|
||||
},
|
||||
"early_mean_ms": 1575.63,
|
||||
"late_mean_ms": 1564.72,
|
||||
"drift_ratio_late_over_early": 0.99
|
||||
},
|
||||
"gates": [
|
||||
{
|
||||
"rule": {
|
||||
"field": "errors",
|
||||
"op": "<=",
|
||||
"min": 0,
|
||||
"why": "no read errors across the soak window \u2014 connection leaks/pool exhaustion would surface here"
|
||||
},
|
||||
"value": 0,
|
||||
"pass": true
|
||||
},
|
||||
{
|
||||
"rule": {
|
||||
"field": "drift_ratio_late_over_early",
|
||||
"op": "<=",
|
||||
"min": 1.5,
|
||||
"why": "late passes must not be much slower than early ones; >1.5 suggests a leak or degrading pool"
|
||||
},
|
||||
"value": 0.99,
|
||||
"pass": true
|
||||
}
|
||||
]
|
||||
},
|
||||
"subject-pull": {
|
||||
"result": {
|
||||
"subjects": [
|
||||
{
|
||||
"label": "apta",
|
||||
"query": "APTA",
|
||||
"resolve_floor": {
|
||||
"n": 1,
|
||||
"min_ms": 0.75,
|
||||
"median_ms": 0.75,
|
||||
"p90_ms": 0.75,
|
||||
"p95_ms": 0.75,
|
||||
"max_ms": 0.75,
|
||||
"mean_ms": 0.75
|
||||
},
|
||||
"search_only": {
|
||||
"n": 1,
|
||||
"min_ms": 61.87,
|
||||
"median_ms": 61.87,
|
||||
"p90_ms": 61.87,
|
||||
"p95_ms": 61.87,
|
||||
"max_ms": 61.87,
|
||||
"mean_ms": 61.87
|
||||
},
|
||||
"recall_total": {
|
||||
"n": 1,
|
||||
"min_ms": 2231.31,
|
||||
"median_ms": 2231.31,
|
||||
"p90_ms": 2231.31,
|
||||
"p95_ms": 2231.31,
|
||||
"max_ms": 2231.31,
|
||||
"mean_ms": 2231.31
|
||||
}
|
||||
},
|
||||
{
|
||||
"label": "capmetro",
|
||||
"query": "CapMetro",
|
||||
"resolve_floor": {
|
||||
"n": 1,
|
||||
"min_ms": 0.03,
|
||||
"median_ms": 0.03,
|
||||
"p90_ms": 0.03,
|
||||
"p95_ms": 0.03,
|
||||
"max_ms": 0.03,
|
||||
"mean_ms": 0.03
|
||||
},
|
||||
"search_only": {
|
||||
"n": 1,
|
||||
"min_ms": 57.73,
|
||||
"median_ms": 57.73,
|
||||
"p90_ms": 57.73,
|
||||
"p95_ms": 57.73,
|
||||
"max_ms": 57.73,
|
||||
"mean_ms": 57.73
|
||||
},
|
||||
"recall_total": {
|
||||
"n": 1,
|
||||
"min_ms": 2116.06,
|
||||
"median_ms": 2116.06,
|
||||
"p90_ms": 2116.06,
|
||||
"p95_ms": 2116.06,
|
||||
"max_ms": 2116.06,
|
||||
"mean_ms": 2116.06
|
||||
}
|
||||
},
|
||||
{
|
||||
"label": "hub-echo",
|
||||
"query": "echo",
|
||||
"resolve_floor": {
|
||||
"n": 1,
|
||||
"min_ms": 0.03,
|
||||
"median_ms": 0.03,
|
||||
"p90_ms": 0.03,
|
||||
"p95_ms": 0.03,
|
||||
"max_ms": 0.03,
|
||||
"mean_ms": 0.03
|
||||
},
|
||||
"search_only": {
|
||||
"n": 1,
|
||||
"min_ms": 70.56,
|
||||
"median_ms": 70.56,
|
||||
"p90_ms": 70.56,
|
||||
"p95_ms": 70.56,
|
||||
"max_ms": 70.56,
|
||||
"mean_ms": 70.56
|
||||
},
|
||||
"recall_total": {
|
||||
"n": 1,
|
||||
"min_ms": 2403.62,
|
||||
"median_ms": 2403.62,
|
||||
"p90_ms": 2403.62,
|
||||
"p95_ms": 2403.62,
|
||||
"max_ms": 2403.62,
|
||||
"mean_ms": 2403.62
|
||||
}
|
||||
},
|
||||
{
|
||||
"label": "hub-prefs",
|
||||
"query": "operator preferences",
|
||||
"resolve_floor": {
|
||||
"n": 1,
|
||||
"min_ms": 0.04,
|
||||
"median_ms": 0.04,
|
||||
"p90_ms": 0.04,
|
||||
"p95_ms": 0.04,
|
||||
"max_ms": 0.04,
|
||||
"mean_ms": 0.04
|
||||
},
|
||||
"search_only": {
|
||||
"n": 1,
|
||||
"min_ms": 2011.13,
|
||||
"median_ms": 2011.13,
|
||||
"p90_ms": 2011.13,
|
||||
"p95_ms": 2011.13,
|
||||
"max_ms": 2011.13,
|
||||
"mean_ms": 2011.13
|
||||
},
|
||||
"recall_total": {
|
||||
"n": 1,
|
||||
"min_ms": 2994.84,
|
||||
"median_ms": 2994.84,
|
||||
"p90_ms": 2994.84,
|
||||
"p95_ms": 2994.84,
|
||||
"max_ms": 2994.84,
|
||||
"mean_ms": 2994.84
|
||||
}
|
||||
},
|
||||
{
|
||||
"label": "leaf-rivnut",
|
||||
"query": "rivnut torque spec",
|
||||
"resolve_floor": {
|
||||
"n": 1,
|
||||
"min_ms": 2.15,
|
||||
"median_ms": 2.15,
|
||||
"p90_ms": 2.15,
|
||||
"p95_ms": 2.15,
|
||||
"max_ms": 2.15,
|
||||
"mean_ms": 2.15
|
||||
},
|
||||
"search_only": {
|
||||
"n": 1,
|
||||
"min_ms": 2013.58,
|
||||
"median_ms": 2013.58,
|
||||
"p90_ms": 2013.58,
|
||||
"p95_ms": 2013.58,
|
||||
"max_ms": 2013.58,
|
||||
"mean_ms": 2013.58
|
||||
},
|
||||
"recall_total": {
|
||||
"n": 1,
|
||||
"min_ms": 2371.32,
|
||||
"median_ms": 2371.32,
|
||||
"p90_ms": 2371.32,
|
||||
"p95_ms": 2371.32,
|
||||
"max_ms": 2371.32,
|
||||
"mean_ms": 2371.32
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"gates": []
|
||||
},
|
||||
"worker-sweep": {
|
||||
"result": {
|
||||
"note_count": 197,
|
||||
"sweep": [
|
||||
{
|
||||
"workers": 1,
|
||||
"median_ms": 11917.49,
|
||||
"p95_ms": 11917.49
|
||||
},
|
||||
{
|
||||
"workers": 2,
|
||||
"median_ms": 5740.0,
|
||||
"p95_ms": 5740.0
|
||||
},
|
||||
{
|
||||
"workers": 4,
|
||||
"median_ms": 3034.3,
|
||||
"p95_ms": 3034.3
|
||||
},
|
||||
{
|
||||
"workers": 8,
|
||||
"median_ms": 1583.83,
|
||||
"p95_ms": 1583.83
|
||||
},
|
||||
{
|
||||
"workers": 16,
|
||||
"median_ms": 1277.77,
|
||||
"p95_ms": 1277.77
|
||||
}
|
||||
],
|
||||
"knee_workers": 16,
|
||||
"default_workers": 8
|
||||
},
|
||||
"gates": []
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
{
|
||||
"env": {
|
||||
"date": "2026-06-23T20:22:00",
|
||||
"endpoint": "https://echoapi.alwisp.com",
|
||||
"commit": "af16598",
|
||||
"echo_workers": 8,
|
||||
"echo_timeout": 30,
|
||||
"python": "3.10.12",
|
||||
"iterations": null,
|
||||
"count_k": 30
|
||||
},
|
||||
"run_id": "20260623-202200-882c82",
|
||||
"metrics": {
|
||||
"bulk-append": {
|
||||
"result": {
|
||||
"k": 30,
|
||||
"append_write": {
|
||||
"n": 30,
|
||||
"min_ms": 87.11,
|
||||
"median_ms": 104.08,
|
||||
"p90_ms": 190.63,
|
||||
"p95_ms": 198.89,
|
||||
"max_ms": 241.91,
|
||||
"mean_ms": 117.22
|
||||
},
|
||||
"append_skip": {
|
||||
"n": 30,
|
||||
"min_ms": 47.57,
|
||||
"median_ms": 51.42,
|
||||
"p90_ms": 59.82,
|
||||
"p95_ms": 61.57,
|
||||
"max_ms": 73.49,
|
||||
"mean_ms": 53.22
|
||||
},
|
||||
"skips_confirmed": 30,
|
||||
"all_skipped_second_pass": true
|
||||
},
|
||||
"gates": [
|
||||
{
|
||||
"rule": {
|
||||
"field": "all_skipped_second_pass",
|
||||
"op": ">=",
|
||||
"min": 1,
|
||||
"why": "second pass of identical lines must be 100% idempotent skips (boolean true coerced to 1)"
|
||||
},
|
||||
"value": true,
|
||||
"pass": true
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"cleanup": {
|
||||
"deleted": 1
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
{
|
||||
"env": {
|
||||
"date": "2026-06-23T20:21:36",
|
||||
"endpoint": "https://echoapi.alwisp.com",
|
||||
"commit": "af16598",
|
||||
"echo_workers": 8,
|
||||
"echo_timeout": 30,
|
||||
"python": "3.10.12",
|
||||
"iterations": 5,
|
||||
"count_k": 30
|
||||
},
|
||||
"run_id": "20260623-202136-f2f4dc",
|
||||
"metrics": {
|
||||
"bulk-get": {
|
||||
"result": {
|
||||
"k": 30,
|
||||
"workers": 8,
|
||||
"serial": {
|
||||
"n": 5,
|
||||
"min_ms": 1520.47,
|
||||
"median_ms": 1606.86,
|
||||
"p90_ms": 1805.72,
|
||||
"p95_ms": 1835.83,
|
||||
"max_ms": 1865.94,
|
||||
"mean_ms": 1658.6,
|
||||
"warmup": {
|
||||
"n": 1,
|
||||
"min_ms": 1601.2,
|
||||
"median_ms": 1601.2,
|
||||
"p90_ms": 1601.2,
|
||||
"p95_ms": 1601.2,
|
||||
"max_ms": 1601.2,
|
||||
"mean_ms": 1601.2
|
||||
}
|
||||
},
|
||||
"concurrent": {
|
||||
"n": 5,
|
||||
"min_ms": 340.31,
|
||||
"median_ms": 356.48,
|
||||
"p90_ms": 358.82,
|
||||
"p95_ms": 359.32,
|
||||
"max_ms": 359.83,
|
||||
"mean_ms": 351.42,
|
||||
"warmup": {
|
||||
"n": 2,
|
||||
"min_ms": 353.43,
|
||||
"median_ms": 434.94,
|
||||
"p90_ms": 500.15,
|
||||
"p95_ms": 508.3,
|
||||
"max_ms": 516.45,
|
||||
"mean_ms": 434.94
|
||||
}
|
||||
},
|
||||
"speedup_ratio": 4.51
|
||||
},
|
||||
"gates": [
|
||||
{
|
||||
"rule": {
|
||||
"field": "speedup_ratio",
|
||||
"op": ">=",
|
||||
"min": 1.5,
|
||||
"why": "controlled-K concurrent vs serial; lower bar than full-vault since K is small"
|
||||
},
|
||||
"value": 4.51,
|
||||
"pass": true
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"cleanup": {
|
||||
"deleted": 30
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
{
|
||||
"env": {
|
||||
"date": "2026-06-23T20:21:21",
|
||||
"endpoint": "https://echoapi.alwisp.com",
|
||||
"commit": "af16598",
|
||||
"echo_workers": 8,
|
||||
"echo_timeout": 30,
|
||||
"python": "3.10.12",
|
||||
"iterations": null,
|
||||
"count_k": 30
|
||||
},
|
||||
"run_id": "20260623-202121-23b5ac",
|
||||
"metrics": {
|
||||
"bulk-put": {
|
||||
"result": {
|
||||
"k": 30,
|
||||
"per_put": {
|
||||
"n": 30,
|
||||
"min_ms": 100.09,
|
||||
"median_ms": 114.98,
|
||||
"p90_ms": 212.46,
|
||||
"p95_ms": 236.18,
|
||||
"max_ms": 330.77,
|
||||
"mean_ms": 143.25
|
||||
},
|
||||
"total_ms": 4297.5,
|
||||
"puts_per_sec": 7.0
|
||||
},
|
||||
"gates": []
|
||||
}
|
||||
},
|
||||
"cleanup": {
|
||||
"deleted": 30
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
{
|
||||
"env": {
|
||||
"date": "2026-06-23T20:12:16",
|
||||
"endpoint": "https://echoapi.alwisp.com",
|
||||
"commit": "af16598",
|
||||
"echo_workers": 8,
|
||||
"echo_timeout": 30,
|
||||
"python": "3.10.12",
|
||||
"iterations": null,
|
||||
"count_k": 50
|
||||
},
|
||||
"run_id": "20260623-201216-e77971",
|
||||
"metrics": {
|
||||
"cache": {
|
||||
"result": {
|
||||
"requested": 60,
|
||||
"unique": 20,
|
||||
"returned": 20,
|
||||
"deduped": true,
|
||||
"elapsed_ms": 307.13
|
||||
},
|
||||
"gates": [
|
||||
{
|
||||
"rule": {
|
||||
"field": "deduped",
|
||||
"op": ">=",
|
||||
"min": 1,
|
||||
"why": "read_many must collapse a 3x-duplicated path list to the unique set"
|
||||
},
|
||||
"value": true,
|
||||
"pass": true
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,76 @@
|
||||
{
|
||||
"env": {
|
||||
"date": "2026-06-23T21:22:29",
|
||||
"endpoint": "https://echoapi.alwisp.com",
|
||||
"commit": "af16598",
|
||||
"echo_workers": 8,
|
||||
"echo_timeout": 30,
|
||||
"python": "3.10.12",
|
||||
"iterations": 3,
|
||||
"count_k": 50
|
||||
},
|
||||
"run_id": "20260623-212229-8c3ce2",
|
||||
"metrics": {
|
||||
"expand-graph": {
|
||||
"result": {
|
||||
"subjects": [
|
||||
{
|
||||
"label": "hub-echo",
|
||||
"query": "echo",
|
||||
"neighbours": 120,
|
||||
"serial_ms": 2227.67,
|
||||
"concurrent_ms": 642.2,
|
||||
"speedup": 3.47,
|
||||
"ranking_identical": true,
|
||||
"scores_identical": true
|
||||
},
|
||||
{
|
||||
"label": "hub-prefs",
|
||||
"query": "operator preferences",
|
||||
"neighbours": 126,
|
||||
"serial_ms": 2910.46,
|
||||
"concurrent_ms": 698.8,
|
||||
"speedup": 4.16,
|
||||
"ranking_identical": true,
|
||||
"scores_identical": true
|
||||
}
|
||||
],
|
||||
"min_speedup": 3.47,
|
||||
"all_ranking_identical": true,
|
||||
"all_scores_identical": true
|
||||
},
|
||||
"gates": [
|
||||
{
|
||||
"rule": {
|
||||
"field": "min_speedup",
|
||||
"op": ">=",
|
||||
"min": 2.0,
|
||||
"why": "recall()'s graph layer must fetch each BFS hop concurrently (read_many), not serially per node; <2x means expand_graph regressed to the pre-fix serial walk"
|
||||
},
|
||||
"value": 3.47,
|
||||
"pass": true
|
||||
},
|
||||
{
|
||||
"rule": {
|
||||
"field": "all_ranking_identical",
|
||||
"op": ">=",
|
||||
"min": 1,
|
||||
"why": "the concurrent expansion must return the same ranked neighbours as the serial reference"
|
||||
},
|
||||
"value": true,
|
||||
"pass": true
|
||||
},
|
||||
{
|
||||
"rule": {
|
||||
"field": "all_scores_identical",
|
||||
"op": ">=",
|
||||
"min": 1,
|
||||
"why": "decayed scores must match the serial reference to within float tolerance"
|
||||
},
|
||||
"value": true,
|
||||
"pass": true
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
{
|
||||
"env": {
|
||||
"date": "2026-06-23T20:20:57",
|
||||
"endpoint": "https://echoapi.alwisp.com",
|
||||
"commit": "af16598",
|
||||
"echo_workers": 8,
|
||||
"echo_timeout": 30,
|
||||
"python": "3.10.12",
|
||||
"iterations": null,
|
||||
"count_k": 50
|
||||
},
|
||||
"run_id": "20260623-202057-c41a52",
|
||||
"metrics": {
|
||||
"index": {
|
||||
"result": {
|
||||
"script": "sweep.py",
|
||||
"elapsed_ms": 4629.0,
|
||||
"exit_code": 0,
|
||||
"ok": true,
|
||||
"stderr_tail": []
|
||||
},
|
||||
"gates": []
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
{
|
||||
"env": {
|
||||
"date": "2026-06-23T20:21:01",
|
||||
"endpoint": "https://echoapi.alwisp.com",
|
||||
"commit": "af16598",
|
||||
"echo_workers": 8,
|
||||
"echo_timeout": 30,
|
||||
"python": "3.10.12",
|
||||
"iterations": null,
|
||||
"count_k": 50
|
||||
},
|
||||
"run_id": "20260623-202101-569382",
|
||||
"metrics": {
|
||||
"lint": {
|
||||
"result": {
|
||||
"script": "vault_lint.py",
|
||||
"elapsed_ms": 5126.4,
|
||||
"exit_code": 1,
|
||||
"ok": true,
|
||||
"stderr_tail": []
|
||||
},
|
||||
"gates": []
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
{
|
||||
"env": {
|
||||
"date": "2026-06-23T20:21:28",
|
||||
"endpoint": "https://echoapi.alwisp.com",
|
||||
"commit": "af16598",
|
||||
"echo_workers": 8,
|
||||
"echo_timeout": 30,
|
||||
"python": "3.10.12",
|
||||
"iterations": null,
|
||||
"count_k": 50
|
||||
},
|
||||
"run_id": "20260623-202128-53b113",
|
||||
"metrics": {
|
||||
"lock": {
|
||||
"result": {
|
||||
"acquire_free_ms": 159.18,
|
||||
"acquire_free_rc": 0,
|
||||
"contended_ms": 57.89,
|
||||
"contended_rc_expected_75": 75,
|
||||
"release_ms": 132.92,
|
||||
"ok": true
|
||||
},
|
||||
"gates": [
|
||||
{
|
||||
"rule": {
|
||||
"field": "ok",
|
||||
"op": ">=",
|
||||
"min": 1,
|
||||
"why": "free acquire returns 0 and a contended acquire fast-fails with exit 75"
|
||||
},
|
||||
"value": true,
|
||||
"pass": true
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"cleanup": {
|
||||
"deleted": 0
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
{
|
||||
"env": {
|
||||
"date": "2026-06-23T20:12:12",
|
||||
"endpoint": "https://echoapi.alwisp.com",
|
||||
"commit": "af16598",
|
||||
"echo_workers": 8,
|
||||
"echo_timeout": 30,
|
||||
"python": "3.10.12",
|
||||
"iterations": 12,
|
||||
"count_k": 50
|
||||
},
|
||||
"run_id": "20260623-201212-30c2b8",
|
||||
"metrics": {
|
||||
"pool-warmup": {
|
||||
"result": {
|
||||
"probe": "_agent/echo-vault.md",
|
||||
"cold_ms": 163.25,
|
||||
"warm": {
|
||||
"n": 12,
|
||||
"min_ms": 45.64,
|
||||
"median_ms": 50.29,
|
||||
"p90_ms": 52.51,
|
||||
"p95_ms": 53.79,
|
||||
"max_ms": 55.28,
|
||||
"mean_ms": 50.26,
|
||||
"warmup": {
|
||||
"n": 1,
|
||||
"min_ms": 51.68,
|
||||
"median_ms": 51.68,
|
||||
"p90_ms": 51.68,
|
||||
"p95_ms": 51.68,
|
||||
"max_ms": 51.68,
|
||||
"mean_ms": 51.68
|
||||
}
|
||||
},
|
||||
"cold_over_warm_ratio": 3.25
|
||||
},
|
||||
"gates": [
|
||||
{
|
||||
"rule": {
|
||||
"field": "cold_over_warm_ratio",
|
||||
"op": ">=",
|
||||
"min": 1.5,
|
||||
"why": "cold request must be meaningfully slower than warm \u2014 proves keep-alive is reusing the connection; a ratio near 1.0 means every request is re-handshaking (the pre-1.1.0 bug)"
|
||||
},
|
||||
"value": 3.25,
|
||||
"pass": true
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
{
|
||||
"env": {
|
||||
"date": "2026-06-23T20:22:05",
|
||||
"endpoint": "https://echoapi.alwisp.com",
|
||||
"commit": "af16598",
|
||||
"echo_workers": 8,
|
||||
"echo_timeout": 30,
|
||||
"python": "3.10.12",
|
||||
"iterations": null,
|
||||
"count_k": 10
|
||||
},
|
||||
"run_id": "20260623-202205-378725",
|
||||
"metrics": {
|
||||
"queue": {
|
||||
"result": {
|
||||
"k": 10,
|
||||
"enqueue": {
|
||||
"n": 10,
|
||||
"min_ms": 0.07,
|
||||
"median_ms": 0.08,
|
||||
"p90_ms": 0.18,
|
||||
"p95_ms": 0.2,
|
||||
"max_ms": 0.22,
|
||||
"mean_ms": 0.1
|
||||
},
|
||||
"pending_after_enqueue": 10,
|
||||
"note": "flush replay intentionally not run against the live vault; enqueue path timed only. See README queue caveat."
|
||||
},
|
||||
"gates": []
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
{
|
||||
"env": {
|
||||
"date": "2026-06-23T20:17:09",
|
||||
"endpoint": "https://echoapi.alwisp.com",
|
||||
"commit": "af16598",
|
||||
"echo_workers": 8,
|
||||
"echo_timeout": 30,
|
||||
"python": "3.10.12",
|
||||
"iterations": 4,
|
||||
"count_k": 50
|
||||
},
|
||||
"run_id": "20260623-201709-b08a2e",
|
||||
"metrics": {
|
||||
"read-full": {
|
||||
"result": {
|
||||
"note_count": 197,
|
||||
"workers": 8,
|
||||
"serial": {
|
||||
"n": 1,
|
||||
"min_ms": 11376.08,
|
||||
"median_ms": 11376.08,
|
||||
"p90_ms": 11376.08,
|
||||
"p95_ms": 11376.08,
|
||||
"max_ms": 11376.08,
|
||||
"mean_ms": 11376.08
|
||||
},
|
||||
"concurrent": {
|
||||
"n": 4,
|
||||
"min_ms": 1559.79,
|
||||
"median_ms": 1595.31,
|
||||
"p90_ms": 1740.92,
|
||||
"p95_ms": 1768.05,
|
||||
"max_ms": 1795.17,
|
||||
"mean_ms": 1636.4,
|
||||
"warmup": {
|
||||
"n": 2,
|
||||
"min_ms": 1641.35,
|
||||
"median_ms": 1652.91,
|
||||
"p90_ms": 1662.17,
|
||||
"p95_ms": 1663.32,
|
||||
"max_ms": 1664.48,
|
||||
"mean_ms": 1652.91
|
||||
}
|
||||
},
|
||||
"speedup_ratio": 7.13,
|
||||
"notes_per_sec_concurrent": 123.5
|
||||
},
|
||||
"gates": [
|
||||
{
|
||||
"rule": {
|
||||
"field": "speedup_ratio",
|
||||
"op": ">=",
|
||||
"min": 1.8,
|
||||
"why": "concurrent read_many must beat serial by the 1.1.0 concurrency margin; <1.8 means concurrency regressed"
|
||||
},
|
||||
"value": 7.13,
|
||||
"pass": true
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,360 @@
|
||||
{
|
||||
"env": {
|
||||
"date": "2026-06-23T20:12:19",
|
||||
"endpoint": "https://echoapi.alwisp.com",
|
||||
"commit": "af16598",
|
||||
"echo_workers": 8,
|
||||
"echo_timeout": 30,
|
||||
"python": "3.10.12",
|
||||
"iterations": 6,
|
||||
"count_k": 50
|
||||
},
|
||||
"run_id": "20260623-201219-819b5e",
|
||||
"metrics": {
|
||||
"resolve": {
|
||||
"result": {
|
||||
"cases": [
|
||||
{
|
||||
"mention": "echo",
|
||||
"expect": "exact",
|
||||
"expect_slug": "echo",
|
||||
"resolved_slug": "echo",
|
||||
"candidate_slugs": [
|
||||
"2026-06-19-goldbrain-full-echo-architect",
|
||||
"echo",
|
||||
"echo-memory-codex-plugin",
|
||||
"echo-plugin-build",
|
||||
"echo-skill-improvements"
|
||||
],
|
||||
"verdict": "PASS",
|
||||
"resolve_timing": {
|
||||
"n": 6,
|
||||
"min_ms": 0.0,
|
||||
"median_ms": 0.0,
|
||||
"p90_ms": 0.0,
|
||||
"p95_ms": 0.0,
|
||||
"max_ms": 0.0,
|
||||
"mean_ms": 0.0,
|
||||
"warmup": {
|
||||
"n": 2,
|
||||
"min_ms": 0.01,
|
||||
"median_ms": 0.1,
|
||||
"p90_ms": 0.17,
|
||||
"p95_ms": 0.18,
|
||||
"max_ms": 0.19,
|
||||
"mean_ms": 0.1
|
||||
}
|
||||
},
|
||||
"fuzzy_timing": {
|
||||
"n": 6,
|
||||
"min_ms": 1.35,
|
||||
"median_ms": 1.49,
|
||||
"p90_ms": 1.67,
|
||||
"p95_ms": 1.68,
|
||||
"max_ms": 1.68,
|
||||
"mean_ms": 1.51,
|
||||
"warmup": {
|
||||
"n": 2,
|
||||
"min_ms": 1.82,
|
||||
"median_ms": 1.83,
|
||||
"p90_ms": 1.84,
|
||||
"p95_ms": 1.84,
|
||||
"max_ms": 1.85,
|
||||
"mean_ms": 1.83
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"mention": "echo memory",
|
||||
"expect": "candidates",
|
||||
"expect_slug": "echo",
|
||||
"resolved_slug": "echo",
|
||||
"candidate_slugs": [
|
||||
"echo",
|
||||
"echo-memory-codex-plugin",
|
||||
"echo-plugin-build",
|
||||
"echo-skill-improvements",
|
||||
"2026-06-19-goldbrain-full-echo-architect"
|
||||
],
|
||||
"verdict": "INFO",
|
||||
"resolve_timing": {
|
||||
"n": 6,
|
||||
"min_ms": 0.55,
|
||||
"median_ms": 0.56,
|
||||
"p90_ms": 0.57,
|
||||
"p95_ms": 0.58,
|
||||
"max_ms": 0.58,
|
||||
"mean_ms": 0.56,
|
||||
"warmup": {
|
||||
"n": 2,
|
||||
"min_ms": 0.61,
|
||||
"median_ms": 0.61,
|
||||
"p90_ms": 0.61,
|
||||
"p95_ms": 0.61,
|
||||
"max_ms": 0.61,
|
||||
"mean_ms": 0.61
|
||||
}
|
||||
},
|
||||
"fuzzy_timing": {
|
||||
"n": 6,
|
||||
"min_ms": 0.97,
|
||||
"median_ms": 0.99,
|
||||
"p90_ms": 1.06,
|
||||
"p95_ms": 1.09,
|
||||
"max_ms": 1.11,
|
||||
"mean_ms": 1.01,
|
||||
"warmup": {
|
||||
"n": 2,
|
||||
"min_ms": 1.15,
|
||||
"median_ms": 1.18,
|
||||
"p90_ms": 1.2,
|
||||
"p95_ms": 1.21,
|
||||
"max_ms": 1.21,
|
||||
"mean_ms": 1.18
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"mention": "ECHO plugin",
|
||||
"expect": "candidates",
|
||||
"expect_slug": "echo",
|
||||
"resolved_slug": "echo",
|
||||
"candidate_slugs": [
|
||||
"echo",
|
||||
"echo-memory-codex-plugin",
|
||||
"echo-plugin-build",
|
||||
"2026-06-19-goldbrain-full-echo-architect",
|
||||
"alabama-wisp-brand-docs"
|
||||
],
|
||||
"verdict": "INFO",
|
||||
"resolve_timing": {
|
||||
"n": 6,
|
||||
"min_ms": 0.41,
|
||||
"median_ms": 0.41,
|
||||
"p90_ms": 0.42,
|
||||
"p95_ms": 0.42,
|
||||
"max_ms": 0.42,
|
||||
"mean_ms": 0.41,
|
||||
"warmup": {
|
||||
"n": 2,
|
||||
"min_ms": 0.42,
|
||||
"median_ms": 0.45,
|
||||
"p90_ms": 0.47,
|
||||
"p95_ms": 0.48,
|
||||
"max_ms": 0.48,
|
||||
"mean_ms": 0.45
|
||||
}
|
||||
},
|
||||
"fuzzy_timing": {
|
||||
"n": 6,
|
||||
"min_ms": 0.79,
|
||||
"median_ms": 0.8,
|
||||
"p90_ms": 0.88,
|
||||
"p95_ms": 0.9,
|
||||
"max_ms": 0.92,
|
||||
"mean_ms": 0.83,
|
||||
"warmup": {
|
||||
"n": 2,
|
||||
"min_ms": 0.89,
|
||||
"median_ms": 0.89,
|
||||
"p90_ms": 0.9,
|
||||
"p95_ms": 0.9,
|
||||
"max_ms": 0.9,
|
||||
"mean_ms": 0.89
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"mention": "goldbrain",
|
||||
"expect": "exact",
|
||||
"expect_slug": "goldbrain",
|
||||
"resolved_slug": "goldbrain",
|
||||
"candidate_slugs": [
|
||||
"2026-06-19-goldbrain-full-echo-architect",
|
||||
"goldbrain"
|
||||
],
|
||||
"verdict": "PASS",
|
||||
"resolve_timing": {
|
||||
"n": 6,
|
||||
"min_ms": 0.0,
|
||||
"median_ms": 0.0,
|
||||
"p90_ms": 0.0,
|
||||
"p95_ms": 0.0,
|
||||
"max_ms": 0.0,
|
||||
"mean_ms": 0.0,
|
||||
"warmup": {
|
||||
"n": 2,
|
||||
"min_ms": 0.0,
|
||||
"median_ms": 0.0,
|
||||
"p90_ms": 0.0,
|
||||
"p95_ms": 0.0,
|
||||
"max_ms": 0.0,
|
||||
"mean_ms": 0.0
|
||||
}
|
||||
},
|
||||
"fuzzy_timing": {
|
||||
"n": 6,
|
||||
"min_ms": 0.73,
|
||||
"median_ms": 0.75,
|
||||
"p90_ms": 0.85,
|
||||
"p95_ms": 0.88,
|
||||
"max_ms": 0.91,
|
||||
"mean_ms": 0.78,
|
||||
"warmup": {
|
||||
"n": 2,
|
||||
"min_ms": 0.8,
|
||||
"median_ms": 0.81,
|
||||
"p90_ms": 0.82,
|
||||
"p95_ms": 0.82,
|
||||
"max_ms": 0.82,
|
||||
"mean_ms": 0.81
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"mention": "jason stedwell",
|
||||
"expect": "exact",
|
||||
"expect_slug": "jason-stedwell",
|
||||
"resolved_slug": "jason-stedwell",
|
||||
"candidate_slugs": [
|
||||
"jason-stedwell",
|
||||
"jason-mcp-gateway"
|
||||
],
|
||||
"verdict": "PASS",
|
||||
"resolve_timing": {
|
||||
"n": 6,
|
||||
"min_ms": 0.0,
|
||||
"median_ms": 0.0,
|
||||
"p90_ms": 0.0,
|
||||
"p95_ms": 0.0,
|
||||
"max_ms": 0.0,
|
||||
"mean_ms": 0.0,
|
||||
"warmup": {
|
||||
"n": 2,
|
||||
"min_ms": 0.0,
|
||||
"median_ms": 0.0,
|
||||
"p90_ms": 0.0,
|
||||
"p95_ms": 0.0,
|
||||
"max_ms": 0.0,
|
||||
"mean_ms": 0.0
|
||||
}
|
||||
},
|
||||
"fuzzy_timing": {
|
||||
"n": 6,
|
||||
"min_ms": 0.69,
|
||||
"median_ms": 0.72,
|
||||
"p90_ms": 0.75,
|
||||
"p95_ms": 0.75,
|
||||
"max_ms": 0.75,
|
||||
"mean_ms": 0.72,
|
||||
"warmup": {
|
||||
"n": 2,
|
||||
"min_ms": 0.74,
|
||||
"median_ms": 0.74,
|
||||
"p90_ms": 0.75,
|
||||
"p95_ms": 0.75,
|
||||
"max_ms": 0.75,
|
||||
"mean_ms": 0.74
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"mention": "jason",
|
||||
"expect": "candidates",
|
||||
"expect_slug": "jason-stedwell",
|
||||
"resolved_slug": "jason-stedwell",
|
||||
"candidate_slugs": [
|
||||
"jason-mcp-gateway",
|
||||
"jason-stedwell"
|
||||
],
|
||||
"verdict": "INFO",
|
||||
"resolve_timing": {
|
||||
"n": 6,
|
||||
"min_ms": 0.61,
|
||||
"median_ms": 0.65,
|
||||
"p90_ms": 0.66,
|
||||
"p95_ms": 0.67,
|
||||
"max_ms": 0.67,
|
||||
"mean_ms": 0.64,
|
||||
"warmup": {
|
||||
"n": 2,
|
||||
"min_ms": 0.66,
|
||||
"median_ms": 0.68,
|
||||
"p90_ms": 0.69,
|
||||
"p95_ms": 0.69,
|
||||
"max_ms": 0.69,
|
||||
"mean_ms": 0.68
|
||||
}
|
||||
},
|
||||
"fuzzy_timing": {
|
||||
"n": 6,
|
||||
"min_ms": 0.63,
|
||||
"median_ms": 0.63,
|
||||
"p90_ms": 0.65,
|
||||
"p95_ms": 0.65,
|
||||
"max_ms": 0.66,
|
||||
"mean_ms": 0.64,
|
||||
"warmup": {
|
||||
"n": 2,
|
||||
"min_ms": 0.68,
|
||||
"median_ms": 0.69,
|
||||
"p90_ms": 0.7,
|
||||
"p95_ms": 0.7,
|
||||
"max_ms": 0.7,
|
||||
"mean_ms": 0.69
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"mention": "zzqx nonexistent entity",
|
||||
"expect": "miss",
|
||||
"expect_slug": null,
|
||||
"resolved_slug": null,
|
||||
"candidate_slugs": [],
|
||||
"verdict": "PASS",
|
||||
"resolve_timing": {
|
||||
"n": 6,
|
||||
"min_ms": 0.62,
|
||||
"median_ms": 0.63,
|
||||
"p90_ms": 0.67,
|
||||
"p95_ms": 0.67,
|
||||
"max_ms": 0.68,
|
||||
"mean_ms": 0.64,
|
||||
"warmup": {
|
||||
"n": 2,
|
||||
"min_ms": 0.66,
|
||||
"median_ms": 0.67,
|
||||
"p90_ms": 0.67,
|
||||
"p95_ms": 0.67,
|
||||
"max_ms": 0.67,
|
||||
"mean_ms": 0.67
|
||||
}
|
||||
},
|
||||
"fuzzy_timing": {
|
||||
"n": 6,
|
||||
"min_ms": 0.61,
|
||||
"median_ms": 0.62,
|
||||
"p90_ms": 0.65,
|
||||
"p95_ms": 0.65,
|
||||
"max_ms": 0.66,
|
||||
"mean_ms": 0.62,
|
||||
"warmup": {
|
||||
"n": 2,
|
||||
"min_ms": 0.62,
|
||||
"median_ms": 0.63,
|
||||
"p90_ms": 0.64,
|
||||
"p95_ms": 0.64,
|
||||
"max_ms": 0.64,
|
||||
"mean_ms": 0.63
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"passes": 4,
|
||||
"needs_tuning": 3,
|
||||
"note": "verdicts are PASS/INFO only; promote to FAIL once the table is confirmed against the live vault"
|
||||
},
|
||||
"gates": []
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,56 @@
|
||||
{
|
||||
"env": {
|
||||
"date": "2026-06-23T20:22:15",
|
||||
"endpoint": "https://echoapi.alwisp.com",
|
||||
"commit": "af16598",
|
||||
"echo_workers": 8,
|
||||
"echo_timeout": 30,
|
||||
"python": "3.10.12",
|
||||
"iterations": null,
|
||||
"count_k": 50
|
||||
},
|
||||
"run_id": "20260623-202215-7b414f",
|
||||
"metrics": {
|
||||
"soak": {
|
||||
"result": {
|
||||
"seconds": 22,
|
||||
"iterations": 15,
|
||||
"errors": 0,
|
||||
"per_pass": {
|
||||
"n": 15,
|
||||
"min_ms": 1472.88,
|
||||
"median_ms": 1551.67,
|
||||
"p90_ms": 1648.45,
|
||||
"p95_ms": 1702.66,
|
||||
"max_ms": 1820.58,
|
||||
"mean_ms": 1569.81
|
||||
},
|
||||
"early_mean_ms": 1575.63,
|
||||
"late_mean_ms": 1564.72,
|
||||
"drift_ratio_late_over_early": 0.99
|
||||
},
|
||||
"gates": [
|
||||
{
|
||||
"rule": {
|
||||
"field": "errors",
|
||||
"op": "<=",
|
||||
"min": 0,
|
||||
"why": "no read errors across the soak window \u2014 connection leaks/pool exhaustion would surface here"
|
||||
},
|
||||
"value": 0,
|
||||
"pass": true
|
||||
},
|
||||
{
|
||||
"rule": {
|
||||
"field": "drift_ratio_late_over_early",
|
||||
"op": "<=",
|
||||
"min": 1.5,
|
||||
"why": "late passes must not be much slower than early ones; >1.5 suggests a leak or degrading pool"
|
||||
},
|
||||
"value": 0.99,
|
||||
"pass": true
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
+177
@@ -0,0 +1,177 @@
|
||||
{
|
||||
"env": {
|
||||
"date": "2026-06-23T21:18:12",
|
||||
"endpoint": "https://echoapi.alwisp.com",
|
||||
"commit": "af16598",
|
||||
"echo_workers": 8,
|
||||
"echo_timeout": 30,
|
||||
"python": "3.10.12",
|
||||
"iterations": 1,
|
||||
"count_k": 50
|
||||
},
|
||||
"run_id": "20260623-211812-ea4b36",
|
||||
"metrics": {
|
||||
"subject-pull": {
|
||||
"result": {
|
||||
"subjects": [
|
||||
{
|
||||
"label": "apta",
|
||||
"query": "APTA",
|
||||
"resolve_floor": {
|
||||
"n": 1,
|
||||
"min_ms": 0.75,
|
||||
"median_ms": 0.75,
|
||||
"p90_ms": 0.75,
|
||||
"p95_ms": 0.75,
|
||||
"max_ms": 0.75,
|
||||
"mean_ms": 0.75
|
||||
},
|
||||
"search_only": {
|
||||
"n": 1,
|
||||
"min_ms": 61.87,
|
||||
"median_ms": 61.87,
|
||||
"p90_ms": 61.87,
|
||||
"p95_ms": 61.87,
|
||||
"max_ms": 61.87,
|
||||
"mean_ms": 61.87
|
||||
},
|
||||
"recall_total": {
|
||||
"n": 1,
|
||||
"min_ms": 2231.31,
|
||||
"median_ms": 2231.31,
|
||||
"p90_ms": 2231.31,
|
||||
"p95_ms": 2231.31,
|
||||
"max_ms": 2231.31,
|
||||
"mean_ms": 2231.31
|
||||
}
|
||||
},
|
||||
{
|
||||
"label": "capmetro",
|
||||
"query": "CapMetro",
|
||||
"resolve_floor": {
|
||||
"n": 1,
|
||||
"min_ms": 0.03,
|
||||
"median_ms": 0.03,
|
||||
"p90_ms": 0.03,
|
||||
"p95_ms": 0.03,
|
||||
"max_ms": 0.03,
|
||||
"mean_ms": 0.03
|
||||
},
|
||||
"search_only": {
|
||||
"n": 1,
|
||||
"min_ms": 57.73,
|
||||
"median_ms": 57.73,
|
||||
"p90_ms": 57.73,
|
||||
"p95_ms": 57.73,
|
||||
"max_ms": 57.73,
|
||||
"mean_ms": 57.73
|
||||
},
|
||||
"recall_total": {
|
||||
"n": 1,
|
||||
"min_ms": 2116.06,
|
||||
"median_ms": 2116.06,
|
||||
"p90_ms": 2116.06,
|
||||
"p95_ms": 2116.06,
|
||||
"max_ms": 2116.06,
|
||||
"mean_ms": 2116.06
|
||||
}
|
||||
},
|
||||
{
|
||||
"label": "hub-echo",
|
||||
"query": "echo",
|
||||
"resolve_floor": {
|
||||
"n": 1,
|
||||
"min_ms": 0.03,
|
||||
"median_ms": 0.03,
|
||||
"p90_ms": 0.03,
|
||||
"p95_ms": 0.03,
|
||||
"max_ms": 0.03,
|
||||
"mean_ms": 0.03
|
||||
},
|
||||
"search_only": {
|
||||
"n": 1,
|
||||
"min_ms": 70.56,
|
||||
"median_ms": 70.56,
|
||||
"p90_ms": 70.56,
|
||||
"p95_ms": 70.56,
|
||||
"max_ms": 70.56,
|
||||
"mean_ms": 70.56
|
||||
},
|
||||
"recall_total": {
|
||||
"n": 1,
|
||||
"min_ms": 2403.62,
|
||||
"median_ms": 2403.62,
|
||||
"p90_ms": 2403.62,
|
||||
"p95_ms": 2403.62,
|
||||
"max_ms": 2403.62,
|
||||
"mean_ms": 2403.62
|
||||
}
|
||||
},
|
||||
{
|
||||
"label": "hub-prefs",
|
||||
"query": "operator preferences",
|
||||
"resolve_floor": {
|
||||
"n": 1,
|
||||
"min_ms": 0.04,
|
||||
"median_ms": 0.04,
|
||||
"p90_ms": 0.04,
|
||||
"p95_ms": 0.04,
|
||||
"max_ms": 0.04,
|
||||
"mean_ms": 0.04
|
||||
},
|
||||
"search_only": {
|
||||
"n": 1,
|
||||
"min_ms": 2011.13,
|
||||
"median_ms": 2011.13,
|
||||
"p90_ms": 2011.13,
|
||||
"p95_ms": 2011.13,
|
||||
"max_ms": 2011.13,
|
||||
"mean_ms": 2011.13
|
||||
},
|
||||
"recall_total": {
|
||||
"n": 1,
|
||||
"min_ms": 2994.84,
|
||||
"median_ms": 2994.84,
|
||||
"p90_ms": 2994.84,
|
||||
"p95_ms": 2994.84,
|
||||
"max_ms": 2994.84,
|
||||
"mean_ms": 2994.84
|
||||
}
|
||||
},
|
||||
{
|
||||
"label": "leaf-rivnut",
|
||||
"query": "rivnut torque spec",
|
||||
"resolve_floor": {
|
||||
"n": 1,
|
||||
"min_ms": 2.15,
|
||||
"median_ms": 2.15,
|
||||
"p90_ms": 2.15,
|
||||
"p95_ms": 2.15,
|
||||
"max_ms": 2.15,
|
||||
"mean_ms": 2.15
|
||||
},
|
||||
"search_only": {
|
||||
"n": 1,
|
||||
"min_ms": 2013.58,
|
||||
"median_ms": 2013.58,
|
||||
"p90_ms": 2013.58,
|
||||
"p95_ms": 2013.58,
|
||||
"max_ms": 2013.58,
|
||||
"mean_ms": 2013.58
|
||||
},
|
||||
"recall_total": {
|
||||
"n": 1,
|
||||
"min_ms": 2371.32,
|
||||
"median_ms": 2371.32,
|
||||
"p90_ms": 2371.32,
|
||||
"p95_ms": 2371.32,
|
||||
"max_ms": 2371.32,
|
||||
"mean_ms": 2371.32
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"gates": []
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,177 @@
|
||||
{
|
||||
"env": {
|
||||
"date": "2026-06-23T20:20:23",
|
||||
"endpoint": "https://echoapi.alwisp.com",
|
||||
"commit": "af16598",
|
||||
"echo_workers": 8,
|
||||
"echo_timeout": 30,
|
||||
"python": "3.10.12",
|
||||
"iterations": 1,
|
||||
"count_k": 50
|
||||
},
|
||||
"run_id": "20260623-202023-274ddd",
|
||||
"metrics": {
|
||||
"subject-pull": {
|
||||
"result": {
|
||||
"subjects": [
|
||||
{
|
||||
"label": "apta",
|
||||
"query": "APTA",
|
||||
"resolve_floor": {
|
||||
"n": 1,
|
||||
"min_ms": 0.97,
|
||||
"median_ms": 0.97,
|
||||
"p90_ms": 0.97,
|
||||
"p95_ms": 0.97,
|
||||
"max_ms": 0.97,
|
||||
"mean_ms": 0.97
|
||||
},
|
||||
"search_only": {
|
||||
"n": 1,
|
||||
"min_ms": 71.55,
|
||||
"median_ms": 71.55,
|
||||
"p90_ms": 71.55,
|
||||
"p95_ms": 71.55,
|
||||
"max_ms": 71.55,
|
||||
"mean_ms": 71.55
|
||||
},
|
||||
"recall_total": {
|
||||
"n": 1,
|
||||
"min_ms": 3836.47,
|
||||
"median_ms": 3836.47,
|
||||
"p90_ms": 3836.47,
|
||||
"p95_ms": 3836.47,
|
||||
"max_ms": 3836.47,
|
||||
"mean_ms": 3836.47
|
||||
}
|
||||
},
|
||||
{
|
||||
"label": "capmetro",
|
||||
"query": "CapMetro",
|
||||
"resolve_floor": {
|
||||
"n": 1,
|
||||
"min_ms": 0.05,
|
||||
"median_ms": 0.05,
|
||||
"p90_ms": 0.05,
|
||||
"p95_ms": 0.05,
|
||||
"max_ms": 0.05,
|
||||
"mean_ms": 0.05
|
||||
},
|
||||
"search_only": {
|
||||
"n": 1,
|
||||
"min_ms": 57.66,
|
||||
"median_ms": 57.66,
|
||||
"p90_ms": 57.66,
|
||||
"p95_ms": 57.66,
|
||||
"max_ms": 57.66,
|
||||
"mean_ms": 57.66
|
||||
},
|
||||
"recall_total": {
|
||||
"n": 1,
|
||||
"min_ms": 2800.54,
|
||||
"median_ms": 2800.54,
|
||||
"p90_ms": 2800.54,
|
||||
"p95_ms": 2800.54,
|
||||
"max_ms": 2800.54,
|
||||
"mean_ms": 2800.54
|
||||
}
|
||||
},
|
||||
{
|
||||
"label": "hub-echo",
|
||||
"query": "echo",
|
||||
"resolve_floor": {
|
||||
"n": 1,
|
||||
"min_ms": 0.02,
|
||||
"median_ms": 0.02,
|
||||
"p90_ms": 0.02,
|
||||
"p95_ms": 0.02,
|
||||
"max_ms": 0.02,
|
||||
"mean_ms": 0.02
|
||||
},
|
||||
"search_only": {
|
||||
"n": 1,
|
||||
"min_ms": 67.39,
|
||||
"median_ms": 67.39,
|
||||
"p90_ms": 67.39,
|
||||
"p95_ms": 67.39,
|
||||
"max_ms": 67.39,
|
||||
"mean_ms": 67.39
|
||||
},
|
||||
"recall_total": {
|
||||
"n": 1,
|
||||
"min_ms": 4114.19,
|
||||
"median_ms": 4114.19,
|
||||
"p90_ms": 4114.19,
|
||||
"p95_ms": 4114.19,
|
||||
"max_ms": 4114.19,
|
||||
"mean_ms": 4114.19
|
||||
}
|
||||
},
|
||||
{
|
||||
"label": "hub-prefs",
|
||||
"query": "operator preferences",
|
||||
"resolve_floor": {
|
||||
"n": 1,
|
||||
"min_ms": 0.04,
|
||||
"median_ms": 0.04,
|
||||
"p90_ms": 0.04,
|
||||
"p95_ms": 0.04,
|
||||
"max_ms": 0.04,
|
||||
"mean_ms": 0.04
|
||||
},
|
||||
"search_only": {
|
||||
"n": 1,
|
||||
"min_ms": 2013.4,
|
||||
"median_ms": 2013.4,
|
||||
"p90_ms": 2013.4,
|
||||
"p95_ms": 2013.4,
|
||||
"max_ms": 2013.4,
|
||||
"mean_ms": 2013.4
|
||||
},
|
||||
"recall_total": {
|
||||
"n": 1,
|
||||
"min_ms": 4748.56,
|
||||
"median_ms": 4748.56,
|
||||
"p90_ms": 4748.56,
|
||||
"p95_ms": 4748.56,
|
||||
"max_ms": 4748.56,
|
||||
"mean_ms": 4748.56
|
||||
}
|
||||
},
|
||||
{
|
||||
"label": "leaf-rivnut",
|
||||
"query": "rivnut torque spec",
|
||||
"resolve_floor": {
|
||||
"n": 1,
|
||||
"min_ms": 2.14,
|
||||
"median_ms": 2.14,
|
||||
"p90_ms": 2.14,
|
||||
"p95_ms": 2.14,
|
||||
"max_ms": 2.14,
|
||||
"mean_ms": 2.14
|
||||
},
|
||||
"search_only": {
|
||||
"n": 1,
|
||||
"min_ms": 2014.69,
|
||||
"median_ms": 2014.69,
|
||||
"p90_ms": 2014.69,
|
||||
"p95_ms": 2014.69,
|
||||
"max_ms": 2014.69,
|
||||
"mean_ms": 2014.69
|
||||
},
|
||||
"recall_total": {
|
||||
"n": 1,
|
||||
"min_ms": 4363.28,
|
||||
"median_ms": 4363.28,
|
||||
"p90_ms": 4363.28,
|
||||
"p95_ms": 4363.28,
|
||||
"max_ms": 4363.28,
|
||||
"mean_ms": 4363.28
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"gates": []
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
{
|
||||
"env": {
|
||||
"date": "2026-06-23T20:18:05",
|
||||
"endpoint": "https://echoapi.alwisp.com",
|
||||
"commit": "af16598",
|
||||
"echo_workers": 8,
|
||||
"echo_timeout": 30,
|
||||
"python": "3.10.12",
|
||||
"iterations": 2,
|
||||
"count_k": 50
|
||||
},
|
||||
"run_id": "20260623-201805-53dcdb",
|
||||
"metrics": {
|
||||
"worker-sweep": {
|
||||
"result": {
|
||||
"note_count": 197,
|
||||
"sweep": [
|
||||
{
|
||||
"workers": 1,
|
||||
"median_ms": 11917.49,
|
||||
"p95_ms": 11917.49
|
||||
},
|
||||
{
|
||||
"workers": 2,
|
||||
"median_ms": 5740.0,
|
||||
"p95_ms": 5740.0
|
||||
},
|
||||
{
|
||||
"workers": 4,
|
||||
"median_ms": 3034.3,
|
||||
"p95_ms": 3034.3
|
||||
},
|
||||
{
|
||||
"workers": 8,
|
||||
"median_ms": 1583.83,
|
||||
"p95_ms": 1583.83
|
||||
},
|
||||
{
|
||||
"workers": 16,
|
||||
"median_ms": 1277.77,
|
||||
"p95_ms": 1277.77
|
||||
}
|
||||
],
|
||||
"knee_workers": 16,
|
||||
"default_workers": 8
|
||||
},
|
||||
"gates": []
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,545 @@
|
||||
{
|
||||
"title": "ECHO Memory Plugin \u2014 Performance Benchmark",
|
||||
"subtitle": "Validation of the 1.1.0 concurrency and 1.2.0 entity-resolution releases",
|
||||
"reference": "echo-v.05 @ af16598",
|
||||
"status": "11/11 gate checks pass",
|
||||
"summary": "Full-vault reads run 7.1x faster concurrent than serial (11.4 s down to 1.6 s, 197 notes). Entity resolution is sub-millisecond. Profiling traced recall() latency to serial graph expansion (not BM25); the fix makes expansion concurrent (3.5-4.2x, identical results). All 11 gate checks pass. Run live against echoapi.alwisp.com on 2026-06-23.",
|
||||
"sections": [
|
||||
{
|
||||
"heading": "Test Environment",
|
||||
"type": "kv",
|
||||
"rows": [
|
||||
{
|
||||
"label": "Endpoint",
|
||||
"value": "echoapi.alwisp.com (live)"
|
||||
},
|
||||
{
|
||||
"label": "Vault size",
|
||||
"value": "197 notes"
|
||||
},
|
||||
{
|
||||
"label": "Plugin commit",
|
||||
"value": "af16598"
|
||||
},
|
||||
{
|
||||
"label": "Releases under test",
|
||||
"value": "1.1.0 pooling + concurrency, 1.2.0 resolver"
|
||||
},
|
||||
{
|
||||
"label": "Concurrency (default)",
|
||||
"value": "ECHO_WORKERS = 8"
|
||||
},
|
||||
{
|
||||
"label": "Socket timeout",
|
||||
"value": "ECHO_TIMEOUT = 30 s"
|
||||
},
|
||||
{
|
||||
"label": "Runner",
|
||||
"value": "Python 3.10.12"
|
||||
},
|
||||
{
|
||||
"label": "Date",
|
||||
"value": "June 23, 2026"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"heading": "Headline Results",
|
||||
"type": "table",
|
||||
"columns": [
|
||||
"Operation",
|
||||
"Result",
|
||||
"Verdict"
|
||||
],
|
||||
"rows": [
|
||||
[
|
||||
"Full-vault read (concurrent vs serial)",
|
||||
{
|
||||
"value": "7.13x",
|
||||
"num": true
|
||||
},
|
||||
"Pass"
|
||||
],
|
||||
[
|
||||
"Full-vault read throughput",
|
||||
{
|
||||
"value": "123.5 notes/s",
|
||||
"num": true
|
||||
},
|
||||
"Pass"
|
||||
],
|
||||
[
|
||||
"Cold vs warm request (keep-alive)",
|
||||
{
|
||||
"value": "3.25x",
|
||||
"num": true
|
||||
},
|
||||
"Pass"
|
||||
],
|
||||
[
|
||||
"Entity resolve (in-memory index)",
|
||||
{
|
||||
"value": "<1 ms",
|
||||
"num": true
|
||||
},
|
||||
"Pass"
|
||||
],
|
||||
[
|
||||
"Idempotent append skip",
|
||||
{
|
||||
"value": "100%",
|
||||
"num": true
|
||||
},
|
||||
"Pass"
|
||||
],
|
||||
[
|
||||
"Soak read errors (22 s)",
|
||||
{
|
||||
"value": "0",
|
||||
"num": true
|
||||
},
|
||||
"Pass"
|
||||
]
|
||||
]
|
||||
},
|
||||
{
|
||||
"heading": "Full-Vault Read \u2014 Concurrency Scaling",
|
||||
"type": "table",
|
||||
"columns": [
|
||||
"Workers",
|
||||
"Median (197 notes)",
|
||||
"Speedup vs serial"
|
||||
],
|
||||
"rows": [
|
||||
[
|
||||
"1 (serial)",
|
||||
{
|
||||
"value": "11,917 ms",
|
||||
"num": true
|
||||
},
|
||||
{
|
||||
"value": "1.0x",
|
||||
"num": true
|
||||
}
|
||||
],
|
||||
[
|
||||
"2",
|
||||
{
|
||||
"value": "5,740 ms",
|
||||
"num": true
|
||||
},
|
||||
{
|
||||
"value": "2.1x",
|
||||
"num": true
|
||||
}
|
||||
],
|
||||
[
|
||||
"4",
|
||||
{
|
||||
"value": "3,034 ms",
|
||||
"num": true
|
||||
},
|
||||
{
|
||||
"value": "3.9x",
|
||||
"num": true
|
||||
}
|
||||
],
|
||||
[
|
||||
"8 (default)",
|
||||
{
|
||||
"value": "1,584 ms",
|
||||
"num": true
|
||||
},
|
||||
{
|
||||
"value": "7.5x",
|
||||
"num": true
|
||||
}
|
||||
],
|
||||
[
|
||||
"16",
|
||||
{
|
||||
"value": "1,278 ms",
|
||||
"num": true
|
||||
},
|
||||
{
|
||||
"value": "9.3x",
|
||||
"num": true
|
||||
}
|
||||
]
|
||||
]
|
||||
},
|
||||
{
|
||||
"heading": "Subject Pulls \u2014 recall() (after concurrency fix)",
|
||||
"type": "table",
|
||||
"columns": [
|
||||
"Subject",
|
||||
"resolve()",
|
||||
"search",
|
||||
"recall() total"
|
||||
],
|
||||
"rows": [
|
||||
[
|
||||
"APTA",
|
||||
{
|
||||
"value": "0.75 ms",
|
||||
"num": true
|
||||
},
|
||||
{
|
||||
"value": "62 ms",
|
||||
"num": true
|
||||
},
|
||||
{
|
||||
"value": "2,231 ms",
|
||||
"num": true
|
||||
}
|
||||
],
|
||||
[
|
||||
"CapMetro",
|
||||
{
|
||||
"value": "0.03 ms",
|
||||
"num": true
|
||||
},
|
||||
{
|
||||
"value": "58 ms",
|
||||
"num": true
|
||||
},
|
||||
{
|
||||
"value": "2,116 ms",
|
||||
"num": true
|
||||
}
|
||||
],
|
||||
[
|
||||
"echo (hub)",
|
||||
{
|
||||
"value": "0.03 ms",
|
||||
"num": true
|
||||
},
|
||||
{
|
||||
"value": "71 ms",
|
||||
"num": true
|
||||
},
|
||||
{
|
||||
"value": "2,404 ms",
|
||||
"num": true
|
||||
}
|
||||
],
|
||||
[
|
||||
"operator preferences (hub)",
|
||||
{
|
||||
"value": "0.04 ms",
|
||||
"num": true
|
||||
},
|
||||
{
|
||||
"value": "2,011 ms",
|
||||
"num": true
|
||||
},
|
||||
{
|
||||
"value": "2,995 ms",
|
||||
"num": true
|
||||
}
|
||||
],
|
||||
[
|
||||
"rivnut torque spec (leaf)",
|
||||
{
|
||||
"value": "2.15 ms",
|
||||
"num": true
|
||||
},
|
||||
{
|
||||
"value": "2,014 ms",
|
||||
"num": true
|
||||
},
|
||||
{
|
||||
"value": "2,371 ms",
|
||||
"num": true
|
||||
}
|
||||
]
|
||||
]
|
||||
},
|
||||
{
|
||||
"heading": "recall() Graph Expansion \u2014 Concurrency Fix",
|
||||
"type": "table",
|
||||
"columns": [
|
||||
"Hub subject",
|
||||
"Neighbours",
|
||||
"Serial",
|
||||
"Concurrent",
|
||||
"Speedup",
|
||||
"Results identical"
|
||||
],
|
||||
"rows": [
|
||||
[
|
||||
"echo",
|
||||
{
|
||||
"value": "120",
|
||||
"num": true
|
||||
},
|
||||
{
|
||||
"value": "2,228 ms",
|
||||
"num": true
|
||||
},
|
||||
{
|
||||
"value": "642 ms",
|
||||
"num": true
|
||||
},
|
||||
{
|
||||
"value": "3.47x",
|
||||
"num": true
|
||||
},
|
||||
"Yes"
|
||||
],
|
||||
[
|
||||
"operator preferences",
|
||||
{
|
||||
"value": "126",
|
||||
"num": true
|
||||
},
|
||||
{
|
||||
"value": "2,910 ms",
|
||||
"num": true
|
||||
},
|
||||
{
|
||||
"value": "699 ms",
|
||||
"num": true
|
||||
},
|
||||
{
|
||||
"value": "4.16x",
|
||||
"num": true
|
||||
},
|
||||
"Yes"
|
||||
]
|
||||
]
|
||||
},
|
||||
{
|
||||
"heading": "Bulk Write Operations",
|
||||
"type": "table",
|
||||
"columns": [
|
||||
"Operation",
|
||||
"Per-op median",
|
||||
"Aggregate"
|
||||
],
|
||||
"rows": [
|
||||
[
|
||||
"PUT (read-back verified)",
|
||||
{
|
||||
"value": "115 ms",
|
||||
"num": true
|
||||
},
|
||||
{
|
||||
"value": "7.0 PUT/s",
|
||||
"num": true
|
||||
}
|
||||
],
|
||||
[
|
||||
"Bulk GET \u2014 concurrent vs serial (K=30)",
|
||||
{
|
||||
"value": "356 vs 1,607 ms",
|
||||
"num": true
|
||||
},
|
||||
{
|
||||
"value": "4.51x",
|
||||
"num": true
|
||||
}
|
||||
],
|
||||
[
|
||||
"APPEND \u2014 write",
|
||||
{
|
||||
"value": "104 ms",
|
||||
"num": true
|
||||
},
|
||||
{
|
||||
"value": "1 round-trip + GET",
|
||||
"num": false
|
||||
}
|
||||
],
|
||||
[
|
||||
"APPEND \u2014 idempotent skip",
|
||||
{
|
||||
"value": "51 ms",
|
||||
"num": true
|
||||
},
|
||||
{
|
||||
"value": "100% skipped on re-run",
|
||||
"num": false
|
||||
}
|
||||
]
|
||||
]
|
||||
},
|
||||
{
|
||||
"heading": "Concurrency, Locking & Resilience",
|
||||
"type": "table",
|
||||
"columns": [
|
||||
"Check",
|
||||
"Measured",
|
||||
"Expected"
|
||||
],
|
||||
"rows": [
|
||||
[
|
||||
"Lock \u2014 acquire when free",
|
||||
{
|
||||
"value": "159 ms (rc 0)",
|
||||
"num": true
|
||||
},
|
||||
"rc 0"
|
||||
],
|
||||
[
|
||||
"Lock \u2014 contended acquire",
|
||||
{
|
||||
"value": "58 ms (rc 75)",
|
||||
"num": true
|
||||
},
|
||||
"rc 75 fast-fail"
|
||||
],
|
||||
[
|
||||
"Lock \u2014 release",
|
||||
{
|
||||
"value": "133 ms",
|
||||
"num": true
|
||||
},
|
||||
"owned release"
|
||||
],
|
||||
[
|
||||
"Offline queue \u2014 enqueue",
|
||||
{
|
||||
"value": "0.08 ms",
|
||||
"num": true
|
||||
},
|
||||
"local write-ahead"
|
||||
],
|
||||
[
|
||||
"read_many dedup (60 -> 20)",
|
||||
{
|
||||
"value": "deduped",
|
||||
"num": false
|
||||
},
|
||||
"collapse to unique set"
|
||||
],
|
||||
[
|
||||
"Soak drift (late vs early)",
|
||||
{
|
||||
"value": "0.99x",
|
||||
"num": true
|
||||
},
|
||||
"<= 1.5x, no leak"
|
||||
]
|
||||
]
|
||||
},
|
||||
{
|
||||
"heading": "Regression Gates (relative-ratio policy)",
|
||||
"type": "table",
|
||||
"columns": [
|
||||
"Gate",
|
||||
"Threshold",
|
||||
"Measured",
|
||||
"Result"
|
||||
],
|
||||
"rows": [
|
||||
[
|
||||
"read-full speedup",
|
||||
">= 1.8x",
|
||||
{
|
||||
"value": "7.13x",
|
||||
"num": true
|
||||
},
|
||||
"Pass"
|
||||
],
|
||||
[
|
||||
"bulk-get speedup",
|
||||
">= 1.5x",
|
||||
{
|
||||
"value": "4.51x",
|
||||
"num": true
|
||||
},
|
||||
"Pass"
|
||||
],
|
||||
[
|
||||
"pool warm-up cold/warm",
|
||||
">= 1.5x",
|
||||
{
|
||||
"value": "3.25x",
|
||||
"num": true
|
||||
},
|
||||
"Pass"
|
||||
],
|
||||
[
|
||||
"append idempotency",
|
||||
"100% skip",
|
||||
{
|
||||
"value": "100%",
|
||||
"num": true
|
||||
},
|
||||
"Pass"
|
||||
],
|
||||
[
|
||||
"soak errors",
|
||||
"0",
|
||||
{
|
||||
"value": "0",
|
||||
"num": true
|
||||
},
|
||||
"Pass"
|
||||
],
|
||||
[
|
||||
"soak drift",
|
||||
"<= 1.5x",
|
||||
{
|
||||
"value": "0.99x",
|
||||
"num": true
|
||||
},
|
||||
"Pass"
|
||||
],
|
||||
[
|
||||
"lock semantics",
|
||||
"rc 0 / rc 75",
|
||||
{
|
||||
"value": "0 / 75",
|
||||
"num": true
|
||||
},
|
||||
"Pass"
|
||||
],
|
||||
[
|
||||
"read_many dedup",
|
||||
"unique set",
|
||||
{
|
||||
"value": "20/20",
|
||||
"num": true
|
||||
},
|
||||
"Pass"
|
||||
]
|
||||
]
|
||||
},
|
||||
{
|
||||
"heading": "Findings",
|
||||
"type": "olist",
|
||||
"items": [
|
||||
"Concurrency delivers as designed. Eight workers cut full-vault reads 7.5x; the curve is near-linear to 8 workers and flattens after (8->16 returns only 1.24x). The default of 8 is the right setting for this vault and link.",
|
||||
"Keep-alive is confirmed live. A cold request costs 163 ms against a 50 ms warm median (3.25x). This is the direct measure that the 1.1.0 pooling fix removed the per-request TLS handshake that was timing out full-vault passes.",
|
||||
"Entity resolution is effectively free. resolve() returns in under 1 ms across exact, alias, and shortened mentions. The 1.2.0 alias work resolves shortened names (\"echo memory\", \"ECHO plugin\", \"jason\") straight to the canonical note, so the anti-duplicate guard holds without a fuzzy fallback.",
|
||||
"recall() latency was traced to the graph layer, not BM25. The BM25 index is already persisted and incrementally maintained (load 500 ms, score 0.1 ms). The cost was expand_graph fetching each neighbour serially \u2014 3.0 s for 126 nodes. Fix: the graph BFS now fetches each hop concurrently via the existing read_many. expand_graph is 3.5-4.2x faster with byte-identical ranking and scores; end-to-end recall() dropped from 2.8-4.7 s to 2.1-3.0 s per subject.",
|
||||
"Full-vault maintenance scripts stay under the tool timeout. sweep.py runs in 4.6 s and vault_lint.py in 5.1 s on 197 notes. The original failure mode (passes exceeding the timeout and dropping the session) does not recur.",
|
||||
"Write integrity holds. PUT verifies via read-back at 115 ms; APPEND is whole-line idempotent, skipping at 51 ms with zero duplicate writes on re-run. The advisory lock fast-fails a contended acquire with the expected exit 75."
|
||||
]
|
||||
},
|
||||
{
|
||||
"heading": "Recommendations",
|
||||
"type": "olist",
|
||||
"items": [
|
||||
"Apply read_many to every code path that reads a set of known notes. Done for expand_graph (gated). Remaining: prefetch the _brief result bodies recall prints, and parallelize rebuild()'s vault walk.",
|
||||
"Reuse one in-process read cache across a recall() call so load_index, expansion, and _brief don't re-fetch overlapping notes \u2014 this closes the remaining ~500 ms + serial _brief cost.",
|
||||
"Make 'prefer read_many over a GET loop' a documented contract in the plugin's API reference; serial multi-note reads are a performance bug. The expand-graph gate enforces it for recall.",
|
||||
"Hold ECHO_WORKERS at 8 (16 buys only 1.24x). Promote the resolve correctness table from INFO to FAIL now that live behavior is confirmed. Re-baseline at the next 1.x release or past ~400 notes."
|
||||
]
|
||||
},
|
||||
{
|
||||
"heading": "Method",
|
||||
"type": "text",
|
||||
"body": [
|
||||
"Each metric was timed with a perf-counter harness that reuses the live ECHO client, so it exercises the real pooled and concurrent code path. Reads report median and p95 over repeated iterations after a discarded warm-up; write metrics ran in a disposable namespace under the advisory lock and were deleted on completion (zero residual files confirmed).",
|
||||
"Gates use relative ratios rather than absolute milliseconds, so they stay valid regardless of the network the suite runs from. The harness, fixtures, baselines, and per-metric JSON are checked in under eval/perf/."
|
||||
]
|
||||
}
|
||||
],
|
||||
"note": {
|
||||
"heading": "Next Step",
|
||||
"body": "Prefetch recall()'s _brief reads and parallelize rebuild() via read_many, then re-run subject-pull to confirm recall drops toward ~1 s. expand_graph fix is local-only until ported to the canonical repo. No blockers in 1.1.0 / 1.2.0."
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
[20:14:00] read-full
|
||||
Reference in New Issue
Block a user