diff --git a/mempalace/cli.py b/mempalace/cli.py
index d0f097e..80c8958 100644
--- a/mempalace/cli.py
+++ b/mempalace/cli.py
@@ -177,20 +177,31 @@ def cmd_compress(args):
print(" Run: mempalace init
then mempalace mine ")
sys.exit(1)
- # Query drawers in the wing
+ # Query drawers in batches to avoid SQLite variable limit (~999)
where = {"wing": args.wing} if args.wing else None
- try:
- kwargs = {"include": ["documents", "metadatas"]}
- if where:
- kwargs["where"] = where
- results = col.get(**kwargs)
- except Exception as e:
- print(f"\n Error reading drawers: {e}")
- sys.exit(1)
-
- docs = results["documents"]
- metas = results["metadatas"]
- ids = results["ids"]
+ _BATCH = 500
+ docs, metas, ids = [], [], []
+ offset = 0
+ while True:
+ try:
+ kwargs = {"include": ["documents", "metadatas"], "limit": _BATCH, "offset": offset}
+ if where:
+ kwargs["where"] = where
+ batch = col.get(**kwargs)
+ except Exception as e:
+ if not docs:
+ print(f"\n Error reading drawers: {e}")
+ sys.exit(1)
+ break
+ batch_docs = batch.get("documents", [])
+ if not batch_docs:
+ break
+ docs.extend(batch_docs)
+ metas.extend(batch.get("metadatas", []))
+ ids.extend(batch.get("ids", []))
+ offset += len(batch_docs)
+ if len(batch_docs) < _BATCH:
+ break
if not docs:
wing_label = f" in wing '{args.wing}'" if args.wing else ""
diff --git a/mempalace/layers.py b/mempalace/layers.py
index 2ae0947..6abb99b 100644
--- a/mempalace/layers.py
+++ b/mempalace/layers.py
@@ -96,18 +96,27 @@ class Layer1:
except Exception:
return "## L1 — No palace found. Run: mempalace mine "
- # Fetch all drawers (with optional wing filter)
- kwargs = {"include": ["documents", "metadatas"]}
- if self.wing:
- kwargs["where"] = {"wing": self.wing}
-
- try:
- results = col.get(**kwargs)
- except Exception:
- return "## L1 — No drawers found."
-
- docs = results.get("documents", [])
- metas = results.get("metadatas", [])
+ # Fetch all drawers in batches to avoid SQLite variable limit (~999)
+ _BATCH = 500
+ docs, metas = [], []
+ offset = 0
+ while True:
+ kwargs = {"include": ["documents", "metadatas"], "limit": _BATCH, "offset": offset}
+ if self.wing:
+ kwargs["where"] = {"wing": self.wing}
+ try:
+ batch = col.get(**kwargs)
+ except Exception:
+ break
+ batch_docs = batch.get("documents", [])
+ batch_metas = batch.get("metadatas", [])
+ if not batch_docs:
+ break
+ docs.extend(batch_docs)
+ metas.extend(batch_metas)
+ offset += len(batch_docs)
+ if len(batch_docs) < _BATCH:
+ break
if not docs:
return "## L1 — No memories yet."