diff --git a/mempalace/cli.py b/mempalace/cli.py index d0f097e..80c8958 100644 --- a/mempalace/cli.py +++ b/mempalace/cli.py @@ -177,20 +177,31 @@ def cmd_compress(args): print(" Run: mempalace init then mempalace mine ") sys.exit(1) - # Query drawers in the wing + # Query drawers in batches to avoid SQLite variable limit (~999) where = {"wing": args.wing} if args.wing else None - try: - kwargs = {"include": ["documents", "metadatas"]} - if where: - kwargs["where"] = where - results = col.get(**kwargs) - except Exception as e: - print(f"\n Error reading drawers: {e}") - sys.exit(1) - - docs = results["documents"] - metas = results["metadatas"] - ids = results["ids"] + _BATCH = 500 + docs, metas, ids = [], [], [] + offset = 0 + while True: + try: + kwargs = {"include": ["documents", "metadatas"], "limit": _BATCH, "offset": offset} + if where: + kwargs["where"] = where + batch = col.get(**kwargs) + except Exception as e: + if not docs: + print(f"\n Error reading drawers: {e}") + sys.exit(1) + break + batch_docs = batch.get("documents", []) + if not batch_docs: + break + docs.extend(batch_docs) + metas.extend(batch.get("metadatas", [])) + ids.extend(batch.get("ids", [])) + offset += len(batch_docs) + if len(batch_docs) < _BATCH: + break if not docs: wing_label = f" in wing '{args.wing}'" if args.wing else "" diff --git a/mempalace/layers.py b/mempalace/layers.py index 2ae0947..6abb99b 100644 --- a/mempalace/layers.py +++ b/mempalace/layers.py @@ -96,18 +96,27 @@ class Layer1: except Exception: return "## L1 — No palace found. Run: mempalace mine " - # Fetch all drawers (with optional wing filter) - kwargs = {"include": ["documents", "metadatas"]} - if self.wing: - kwargs["where"] = {"wing": self.wing} - - try: - results = col.get(**kwargs) - except Exception: - return "## L1 — No drawers found." - - docs = results.get("documents", []) - metas = results.get("metadatas", []) + # Fetch all drawers in batches to avoid SQLite variable limit (~999) + _BATCH = 500 + docs, metas = [], [] + offset = 0 + while True: + kwargs = {"include": ["documents", "metadatas"], "limit": _BATCH, "offset": offset} + if self.wing: + kwargs["where"] = {"wing": self.wing} + try: + batch = col.get(**kwargs) + except Exception: + break + batch_docs = batch.get("documents", []) + batch_metas = batch.get("metadatas", []) + if not batch_docs: + break + docs.extend(batch_docs) + metas.extend(batch_metas) + offset += len(batch_docs) + if len(batch_docs) < _BATCH: + break if not docs: return "## L1 — No memories yet."