fix(status): paginate metadata fetch to support large palaces
`col.get(limit=total)` causes SQLite "too many SQL variables" on palaces with >10k drawers (#802) and on older versions the hardcoded limit=10000 silently truncated the count (#850). Paginate in 5k batches using offset and aggregate wing/room counts incrementally. Also use `col.count()` for the header instead of `len(metas)` so the displayed total is always correct. Tested on a 122,686-drawer palace. Fixes #850 Related: #802, #723
This commit is contained in:
+14
-8
@@ -821,17 +821,23 @@ def status(palace_path: str):
|
||||
print(" Run: mempalace init <dir> then mempalace mine <dir>")
|
||||
return
|
||||
|
||||
# Count by wing and room
|
||||
# Count by wing and room — paginate to avoid SQLite "too many SQL
|
||||
# variables" error on large palaces (see #802, #850).
|
||||
total = col.count()
|
||||
r = col.get(limit=total, include=["metadatas"]) if total else {"metadatas": []}
|
||||
metas = r["metadatas"]
|
||||
|
||||
wing_rooms = defaultdict(lambda: defaultdict(int))
|
||||
for m in metas:
|
||||
wing_rooms[m.get("wing", "?")][m.get("room", "?")] += 1
|
||||
wing_rooms: dict = defaultdict(lambda: defaultdict(int))
|
||||
batch_size = 5000
|
||||
offset = 0
|
||||
while offset < total:
|
||||
r = col.get(limit=batch_size, offset=offset, include=["metadatas"])
|
||||
batch = r["metadatas"]
|
||||
if not batch:
|
||||
break
|
||||
for m in batch:
|
||||
wing_rooms[m.get("wing", "?")][m.get("room", "?")] += 1
|
||||
offset += len(batch)
|
||||
|
||||
print(f"\n{'=' * 55}")
|
||||
print(f" MemPalace Status — {len(metas)} drawers")
|
||||
print(f" MemPalace Status — {total} drawers")
|
||||
print(f"{'=' * 55}\n")
|
||||
for wing, rooms in sorted(wing_rooms.items()):
|
||||
print(f" WING: {wing}")
|
||||
|
||||
Reference in New Issue
Block a user