fix(status): paginate metadata fetch to support large palaces
`col.get(limit=total)` causes SQLite "too many SQL variables" on palaces with >10k drawers (#802) and on older versions the hardcoded limit=10000 silently truncated the count (#850). Paginate in 5k batches using offset and aggregate wing/room counts incrementally. Also use `col.count()` for the header instead of `len(metas)` so the displayed total is always correct. Tested on a 122,686-drawer palace. Fixes #850 Related: #802, #723
This commit is contained in:
+14
-8
@@ -821,17 +821,23 @@ def status(palace_path: str):
|
|||||||
print(" Run: mempalace init <dir> then mempalace mine <dir>")
|
print(" Run: mempalace init <dir> then mempalace mine <dir>")
|
||||||
return
|
return
|
||||||
|
|
||||||
# Count by wing and room
|
# Count by wing and room — paginate to avoid SQLite "too many SQL
|
||||||
|
# variables" error on large palaces (see #802, #850).
|
||||||
total = col.count()
|
total = col.count()
|
||||||
r = col.get(limit=total, include=["metadatas"]) if total else {"metadatas": []}
|
wing_rooms: dict = defaultdict(lambda: defaultdict(int))
|
||||||
metas = r["metadatas"]
|
batch_size = 5000
|
||||||
|
offset = 0
|
||||||
wing_rooms = defaultdict(lambda: defaultdict(int))
|
while offset < total:
|
||||||
for m in metas:
|
r = col.get(limit=batch_size, offset=offset, include=["metadatas"])
|
||||||
wing_rooms[m.get("wing", "?")][m.get("room", "?")] += 1
|
batch = r["metadatas"]
|
||||||
|
if not batch:
|
||||||
|
break
|
||||||
|
for m in batch:
|
||||||
|
wing_rooms[m.get("wing", "?")][m.get("room", "?")] += 1
|
||||||
|
offset += len(batch)
|
||||||
|
|
||||||
print(f"\n{'=' * 55}")
|
print(f"\n{'=' * 55}")
|
||||||
print(f" MemPalace Status — {len(metas)} drawers")
|
print(f" MemPalace Status — {total} drawers")
|
||||||
print(f"{'=' * 55}\n")
|
print(f"{'=' * 55}\n")
|
||||||
for wing, rooms in sorted(wing_rooms.items()):
|
for wing, rooms in sorted(wing_rooms.items()):
|
||||||
print(f" WING: {wing}")
|
print(f" WING: {wing}")
|
||||||
|
|||||||
Reference in New Issue
Block a user