fix: auto-repair BLOB seq_ids from chromadb 0.6→1.5 migration (#664)

Note from code review: (1) silent exception swallow on migration failure means caller proceeds with potentially corrupt DB — consider returning a boolean or re-raising in a follow-up. (2) No blob length validation before int.from_bytes — malformed rows could produce wrong seq_id values. Both are edge cases; the fix is still valuable for the common chromadb 0.6→1.5 migration path.
This commit is contained in:
Jeffrey Hein
2026-04-11 23:06:01 -07:00
committed by GitHub
parent 5b67e0740e
commit abc99f4154
2 changed files with 86 additions and 1 deletions
+37
View File
@@ -1,11 +1,47 @@
"""ChromaDB-backed MemPalace collection adapter."""
import logging
import os
import sqlite3
import chromadb
from .base import BaseCollection
logger = logging.getLogger(__name__)
def _fix_blob_seq_ids(palace_path: str):
"""Fix ChromaDB 0.6.x -> 1.5.x migration bug: BLOB seq_ids -> INTEGER.
ChromaDB 0.6.x stored seq_id as big-endian 8-byte BLOBs. ChromaDB 1.5.x
expects INTEGER. The auto-migration doesn't convert existing rows, causing
the Rust compactor to crash with "mismatched types; Rust type u64 (as SQL
type INTEGER) is not compatible with SQL type BLOB".
Must run BEFORE PersistentClient is created (the compactor fires on init).
"""
db_path = os.path.join(palace_path, "chroma.sqlite3")
if not os.path.isfile(db_path):
return
try:
with sqlite3.connect(db_path) as conn:
for table in ("embeddings", "max_seq_id"):
try:
rows = conn.execute(
f"SELECT rowid, seq_id FROM {table} WHERE typeof(seq_id) = 'blob'"
).fetchall()
except sqlite3.OperationalError:
continue
if not rows:
continue
updates = [(int.from_bytes(blob, byteorder="big"), rowid) for rowid, blob in rows]
conn.executemany(f"UPDATE {table} SET seq_id = ? WHERE rowid = ?", updates)
logger.info("Fixed %d BLOB seq_ids in %s", len(updates), table)
conn.commit()
except Exception:
logger.exception("Could not fix BLOB seq_ids in %s", db_path)
class ChromaCollection(BaseCollection):
"""Thin adapter over a ChromaDB collection."""
@@ -46,6 +82,7 @@ class ChromaBackend:
except (OSError, NotImplementedError):
pass
_fix_blob_seq_ids(palace_path)
client = chromadb.PersistentClient(path=palace_path)
if create:
collection = client.get_or_create_collection(collection_name)