Persistent dict in Python, backed up by sqlite3 and pickle, multithread-safe.
npx @tessl/cli install tessl/pypi-sqlitedict@1.7.0A lightweight wrapper around Python's sqlite3 database with a simple, Pythonic dict-like interface and support for multi-thread access. SqliteDict enables persistent storage of arbitrary pickle-able objects as values with string keys, backed by SQLite database files.
pip install sqlitedictfrom sqlitedict import SqliteDictAlternative import patterns:
import sqlitedict
# Use sqlitedict.SqliteDict() or sqlitedict.open()For version access and module constants:
import sqlitedict
print(sqlitedict.__version__) # '1.7.0'
print(sqlitedict.PICKLE_PROTOCOL) # Highest pickle protocol availablefrom sqlitedict import SqliteDict
# Create a persistent dictionary (autocommit for immediate persistence)
mydict = SqliteDict('./my_db.sqlite', autocommit=True)
mydict['some_key'] = any_picklable_object
print(mydict['some_key'])
print(len(mydict)) # all dict functions work
# Context manager usage (recommended for transaction control)
with SqliteDict('./my_db.sqlite') as mydict: # no autocommit
mydict['some_key'] = "first value"
mydict['another_key'] = range(10)
mydict.commit() # explicit commit
mydict['some_key'] = "new value"
# no explicit commit here - changes lost on exit
# Re-open same database
with SqliteDict('./my_db.sqlite') as mydict:
print(mydict['some_key']) # outputs 'first value', not 'new value'SqliteDict provides thread-safe access to SQLite databases through an internal queuing system. The architecture includes:
Standard Python dictionary operations with persistence to SQLite database.
class SqliteDict:
def __init__(self, filename=None, tablename='unnamed', flag='c',
autocommit=False, journal_mode="DELETE", encode=encode, decode=decode):
"""
Initialize a thread-safe sqlite-backed dictionary.
Parameters:
- filename (str, optional): Database file path. If None, uses temporary file
- tablename (str): Table name within database (default: 'unnamed')
- flag (str): Access mode - 'c' (create/read/write), 'r' (readonly),
'w' (overwrite table), 'n' (new database)
- autocommit (bool): Auto-commit after each operation (default: False)
- journal_mode (str): SQLite journal mode (default: "DELETE")
- encode (function): Custom encoding function (default: pickle-based)
- decode (function): Custom decoding function (default: pickle-based)
"""
def __getitem__(self, key):
"""Get value by key. Raises KeyError if key not found."""
def __setitem__(self, key, value):
"""Set value by key. Replaces existing value."""
def __delitem__(self, key):
"""Delete key-value pair. Raises KeyError if key not found."""
def __contains__(self, key):
"""Check if key exists in dictionary."""
def __len__(self):
"""Return number of items. Note: slow for large databases."""
def __bool__(self):
"""Return True if dictionary has any items."""
def __iter__(self):
"""Return iterator over keys."""
def keys(self):
"""Return view/list of all keys."""
def values(self):
"""Return view/list of all values (deserialized)."""
def items(self):
"""Return view/list of (key, value) tuples."""
def iterkeys(self):
"""Return iterator over keys (Python 2 compatibility)."""
def itervalues(self):
"""Return iterator over values (Python 2 compatibility)."""
def iteritems(self):
"""Return iterator over (key, value) tuples (Python 2 compatibility)."""
def update(self, items=(), **kwds):
"""Update dictionary with items from mapping or iterable."""
def clear(self):
"""Remove all items from table."""Transaction control and database lifecycle management.
class SqliteDict:
def commit(self, blocking=True):
"""
Persist all data to disk.
Parameters:
- blocking (bool): Wait for completion when True, queue when False
"""
def close(self, do_log=True, force=False):
"""
Close database connection and cleanup resources.
Parameters:
- do_log (bool): Whether to log the close operation
- force (bool): Force close even if operations pending
"""
def terminate(self):
"""Close connection and delete underlying database file permanently."""
def sync(self):
"""Alias for commit() method."""Automatic resource management and transaction handling.
class SqliteDict:
def __enter__(self):
"""Context manager entry. Returns self."""
def __exit__(self, *exc_info):
"""Context manager exit. Automatically calls close()."""Database introspection and factory functions.
def open(*args, **kwargs):
"""
Factory function to create SqliteDict instance.
Returns:
SqliteDict instance with same parameters as constructor
"""
class SqliteDict:
@staticmethod
def get_tablenames(filename):
"""
Get list of table names in SQLite database file.
Parameters:
- filename (str): Path to SQLite database file
Returns:
List of table name strings
Raises:
IOError: If file does not exist
"""Default and custom serialization functions for data persistence.
def encode(obj):
"""
Default encoding function using pickle.
Parameters:
- obj: Any pickle-able Python object
Returns:
sqlite3.Binary object suitable for SQLite storage
"""
def decode(obj):
"""
Default decoding function using pickle.
Parameters:
- obj: Binary data from SQLite
Returns:
Deserialized Python object
"""
# Lower-level pickle functions (also exposed by module)
def dumps(obj, protocol):
"""
Serialize object using pickle with specified protocol.
Parameters:
- obj: Object to serialize
- protocol: Pickle protocol version (typically PICKLE_PROTOCOL constant)
Returns:
Bytes object containing pickled data
"""
def loads(data):
"""
Deserialize object from pickle data.
Parameters:
- data: Bytes object containing pickled data
Returns:
Deserialized Python object
"""Display and debugging support.
class SqliteDict:
def __str__(self):
"""Return string representation of SqliteDict instance."""
def __repr__(self):
"""Return developer-friendly string representation."""class SqliteDict:
"""
Main dictionary-like class with SQLite persistence.
Inherits from UserDict (Python 3) or DictMixin (Python 2) to provide
standard dictionary interface. All standard dict operations are supported.
"""
VALID_FLAGS = ['c', 'r', 'w', 'n'] # Valid flag values for constructor
class SqliteMultithread(Thread):
"""Internal thread-safe SQLite wrapper (not typically used directly)."""
# Module-level constants
__version__ = str # Version string, e.g. '1.7.0'
PICKLE_PROTOCOL = int # Highest available pickle protocol for current Python versionimport json
from sqlitedict import SqliteDict
# Use JSON instead of pickle
mydict = SqliteDict('./my_db.sqlite', encode=json.dumps, decode=json.loads)
mydict['data'] = {'key': 'value', 'number': 42}import zlib
import pickle
import sqlite3
from sqlitedict import SqliteDict
def compress_encode(obj):
return sqlite3.Binary(zlib.compress(pickle.dumps(obj, pickle.HIGHEST_PROTOCOL)))
def compress_decode(obj):
return pickle.loads(zlib.decompress(bytes(obj)))
mydict = SqliteDict('./compressed_db.sqlite', encode=compress_encode, decode=compress_decode)from sqlitedict import SqliteDict
# Different tables in same database file
users = SqliteDict('./app.db', tablename='users', autocommit=True)
sessions = SqliteDict('./app.db', tablename='sessions', autocommit=True)
users['user123'] = {'name': 'Alice', 'email': 'alice@example.com'}
sessions['session456'] = {'user_id': 'user123', 'expires': '2024-12-31'}
# List all tables in database
tables = SqliteDict.get_tablenames('./app.db')
print(tables) # ['users', 'sessions']from sqlitedict import SqliteDict
# Open existing database in read-only mode
readonly_dict = SqliteDict('./existing.db', flag='r')
value = readonly_dict['some_key']
# readonly_dict['key'] = 'value' # Would raise RuntimeErrorfrom sqlitedict import SqliteDict
# For batch operations, disable autocommit
with SqliteDict('./batch.db', autocommit=False) as batch_dict:
for i in range(1000):
batch_dict[f'key_{i}'] = f'value_{i}'
batch_dict.commit() # Single commit for all operations
# For frequent single operations, enable autocommit
frequent_dict = SqliteDict('./frequent.db', autocommit=True)
frequent_dict['immediate'] = 'persisted_immediately'Common exceptions and error patterns:
__getitem__, __delitem__get_tablenames() when database file doesn't existSqliteDict is thread-safe and allows concurrent access from multiple threads to the same instance. However, concurrent requests are serialized internally, so multi-threading doesn't provide performance benefits - it's a workaround for SQLite's threading limitations in Python.
Mutable Object Limitations: SqliteDict cannot detect when mutable objects are modified in-place. Always reassign modified objects:
# Wrong - changes not persisted
val = mydict.get('key', [])
val.append(1) # SQLite DB not updated
# Correct - explicitly reassign
val = mydict.get('key', [])
val.append(1)
mydict['key'] = val # Now updated in databasePerformance: len() operation is slow for large databases as it performs a full table scan.
SqliteDict supports Python 2.5+ and Python 3.3+. Key compatibility considerations:
keys(), values(), items() return views in Python 3, lists in Python 2iterkeys(), itervalues(), iteritems() provided for Python 2 compatibility