tessl/pypi-srsly
Modern high-performance serialization utilities for Python
msgpack.mddocs/
MessagePack Operations
Binary serialization format with numpy array support, providing compact, fast, and cross-language compatible serialization through an enhanced msgpack implementation.
Capabilities
MessagePack String Operations
Core MessagePack serialization functions for converting between Python objects and binary format.
def msgpack_dumps(data):
"""
Serialize an object to msgpack byte string.
Parameters:
- data: Python object to serialize (supports numpy arrays)
Returns:
bytes: Serialized msgpack data
"""
def msgpack_loads(data, use_list=True):
"""
Deserialize msgpack bytes to Python object.
Parameters:
- data (bytes): Msgpack data to deserialize
- use_list (bool): Use lists instead of tuples for arrays
Returns:
Any: Deserialized Python object
"""MessagePack File Operations
Read and write MessagePack data to files with automatic binary mode handling.
def read_msgpack(location, use_list=True):
"""
Load msgpack data from file.
Parameters:
- location (str | Path): Path to msgpack file
- use_list (bool): Use lists instead of tuples for arrays
Returns:
Any: Loaded and deserialized content
Raises:
ValueError: If file doesn't exist
"""
def write_msgpack(location, data):
"""
Write data to msgpack file.
Parameters:
- location (str | Path): Path for msgpack file
- data: Python object to serialize
Returns:
None
"""Advanced MessagePack Classes
Access to underlying packer and unpacker classes for custom serialization with numpy support.
class Packer:
"""
MessagePack packer with numpy array support.
Automatically encodes numpy arrays using msgpack-numpy extensions.
"""
def __init__(self, **kwargs):
"""
Initialize packer with options.
Parameters:
- default (callable): Custom serialization function
- use_bin_type (bool): Use binary type for bytes
- **kwargs: Additional packer options
"""
def pack(self, obj):
"""
Pack object to bytes.
Parameters:
- obj: Object to pack
Returns:
bytes: Packed data
"""
class Unpacker:
"""
MessagePack unpacker with numpy array support.
Automatically decodes numpy arrays using msgpack-numpy extensions.
"""
def __init__(self, **kwargs):
"""
Initialize unpacker with options.
Parameters:
- object_hook (callable): Custom deserialization function
- raw (bool): Keep raw bytes
- use_list (bool): Use lists instead of tuples
- **kwargs: Additional unpacker options
"""
class ExtType:
"""
Extended type for custom msgpack serialization.
Used to handle custom data types in msgpack format.
"""
def __init__(self, code, data):
"""
Initialize extended type.
Parameters:
- code (int): Type code (0-127)
- data (bytes): Type data
"""Stream and Compatibility Functions
Lower-level functions for streaming and compatibility with standard msgpack API.
def pack(obj, stream, **kwargs):
"""
Pack object and write to stream.
Parameters:
- obj: Object to pack
- stream: File-like object to write to
- **kwargs: Packer options
"""
def packb(obj, **kwargs):
"""
Pack object and return bytes (alias for msgpack_dumps).
Parameters:
- obj: Object to pack
- **kwargs: Packer options
Returns:
bytes: Packed data
"""
def unpack(stream, **kwargs):
"""
Unpack from stream.
Parameters:
- stream: File-like object to read from
- **kwargs: Unpacker options
Returns:
Any: Unpacked object
"""
def unpackb(packed, **kwargs):
"""
Unpack from bytes (alias for msgpack_loads).
Parameters:
- packed (bytes): Packed data
- **kwargs: Unpacker options
Returns:
Any: Unpacked object
"""
# Compatibility aliases
load = unpack
loads = unpackb
dump = pack
dumps = packbUsage Examples
Basic MessagePack Operations
import srsly
import numpy as np
# Serialize various data types
data = {
"text": "Hello, world!",
"numbers": [1, 2, 3, 4, 5],
"nested": {"key": "value"},
"array": np.array([1.0, 2.0, 3.0]) # Numpy arrays supported
}
# Pack to bytes
packed = srsly.msgpack_dumps(data)
print(f"Packed size: {len(packed)} bytes")
# Unpack from bytes
unpacked = srsly.msgpack_loads(packed)
print(f"Original array: {unpacked['array']}")
print(f"Array type: {type(unpacked['array'])}") # numpy.ndarrayFile I/O Operations
import srsly
import numpy as np
# Prepare data with numpy arrays
scientific_data = {
"experiment": "test_001",
"measurements": np.random.random((100, 3)),
"parameters": {"temperature": 23.5, "pressure": 1013.25},
"metadata": {
"timestamp": "2023-01-01T12:00:00Z",
"researcher": "Dr. Smith"
}
}
# Save to msgpack file
srsly.write_msgpack("experiment.msgpack", scientific_data)
# Load from msgpack file
loaded_data = srsly.read_msgpack("experiment.msgpack")
print(f"Experiment: {loaded_data['experiment']}")
print(f"Measurements shape: {loaded_data['measurements'].shape}")Advanced Usage with Custom Classes
import srsly.msgpack as msgpack
import numpy as np
# Direct access to advanced features
data = {
"large_array": np.random.random((1000, 1000)),
"metadata": {"version": 1, "created": "2023-01-01"}
}
# Use custom packer for fine control
packer = msgpack.Packer(use_bin_type=True)
packed_data = packer.pack(data)
# Use custom unpacker
unpacker = msgpack.Unpacker(raw=False, use_list=True)
unpacker.feed(packed_data)
for unpacked in unpacker:
print(f"Loaded array shape: {unpacked['large_array'].shape}")
breakStream Processing
import srsly.msgpack as msgpack
import io
# Stream multiple objects
buffer = io.BytesIO()
# Pack multiple objects to stream
objects = [{"id": i, "data": f"item_{i}"} for i in range(5)]
for obj in objects:
msgpack.pack(obj, buffer)
# Read back from stream
buffer.seek(0)
loaded_objects = []
try:
while True:
obj = msgpack.unpack(buffer)
loaded_objects.append(obj)
except:
pass # End of stream
print(f"Loaded {len(loaded_objects)} objects")
for obj in loaded_objects:
print(f"ID: {obj['id']}, Data: {obj['data']}")