tessl/pypi-zarr

An implementation of chunked, compressed, N-dimensional arrays for Python

Overview

Eval results

Files

Storage Backends

Name: tessl/pypi-zarr
Author: tessl

Storage backend classes for persisting zarr data across different storage systems. These provide the flexibility to use zarr with various storage infrastructures from local filesystems to cloud object stores.

Capabilities

Local and Memory Storage

class LocalStore:
    """Local filesystem storage backend."""
    
    def __init__(
        self,
        root: Union[str, os.PathLike],
        normalize_keys: bool = True,
        map_size: int = 2**26,
        **kwargs
    ): ...
    
    def __getitem__(self, key: str) -> bytes: ...
    def __setitem__(self, key: str, value: bytes) -> None: ...
    def __delitem__(self, key: str) -> None: ...
    def __contains__(self, key: str) -> bool: ...
    def __iter__(self) -> Iterator[str]: ...
    def __len__(self) -> int: ...
    
    def list(self) -> list[str]: ...
    def list_prefix(self, prefix: str) -> list[str]: ...
    def list_dir(self, prefix: str = "") -> list[str]: ...

class MemoryStore:
    """In-memory storage backend for temporary arrays."""
    
    def __init__(
        self,
        normalize_keys: bool = True,
        **kwargs
    ): ...
    
    def __getitem__(self, key: str) -> bytes: ...
    def __setitem__(self, key: str, value: bytes) -> None: ...
    def __delitem__(self, key: str) -> None: ...
    def __contains__(self, key: str) -> bool: ...
    def __iter__(self) -> Iterator[str]: ...
    def __len__(self) -> int: ...
    
    def clear(self) -> None: ...

class GpuMemoryStore:
    """GPU memory storage backend using CuPy arrays."""
    
    def __init__(
        self,
        normalize_keys: bool = True,
        **kwargs
    ): ...
    
    # Same interface as MemoryStore but uses GPU memory

Archive Storage

class ZipStore:
    """ZIP file storage backend."""
    
    def __init__(
        self,
        path: Union[str, os.PathLike, io.IOBase],
        mode: str = 'r',
        compression: int = zipfile.ZIP_STORED,
        allowZip64: bool = True,
        **kwargs
    ): ...
    
    def __getitem__(self, key: str) -> bytes: ...
    def __setitem__(self, key: str, value: bytes) -> None: ...
    def __delitem__(self, key: str) -> None: ...  
    def __contains__(self, key: str) -> bool: ...
    def __iter__(self) -> Iterator[str]: ...
    
    def close(self) -> None: ...
    def flush(self) -> None: ...

Cloud and Network Storage

class FsspecStore:
    """Storage backend using fsspec for various filesystems."""
    
    def __init__(
        self,
        url: str,
        normalize_keys: bool = True,
        key_separator: str = '/',
        **storage_options
    ): ...
    
    def __getitem__(self, key: str) -> bytes: ...
    def __setitem__(self, key: str, value: bytes) -> None: ...
    def __delitem__(self, key: str) -> None: ...
    def __contains__(self, key: str) -> bool: ...
    def __iter__(self) -> Iterator[str]: ...
    
    @property
    def fs(self) -> fsspec.AbstractFileSystem: ...
    @property 
    def path(self) -> str: ...

class ObjectStore:
    """High-performance cloud object storage backend."""
    
    def __init__(
        self,
        url: str,
        normalize_keys: bool = True,
        **kwargs
    ): ...
    
    def __getitem__(self, key: str) -> bytes: ...
    def __setitem__(self, key: str, value: bytes) -> None: ...
    def __delitem__(self, key: str) -> None: ...
    def __contains__(self, key: str) -> bool: ...
    def __iter__(self) -> Iterator[str]: ...
    
    async def get_async(self, key: str) -> bytes: ...
    async def set_async(self, key: str, value: bytes) -> None: ...

Store Wrappers and Utilities

class LoggingStore:
    """Wrapper store that logs all operations for debugging."""
    
    def __init__(
        self,
        store: Store,
        log: logging.Logger = None,
        log_level: int = logging.INFO
    ): ...
    
    # Wraps all store operations with logging

class WrapperStore:
    """Base class for creating store wrappers."""
    
    def __init__(self, store: Store): ...
    
    @property
    def inner_store(self) -> Store: ...

Store Path Utilities

class StorePath:
    """Path-like interface for store locations."""
    
    def __init__(
        self,
        store: Store,
        path: str = ""
    ): ...
    
    @property
    def store(self) -> Store: ...
    @property
    def path(self) -> str: ...
    @property
    def name(self) -> str: ...
    @property  
    def suffix(self) -> str: ...
    @property
    def parent(self) -> StorePath: ...
    
    def __truediv__(self, other: str) -> StorePath: ...
    def __str__(self) -> str: ...

Type Definitions

StoreLike = Union[str, os.PathLike, Store, MutableMapping]
Store = Union[LocalStore, MemoryStore, ZipStore, FsspecStore, ObjectStore, MutableMapping]

Usage Examples

Local Filesystem Storage

import zarr
from zarr.storage import LocalStore

# Create local store
store = LocalStore('path/to/zarr/data')

# Use with zarr operations
arr = zarr.create_array(store, shape=(1000, 1000), chunks=(100, 100))
arr[:100, :100] = 1.0

# Direct store access
store['array_metadata/.zarray'] = b'{"shape": [1000, 1000]}'
metadata_bytes = store['array_metadata/.zarray']

Memory Storage

from zarr.storage import MemoryStore

# Create in-memory store
store = MemoryStore()

# Create temporary arrays
temp_arr = zarr.create_array(store, shape=(100, 100))
temp_arr[:] = 42

# Memory stores are automatically cleaned up

ZIP File Storage

from zarr.storage import ZipStore

# Create ZIP store for archival
with ZipStore('dataset.zip', mode='w') as store:
    arr1 = zarr.create_array(store, 'array1', shape=(500, 500))
    arr2 = zarr.create_array(store, 'array2', shape=(300, 300))
    
    # Data is compressed in ZIP format
    arr1[:] = np.random.random((500, 500))
    arr2[:] = np.random.random((300, 300))

# Read from ZIP store
with ZipStore('dataset.zip', mode='r') as store:
    arr1 = zarr.open_array(store, 'array1')
    data = arr1[:]

Cloud Storage with Fsspec

from zarr.storage import FsspecStore

# S3 storage
s3_store = FsspecStore(
    's3://my-bucket/zarr-data/',
    key='aws_access_key_id',
    secret='aws_secret_access_key'
)

# Create array in S3
arr = zarr.create_array(s3_store, shape=(10000, 10000), chunks=(1000, 1000))

# Google Cloud Storage
gcs_store = FsspecStore('gs://my-bucket/zarr-data/')
arr = zarr.create_array(gcs_store, shape=(5000, 5000))

# HTTP storage (read-only)
http_store = FsspecStore('https://example.com/data.zarr')
arr = zarr.open_array(http_store)

High-Performance Object Storage

from zarr.storage import ObjectStore

# Fast cloud object storage
store = ObjectStore('s3://bucket/path/')

# Supports async operations for better performance  
import asyncio

async def async_write():
    await store.set_async('data/chunk.0.0', chunk_data)
    
asyncio.run(async_write())

Store Composition and Wrapping

from zarr.storage import LoggingStore, LocalStore
import logging

# Add logging to any store
base_store = LocalStore('data/')
logged_store = LoggingStore(base_store, log_level=logging.DEBUG)

# All operations will be logged
arr = zarr.create_array(logged_store, shape=(100, 100))

Store Path Navigation

from zarr.storage import StorePath, LocalStore

store = LocalStore('root/data/')
path = StorePath(store, 'experiments/run1')

# Path-like operations
sub_path = path / 'results' / 'array1'
parent = sub_path.parent

# Use with zarr
arr = zarr.create_array(sub_path, shape=(100, 100))

Store Configuration for Different Use Cases

# High-performance local storage
fast_store = LocalStore('/fast/ssd/path', map_size=2**30)

# Network-optimized storage
network_store = FsspecStore(
    's3://bucket/data/',
    cache_type='blockcache',
    block_size=1024*1024
)

# Memory-efficient storage
efficient_store = LocalStore('/path', normalize_keys=False)

Install with Tessl CLI