tessl/pypi-smmap

A pure Python implementation of a sliding window memory map manager

—

Pending

Overview

Eval results

Files

Utilities

Name: tessl/pypi-smmap
Author: tessl

Helper functions for memory alignment and system detection, plus utility classes for window management and memory region tracking. These components support the core memory mapping functionality with cross-platform compatibility and resource management.

Capabilities

Memory Alignment

Function for aligning memory offsets and sizes to system page boundaries, required for efficient memory mapping operations.

def align_to_mmap(num, round_up):
    """
    Align integer to closest memory page boundary.
    
    Parameters:
    - num (int): Number to align to page boundary
    - round_up (bool): If True, round up to next page boundary.
                      If False, round down to previous page boundary.
    
    Returns:
    int: Number aligned to page boundary (typically 4096-byte multiples)
    
    Example:
    - align_to_mmap(1, True) returns 4096
    - align_to_mmap(1, False) returns 0
    - align_to_mmap(4096, True) returns 4096
    - align_to_mmap(4097, False) returns 4096
    """

Usage Example

import smmap

# Align offsets for optimal memory mapping
offset = 1234
size = 5678

# Align start offset down to page boundary
aligned_offset = smmap.align_to_mmap(offset, False)
print(f"Original: {offset}, Aligned: {aligned_offset}")  # Original: 1234, Aligned: 0

# Adjust size to maintain coverage and align to page boundary
adjusted_size = size + (offset - aligned_offset)
aligned_size = smmap.align_to_mmap(adjusted_size, True)
print(f"Adjusted size: {aligned_size}")  # Aligned to next page boundary

# Use aligned values for memory mapping
cursor.use_region(offset=aligned_offset, size=aligned_size)

System Architecture Detection

Function to detect system architecture for optimizing memory management strategies.

def is_64_bit():
    """
    Detect if system is 64-bit architecture.
    
    Returns:
    bool: True if 64-bit system, False if 32-bit system
    
    Note: Used internally to select appropriate default window sizes
    and memory limits. 64-bit systems can handle larger memory mappings.
    """

Usage Example

import smmap

# Configure memory manager based on architecture
if smmap.is_64_bit():
    # 64-bit: Use larger windows and memory limits
    window_size = 1024 * 1024 * 1024  # 1GB windows
    max_memory = 8 * 1024 * 1024 * 1024  # 8GB limit
else:
    # 32-bit: Use conservative settings
    window_size = 64 * 1024 * 1024  # 64MB windows  
    max_memory = 1 * 1024 * 1024 * 1024  # 1GB limit

manager = smmap.SlidingWindowMapManager(
    window_size=window_size,
    max_memory_size=max_memory
)

Window Management

Utility class for calculating and adjusting memory window positions and sizes.

class MapWindow:
    def __init__(self, offset, size):
        """
        Initialize memory window specification.
        
        Parameters:
        - offset (int): Offset into file in bytes
        - size (int): Window size in bytes
        """
    
    def ofs_end(self):
        """
        Get end offset of window.
        
        Returns:
        int: Offset to byte after window end
        """
    
    def align(self):
        """
        Align window to memory page boundaries.
        
        Adjusts offset down to page boundary and size up to
        maintain coverage while ensuring page alignment.
        """
    
    def extend_left_to(self, window, max_size):
        """
        Extend window leftward toward another window.
        
        Parameters:
        - window (MapWindow): Left neighboring window
        - max_size (int): Maximum allowed window size
        
        Adjusts this window to start where the left window ends,
        but doesn't exceed max_size. Maintains coverage of original area.
        """
    
    def extend_right_to(self, window, max_size):
        """
        Extend window rightward toward another window.
        
        Parameters:
        - window (MapWindow): Right neighboring window  
        - max_size (int): Maximum allowed window size
        
        Adjusts this window to end where the right window begins,
        but doesn't exceed max_size.
        """
    
    @classmethod
    def from_region(cls, region):
        """
        Create window from existing memory region.
        
        Parameters:
        - region (MapRegion): Existing mapped region
        
        Returns:
        MapWindow: Window specification matching the region
        """

Memory Region Management

Low-level class representing a mapped memory region with automatic resource cleanup.

class MapRegion:
    def __init__(self, path_or_fd, ofs, size, flags=0):
        """
        Initialize memory-mapped region.
        
        Parameters:
        - path_or_fd (str | int): File path or open file descriptor
        - ofs (int): Aligned offset into file (must be page-aligned)
        - size (int): Size to map. If larger than file, maps to end of file.
        - flags (int): Additional flags for os.open
        
        Raises:
        Exception: If memory cannot be allocated or file cannot be opened
        """
    
    def buffer(self):
        """
        Get memory buffer for mapped region.
        
        Returns:
        mmap.mmap: Memory map buffer object
        """
    
    def map(self):
        """
        Get memory map object.
        
        Returns:
        mmap.mmap: Same as buffer() - the underlying memory map
        """
    
    def ofs_begin(self):
        """
        Get absolute offset to first byte of mapping.
        
        Returns:
        int: File offset in bytes
        """
    
    def ofs_end(self):
        """
        Get absolute offset to byte after mapping end.
        
        Returns:
        int: File offset in bytes
        """
    
    def size(self):
        """
        Get size of mapped region.
        
        Returns:
        int: Size in bytes
        """
    
    def includes_ofs(self, ofs):
        """
        Check if offset is within mapped region.
        
        Parameters:
        - ofs (int): Absolute file offset to check
        
        Returns:
        bool: True if offset is accessible through this region
        """
    
    def client_count(self):
        """
        Get number of clients using this region.
        
        Returns:
        int: Reference count
        """
    
    def increment_client_count(self, ofs=1):
        """
        Adjust client reference count.
        
        Parameters:
        - ofs (int): Increment (positive) or decrement (negative)
        
        Returns:
        bool: True if region was released (count reached 0)
        """
    
    def release(self):
        """
        Release mapped memory resources.
        
        Note: Should only be called when client_count() is zero.
        Called automatically when reference count reaches zero.
        """

Region List Management

Container class for managing multiple memory regions associated with a single file.

class MapRegionList(list):
    def __init__(self, path_or_fd):
        """
        Initialize region list for a file.
        
        Parameters:
        - path_or_fd (str | int): File path or descriptor
        """
    
    def path_or_fd(self):
        """
        Get file path or descriptor for this region list.
        
        Returns:
        str | int: File path or file descriptor
        """
    
    def file_size(self):
        """
        Get total size of associated file.
        
        Returns:
        int: File size in bytes
        
        Note: Size is cached after first access for performance.
        """

System Constants

ALLOCATIONGRANULARITY: int

System memory allocation granularity constant imported from Python's mmap module. Represents the page size used for memory alignment (typically 4096 bytes on most systems, 65536 on some Windows systems).

Usage Example

import smmap

print(f"System page size: {smmap.ALLOCATIONGRANULARITY} bytes")

# Use for manual alignment calculations
offset = 12345
aligned_offset = (offset // smmap.ALLOCATIONGRANULARITY) * smmap.ALLOCATIONGRANULARITY
print(f"Manual alignment: {offset} -> {aligned_offset}")

# Or use the convenience function
aligned_offset = smmap.align_to_mmap(offset, False)
print(f"Function alignment: {offset} -> {aligned_offset}")

Advanced Usage Patterns

Custom Window Management

import smmap

# Create custom window layout for optimal access
window1 = smmap.MapWindow(0, 1024*1024)  # 1MB at start
window2 = smmap.MapWindow(10*1024*1024, 512*1024)  # 512KB at 10MB

# Align windows for efficient mapping  
window1.align()
window2.align()

# Create gap-filling window
gap_window = smmap.MapWindow(window1.ofs_end(), 
                            window2.ofs_begin() - window1.ofs_end())
gap_window.align()

print(f"Window 1: {window1.ofs} - {window1.ofs_end()}")
print(f"Gap: {gap_window.ofs} - {gap_window.ofs_end()}")
print(f"Window 2: {window2.ofs} - {window2.ofs_end()}")

Region Information

# Access low-level region information
cursor = manager.make_cursor('/path/to/file.dat')
cursor.use_region(offset=0, size=1024*1024)

if cursor.is_valid():
    region = cursor.region()
    print(f"Region covers: {region.ofs_begin()} to {region.ofs_end()}")
    print(f"Region size: {region.size()} bytes")
    print(f"Client count: {region.client_count()}")
    print(f"Includes offset 512000: {region.includes_ofs(512000)}")

Architecture-Aware Configuration

import smmap
import sys

# Configure based on system capabilities
def create_optimized_manager():
    if smmap.is_64_bit():
        # 64-bit: aggressive settings
        return smmap.StaticWindowMapManager(
            max_memory_size=8 * 1024**3,  # 8GB
            max_open_handles=1000
        )
    else:
        # 32-bit: conservative settings  
        return smmap.SlidingWindowMapManager(
            window_size=64 * 1024**2,    # 64MB windows
            max_memory_size=1 * 1024**3,  # 1GB total
            max_open_handles=100
        )

# Platform-specific optimizations
manager = create_optimized_manager()
print(f"Created {type(manager).__name__} for {sys.platform}")
print(f"64-bit system: {smmap.is_64_bit()}")
print(f"Page size: {smmap.ALLOCATIONGRANULARITY} bytes")

Install with Tessl CLI