tessl/pypi-pytype

Python type inferencer that analyzes code without requiring explicit type annotations

—

Pending

Overview

Eval results

Files

Module Loading

Name: tessl/pypi-pytype
Author: tessl

Infrastructure for loading and resolving Python modules, PyTD files, and type information with support for custom import mappings and pickled type data. The module loading system enables PyType to analyze complex projects with intricate dependency structures.

Capabilities

Loader Creation

Create configured loaders for different analysis scenarios, providing the foundation for module resolution and type information loading.

def create_loader(options, missing_modules=()):
    """
    Create PyTD loader with specified configuration.
    
    Parameters:
    - options (Options): Configuration options including Python paths,
                        import mappings, and module resolution settings
    - missing_modules (tuple): Modules to treat as missing/unavailable
    
    Returns:
    Loader: Configured loader instance for module resolution
    
    Raises:
    ImportError: If loader configuration fails
    """

Example usage:

from pytype import config, load_pytd

# Create options with custom Python path
options = config.Options.create()
options.pythonpath = ["/my/project/src", "/my/project/lib"]
options.imports_map = "/my/project/.pytype/imports.txt"

# Create loader with known missing modules
loader = load_pytd.create_loader(
    options, 
    missing_modules=("unavailable_dep", "optional_package")
)

Loader Class

Main loader class providing comprehensive module resolution and type information loading capabilities.

class Loader:
    """
    Main loader for PyTD modules and type information.
    
    Handles module resolution, import mapping, caching, and provides
    the core infrastructure for PyType's analysis process.
    """
    
    def __init__(self, python_version, python_exe, module_utils):
        """
        Initialize loader with environment configuration.
        
        Parameters:
        - python_version (tuple): Target Python version
        - python_exe (str): Path to Python executable
        - module_utils: Module resolution utilities
        """
    
    def load_file(self, module_name, filename):
        """
        Load module from specific file path.
        
        Parameters:
        - module_name (str): Module name for loading
        - filename (str): Path to module file
        
        Returns:
        Module: Loaded module with type information
        """
    
    def import_name(self, module_name):
        """
        Import module by name with full resolution.
        
        Parameters:
        - module_name (str): Fully qualified module name
        
        Returns:
        Module: Loaded module with resolved dependencies
        
        Raises:
        ImportError: If module cannot be resolved or loaded
        """
    
    def lookup_builtin(self, name):
        """
        Look up built-in type or function.
        
        Parameters:
        - name (str): Built-in name to resolve
        
        Returns:
        Type information for built-in, or None if not found
        """
    
    def has_module(self, module_name):
        """
        Check if module is available for loading.
        
        Parameters:
        - module_name (str): Module name to check
        
        Returns:
        bool: True if module can be loaded
        """

Example usage:

from pytype import load_pytd, config

# Create and use loader
options = config.Options.create()
loader = load_pytd.create_loader(options)

# Load specific modules
try:
    os_module = loader.import_name("os")
    json_module = loader.import_name("json")
    
    # Check module availability
    if loader.has_module("requests"):
        requests_module = loader.import_name("requests")
        
except ImportError as e:
    print(f"Failed to load module: {e}")

Module Representation

Classes for representing loaded modules and their metadata.

class Module:
    """
    Represents a parsed and loaded Python module.
    
    Contains the PyTD AST representation along with metadata
    about the module's source and dependencies.
    """
    
    def __init__(self, ast, filename=None, module_name=None):
        """
        Initialize module representation.
        
        Parameters:
        - ast (TypeDeclUnit): PyTD AST for the module
        - filename (str, optional): Source file path
        - module_name (str, optional): Fully qualified module name
        """

class ResolvedModule:
    """
    Dataclass containing resolved module information.
    
    Provides complete information about a successfully resolved
    module including its location, type, and metadata.
    """
    
    def __init__(self, module_name, filename, ast):
        """
        Initialize resolved module information.
        
        Parameters:
        - module_name (str): Fully qualified module name
        - filename (str): Path to module source
        - ast (TypeDeclUnit): Parsed PyTD AST
        """

Pickled PyI Loader

Specialized loader for working with pickled .pyi files, enabling faster loading of pre-processed type information.

class PickledPyiLoader:
    """
    Loader for pickled .pyi files.
    
    Provides optimized loading of pre-processed type stub files,
    significantly improving analysis performance for large projects.
    """
    
    def __init__(self, python_version, module_utils):
        """
        Initialize pickled .pyi loader.
        
        Parameters:
        - python_version (tuple): Target Python version
        - module_utils: Module resolution utilities
        """
    
    def load_from_pickle(self, filename):
        """
        Load PyTD AST from pickled file.
        
        Parameters:
        - filename (str): Path to pickled .pyi file
        
        Returns:
        TypeDeclUnit: Loaded PyTD AST
        
        Raises:
        IOError: If file cannot be read or unpickled
        """
    
    def save_to_pickle(self, ast, filename):
        """
        Save PyTD AST to pickled file.
        
        Parameters:
        - ast (TypeDeclUnit): PyTD AST to pickle
        - filename (str): Output pickle file path
        """

Example usage:

from pytype import load_pytd, config
from pytype.pytd import pytd_utils

# Create pickled loader
options = config.Options.create()
loader = load_pytd.PickledPyiLoader(options.python_version, None)

# Save AST to pickle for faster loading
ast = # ... some PyTD AST
loader.save_to_pickle(ast, "mymodule.pkl")

# Load from pickle
cached_ast = loader.load_from_pickle("mymodule.pkl")

Import Resolution

Advanced import resolution with support for custom mappings and complex project structures.

from pytype import load_pytd, config

# Configure custom import mappings
options = config.Options.create()
options.imports_map = "imports.txt"  # Custom import mappings

# imports.txt content:
# internal.utils myproject.internal.utils
# external_lib /path/to/external/stubs

loader = load_pytd.create_loader(options)

# Import resolution uses mappings automatically
try:
    # This will resolve using the mapping
    utils_module = loader.import_name("internal.utils")
    
    # This will use the external stub path
    external_module = loader.import_name("external_lib")
    
except ImportError as e:
    print(f"Import resolution failed: {e}")

Caching and Performance

The loader system includes comprehensive caching for improved performance:

from pytype import load_pytd, config

options = config.Options.create()
loader = load_pytd.create_loader(options)

# First load - from source
module1 = loader.import_name("collections")  # Loads from source

# Second load - from cache
module2 = loader.import_name("collections")  # Uses cached version

assert module1 is module2  # Same object from cache

Error Classes

class BadDependencyError(Exception):
    """Error loading module dependencies."""

class ImportError(Exception):  
    """Module import resolution failure."""

Error Handling

Comprehensive error handling for module loading failures:

from pytype import load_pytd, config

options = config.Options.create()
loader = load_pytd.create_loader(options)

try:
    # Attempt to load potentially missing module
    module = loader.import_name("nonexistent.module")
    
except ImportError as e:
    print(f"Module not found: {e}")
    
    # Check if module is available before loading
    if loader.has_module("alternative.module"):
        alt_module = loader.import_name("alternative.module")
    else:
        print("No alternative module available")

except Exception as e:
    print(f"Unexpected loading error: {e}")

Integration with Analysis

The module loading system integrates seamlessly with PyType's analysis workflow:

from pytype import io, config, load_pytd

# Create configured loader
options = config.Options.create()
options.pythonpath = ["/my/project"]
loader = load_pytd.create_loader(options)

# Use loader in analysis
source_code = '''
import json
import myproject.utils

def process_data(data):
    config = json.loads(data)
    return myproject.utils.transform(config)
'''

# Analysis uses loader for import resolution
result = io.check_py(source_code, options=options, loader=loader)

Custom Module Resolution

For complex project structures, implement custom module resolution:

from pytype import load_pytd, config

class CustomLoader(load_pytd.Loader):
    """Custom loader with project-specific resolution logic."""
    
    def import_name(self, module_name):
        # Custom resolution logic
        if module_name.startswith("myproject."):
            # Handle internal modules specially
            internal_path = self._resolve_internal_module(module_name)
            return self.load_file(module_name, internal_path)
        
        # Fall back to standard resolution
        return super().import_name(module_name)
    
    def _resolve_internal_module(self, module_name):
        # Project-specific module resolution
        parts = module_name.split(".")
        return f"/project/src/{'/'.join(parts[1:])}.py"

# Use custom loader
options = config.Options.create()
loader = CustomLoader(options.python_version, options.python_exe, None)

Bulk Module Loading

For analyzing projects with many dependencies:

from pytype import load_pytd, config

def preload_common_modules(loader, modules):
    """Preload commonly used modules for better performance."""
    
    loaded = {}
    for module_name in modules:
        try:
            loaded[module_name] = loader.import_name(module_name)
            print(f"Preloaded {module_name}")
        except ImportError:
            print(f"Skipped {module_name} (not available)")
    
    return loaded

# Preload standard library modules
options = config.Options.create()
loader = load_pytd.create_loader(options)

common_modules = [
    "os", "sys", "json", "typing", "collections",
    "itertools", "functools", "datetime"
]

preloaded = preload_common_modules(loader, common_modules)

Install with Tessl CLI