tessl/pypi-pytype

Python type inferencer that analyzes code without requiring explicit type annotations

—

Pending

Overview

Eval results

Files

PYI Parsing

Name: tessl/pypi-pytype
Author: tessl

Parser for Python type stub files (.pyi) that converts textual type declarations into PyTD AST representations for analysis and manipulation. The PYI parser enables PyType to work with existing type stub files and generate canonical type declarations.

Capabilities

String Parsing

Parse .pyi content from strings into PyTD AST format, providing the core parsing functionality for type stub processing.

def parse_string(src, filename=None, options=None):
    """
    Parse .pyi string content to PyTD AST.
    
    Parameters:
    - src (str): .pyi file content as string
    - filename (str, optional): Filename for error reporting
    - options (PyiOptions, optional): Parser configuration options
    
    Returns:
    pytd.TypeDeclUnit: PyTD AST representation of the type declarations
    
    Raises:
    ParseError: If the .pyi content contains syntax errors
    """

Example usage:

from pytype.pyi import parser

pyi_content = '''
from typing import List, Optional

class Calculator:
    def __init__(self) -> None: ...
    def add(self, x: int, y: int) -> int: ...
    def divide(self, x: int, y: int) -> Optional[float]: ...
    
def create_calculator() -> Calculator: ...

PI: float
'''

try:
    ast = parser.parse_string(pyi_content, filename="calculator.pyi")
    print(f"Parsed {len(ast.classes)} classes and {len(ast.functions)} functions")
except parser.ParseError as e:
    print(f"Parse error: {e}")

File Parsing

Parse .pyi files directly from the filesystem with comprehensive error handling and options support.

def parse_pyi(src, filename, options):
    """
    Parse .pyi file with full option support.
    
    Parameters:
    - src (str): .pyi file content
    - filename (str): Path to .pyi file for error reporting
    - options (PyiOptions): Complete parser configuration
    
    Returns:
    pytd.TypeDeclUnit: Parsed PyTD AST
    
    Raises:
    ParseError: If parsing fails due to syntax errors or invalid declarations
    """

Example usage:

from pytype.pyi import parser

# Configure parser options
options = parser.PyiOptions()
options.python_version = (3, 11)

# Read and parse file
with open("mymodule.pyi", "r") as f:
    content = f.read()

ast = parser.parse_pyi(content, "mymodule.pyi", options)

Canonical PYI Generation

Generate canonical .pyi representations from PyTD ASTs, ensuring consistent formatting and optimization of type declarations.

def canonical_pyi(pyi, multiline_args=False, options=None):
    """
    Generate canonical .pyi string from PyTD AST.
    
    Parameters:
    - pyi (pytd.TypeDeclUnit): PyTD AST to convert
    - multiline_args (bool): Whether to format function arguments across multiple lines
    - options (PyiOptions, optional): Formatting options
    
    Returns:
    str: Canonical .pyi file content with consistent formatting
    """

Example usage:

from pytype.pyi import parser
from pytype import io

# Generate AST from source code
source = '''
class DataProcessor:
    def __init__(self, config=None):
        self.config = config or {}
    
    def process(self, data):
        return [item.strip().upper() for item in data if item]
'''

ast = io.generate_pyi_ast(source)

# Convert to canonical .pyi format
canonical = parser.canonical_pyi(ast, multiline_args=True)
print(canonical)

Parser Configuration

Configure parser behavior through options class for different Python versions and parsing modes.

class PyiOptions:
    """
    Configuration options for .pyi parsing.
    
    Controls parser behavior including Python version compatibility,
    error handling, and output formatting preferences.
    """
    
    def __init__(self):
        """Initialize parser options with defaults."""
        
    python_version: tuple       # Target Python version (e.g., (3, 11))
    strict_import: bool         # Strict import checking
    preserve_union_order: bool  # Maintain union type order
    verify: bool               # Verify parsed AST correctness

Example configuration:

from pytype.pyi import parser

# Configure for Python 3.11 with strict checking
options = parser.PyiOptions()
options.python_version = (3, 11)
options.strict_import = True
options.verify = True

# Use in parsing
ast = parser.parse_string(pyi_content, options=options)

Error Handling

Comprehensive error handling for malformed .pyi files and parsing failures.

class ParseError(Exception):
    """
    Exception raised when .pyi parsing fails.
    
    Provides detailed information about syntax errors, invalid type
    declarations, and other parsing failures.
    """
    
    def __init__(self, message, filename=None, lineno=None):
        """
        Initialize parse error.
        
        Parameters:
        - message (str): Error description
        - filename (str, optional): File where error occurred
        - lineno (int, optional): Line number of error
        """

Example error handling:

from pytype.pyi import parser

malformed_pyi = '''
class InvalidClass:
    def bad_method(self, x: InvalidType) -> None: ...  # Unknown type
    def missing_colon(self, x int) -> None: ...        # Syntax error
'''

try:
    ast = parser.parse_string(malformed_pyi, filename="test.pyi")
except parser.ParseError as e:
    print(f"Parse error in {e.filename}:{e.lineno}: {e.message}")

Advanced Parsing Features

Support for complex type declarations and modern Python typing features.

from pytype.pyi import parser

# Parse complex type declarations
complex_pyi = '''
from typing import Generic, TypeVar, Protocol, Literal, Union
from collections.abc import Callable, Iterable

T = TypeVar('T')
U = TypeVar('U', bound=str)

class Container(Generic[T]):
    def __init__(self, items: list[T]) -> None: ...
    def get(self, index: int) -> T: ...
    def map(self, func: Callable[[T], U]) -> Container[U]: ...

class Processor(Protocol):
    def process(self, data: str) -> str: ...

Status = Literal['pending', 'running', 'completed', 'failed']

def handle_result(
    result: Union[str, int, None],
    processor: Processor,
    status: Status = 'pending'
) -> dict[str, str]: ...
'''

ast = parser.parse_string(complex_pyi)
print(f"Parsed {len(ast.type_params)} type parameters")
print(f"Found {len([c for c in ast.classes if c.template])} generic classes")

Integration with PyType Workflow

The PYI parser integrates seamlessly with PyType's analysis workflow:

from pytype.pyi import parser
from pytype.pytd import pytd_utils
from pytype import io, config

# 1. Parse existing stub file
with open("library.pyi", "r") as f:
    existing_stub = f.read()

existing_ast = parser.parse_string(existing_stub, filename="library.pyi")

# 2. Generate new stub from source
source_code = open("library.py", "r").read()
options = config.Options.create()
new_ast = io.generate_pyi_ast(source_code, options=options)

# 3. Combine ASTs
combined_ast = pytd_utils.Concat(existing_ast, new_ast)

# 4. Generate canonical output
canonical_stub = parser.canonical_pyi(combined_ast, multiline_args=True)

# 5. Write updated stub
with open("library_updated.pyi", "w") as f:
    f.write(canonical_stub)

Performance Considerations

For large .pyi files or batch processing:

from pytype.pyi import parser
import os

def parse_stub_directory(stub_dir):
    """Parse all .pyi files in a directory efficiently."""
    
    # Reuse options for better performance
    options = parser.PyiOptions()
    options.python_version = (3, 11)
    
    parsed_asts = {}
    
    for filename in os.listdir(stub_dir):
        if filename.endswith('.pyi'):
            filepath = os.path.join(stub_dir, filename)
            with open(filepath, 'r') as f:
                content = f.read()
            
            try:
                ast = parser.parse_pyi(content, filename, options)
                parsed_asts[filename] = ast
            except parser.ParseError as e:
                print(f"Failed to parse {filename}: {e}")
    
    return parsed_asts

Install with Tessl CLI