tessl/pypi-pyupgrade

A tool and pre-commit hook to automatically upgrade Python syntax for newer versions of the language.

—

Pending

Overview

Eval results

Files

Token Manipulation

Name: tessl/pypi-pyupgrade
Author: tessl

Comprehensive token-level manipulation utilities for precise code transformations. These utilities provide fine-grained control over token streams for implementing complex syntax changes.

Constants

Core constants used for token classification and manipulation.

_OPENING: frozenset[str]
"""Opening bracket characters: '([{'"""

_CLOSING: frozenset[str] 
"""Closing bracket characters: ')]}' """

KEYWORDS: frozenset[str]
"""Python keyword set from keyword.kwlist"""

Capabilities

Token Type Detection

Basic token classification functions.

def is_open(token: Token) -> bool:
    """
    Check if token is opening bracket/parenthesis.
    
    Args:
        token: Token to check
        
    Returns:
        True if token is '(', '[', or '{'
    """

def is_close(token: Token) -> bool:
    """
    Check if token is closing bracket/parenthesis.
    
    Args:
        token: Token to check
        
    Returns:
        True if token is ')', ']', or '}'
    """

def immediately_paren(func: str, tokens: list[Token], i: int) -> bool:
    """
    Check if function name is immediately followed by parenthesis.
    
    Args:
        func: Function name to match
        tokens: Token list
        i: Index to check
        
    Returns:
        True if tokens[i] == func and tokens[i+1] == '('
    """

Token Search Functions

Find specific tokens in token streams.

def find_name(tokens: list[Token], i: int, src: str) -> int:
    """
    Find next NAME token with specific source.
    
    Args:
        tokens: Token list to search
        i: Starting index
        src: Source string to match
        
    Returns:
        Index of matching token
    """

def find_op(tokens: list[Token], i: int, src: str) -> int:
    """
    Find next OP token with specific source.
    
    Args:
        tokens: Token list to search  
        i: Starting index
        src: Operator string to match
        
    Returns:
        Index of matching token
    """

def find_end(tokens: list[Token], i: int) -> int:
    """
    Find end of statement (NEWLINE token).
    
    Args:
        tokens: Token list to search
        i: Starting index
        
    Returns:
        Index after the NEWLINE token
    """

def find_call(tokens: list[Token], i: int) -> int:
    """
    Find function call opening parenthesis.
    
    Args:
        tokens: Token list to search
        i: Starting index
        
    Returns:
        Index of opening parenthesis for function call
        
    Notes:
        Handles nested expressions like ("something").method(...)
    """

Bracket and Block Processing

Handle bracket matching and code blocks.

def find_closing_bracket(tokens: list[Token], i: int) -> int:
    """
    Find matching closing bracket for opening bracket.
    
    Args:
        tokens: Token list
        i: Index of opening bracket
        
    Returns:
        Index of matching closing bracket
        
    Notes:
        Handles nested brackets correctly
    """

def find_block_start(tokens: list[Token], i: int) -> int:
    """
    Find colon starting code block.
    
    Args:
        tokens: Token list  
        i: Starting search index
        
    Returns:
        Index of colon token starting block
    """

class Block(NamedTuple):
    """
    Code block boundaries in token stream.
    
    Attributes:
        start: Block start index
        colon: Colon token index
        block: Block content start index  
        end: Block end index
        line: True if single-line block
    """
    start: int
    colon: int
    block: int
    end: int
    line: bool
    
    @classmethod
    def find(cls, tokens: list[Token], i: int, trim_end: bool = False) -> 'Block':
        """Find code block starting at index i."""
    
    def dedent(self, tokens: list[Token]) -> None:
        """Dedent block content by removing common indentation."""
    
    def replace_condition(self, tokens: list[Token], new: list[Token]) -> None:
        """Replace block condition with new tokens."""

Function Call Processing

Argument Parsing

Parse function call arguments from token stream.

def parse_call_args(tokens: list[Token], i: int) -> tuple[list[tuple[int, int]], int]:
    """
    Parse function call arguments from tokens.
    
    Args:
        tokens: Token list
        i: Index of opening parenthesis
        
    Returns:
        Tuple of (argument_ranges, end_index)
        - argument_ranges: List of (start, end) indices for each argument
        - end_index: Index after closing parenthesis
    """

def arg_str(tokens: list[Token], start: int, end: int) -> str:
    """
    Get argument string from token range.
    
    Args:
        tokens: Token list
        start: Start index
        end: End index
        
    Returns:
        String representation of tokens in range
    """

Call Transformation

Replace function calls with templates.

def replace_call(
    tokens: list[Token],
    start: int, 
    end: int,
    args: list[tuple[int, int]],
    tmpl: str,
    *,
    parens: Sequence[int] = ()
) -> None:
    """
    Replace function call with template.
    
    Args:
        tokens: Token list to modify in-place
        start: Call start index
        end: Call end index  
        args: Argument ranges from parse_call_args
        tmpl: Template string with {args[0]}, {args[1]}, {rest} placeholders
        parens: Argument indices to wrap in parentheses
        
    Notes:
        - Handles multiline arguments safely
        - Preserves comments and whitespace where possible
        - Adds parentheses around arguments that contain newlines
    """

def find_and_replace_call(
    i: int,
    tokens: list[Token], 
    *,
    template: str,
    parens: tuple[int, ...] = ()
) -> None:
    """
    Find function call and replace with template.
    
    Args:
        i: Starting token index
        tokens: Token list to modify
        template: Replacement template
        parens: Argument indices to parenthesize
    """

Token Modification Utilities

Token Replacement

Replace individual tokens and arguments.

def replace_name(i: int, tokens: list[Token], *, name: str, new: str) -> None:
    """
    Replace name token with new name.
    
    Args:
        i: Starting token index
        tokens: Token list to modify
        name: Name to find and replace
        new: Replacement name
    """

def delete_argument(
    i: int,
    tokens: list[Token],
    func_args: Sequence[tuple[int, int]]
) -> None:
    """
    Delete function argument from call.
    
    Args:
        i: Argument index to delete
        tokens: Token list to modify
        func_args: Argument ranges from parse_call_args
    """

def replace_argument(
    i: int,
    tokens: list[Token], 
    func_args: Sequence[tuple[int, int]],
    *,
    new: str
) -> None:
    """
    Replace function argument with new content.
    
    Args:
        i: Argument index to replace
        tokens: Token list to modify
        func_args: Argument ranges
        new: Replacement content
    """

Structural Modifications

Remove code structures like braces, decorators, and base classes.

def remove_brace(tokens: list[Token], i: int) -> None:
    """
    Remove brace token and surrounding whitespace.
    
    Args:
        tokens: Token list to modify
        i: Index of brace token
        
    Notes:
        Removes extra whitespace if brace is on its own line
    """

def remove_decorator(i: int, tokens: list[Token]) -> None:
    """
    Remove decorator from function/class.
    
    Args:
        i: Index within decorator
        tokens: Token list to modify
        
    Notes:
        Finds @ symbol and removes entire decorator line
    """

def remove_base_class(i: int, tokens: list[Token]) -> None:
    """
    Remove base class from class definition.
    
    Args:
        i: Index within base class reference
        tokens: Token list to modify
        
    Notes:
        Handles single/multiple base classes and parentheses correctly
    """

Comprehension Processing

Victim Token Identification

Identify tokens to remove for comprehension transformations.

class Victims(NamedTuple):
    """
    Token indices to remove for comprehension transformations.
    
    Attributes:
        starts: Opening bracket indices to remove
        ends: Closing bracket indices to remove  
        first_comma_index: First comma index (if any)
        arg_index: Argument start index
    """
    starts: list[int]
    ends: list[int]
    first_comma_index: int | None
    arg_index: int

def victims(
    tokens: list[Token],
    start: int,
    arg: ast.expr, 
    gen: bool
) -> Victims:
    """
    Find tokens to remove for comprehension transformation.
    
    Args:
        tokens: Token list
        start: Starting token index
        arg: AST expression for argument
        gen: True if generator expression
        
    Returns:
        Victims object with token indices to remove
        
    Usage:
        Used to transform set([x for x in y]) → {x for x in y}
    """

Advanced Utilities

Constant Folding

Fold constant expressions in tuples.

def constant_fold_tuple(i: int, tokens: list[Token]) -> None:
    """
    Fold constant tuple expressions.
    
    Args:
        i: Starting token index
        tokens: Token list to modify
        
    Example:
        isinstance(x, (int, int, str)) → isinstance(x, (int, str))
    """

Indentation and Spacing

Handle indentation and whitespace.

def has_space_before(i: int, tokens: list[Token]) -> bool:
    """
    Check if token has whitespace before it.
    
    Args:
        i: Token index
        tokens: Token list
        
    Returns:
        True if preceded by whitespace or indent
    """

def indented_amount(i: int, tokens: list[Token]) -> str:
    """
    Get indentation string at token position.
    
    Args:
        i: Token index
        tokens: Token list
        
    Returns:
        Indentation string (spaces/tabs)
        
    Raises:
        ValueError: If not at beginning of line
    """

Usage Examples

Function Call Transformation

from pyupgrade._token_helpers import find_and_replace_call

def transform_print_calls(i: int, tokens: list[Token]) -> None:
    """Transform print statements to print functions."""
    
    # Replace print x, y with print(x, y)
    find_and_replace_call(
        i, tokens,
        template="print({args[0]}, {args[1]})",
        parens=()
    )

Comprehension Transformation

from pyupgrade._token_helpers import victims, remove_brace

def transform_set_comprehension(tokens: list[Token], start: int, arg: ast.expr) -> None:
    """Transform set([x for x in y]) to {x for x in y}."""
    
    # Find tokens to remove
    v = victims(tokens, start, arg, gen=False)
    
    # Remove brackets in reverse order  
    for end_idx in reversed(v.ends):
        remove_brace(tokens, end_idx)
    
    for start_idx in reversed(v.starts):
        remove_brace(tokens, start_idx)

Install with Tessl CLI