tessl/pypi-asttokens

Annotate AST trees with source code positions

—

Pending

Quality

Pending

Does it follow best practices?

Impact

Pending

No eval scenarios have been run

Overview

Eval results

Files

Core AST Processing

Name: tessl/pypi-asttokens
Author: tessl

Main classes for annotating AST trees with source code positions and extracting text from AST nodes. These provide the primary functionality for mapping between AST structures and their corresponding source code.

Capabilities

ASTTokens Class

The primary class for working with tokenized source code and AST nodes. It maintains source code in multiple forms and marks AST nodes with token information.

class ASTTokens:
    def __init__(self, source_text, parse=False, tree=None, filename='<unknown>', tokens=None):
        """
        Initialize ASTTokens with source code.
        
        Parameters:
        - source_text (str): Unicode or UTF8-encoded source code
        - parse (bool): If True, parses source_text with ast.parse()
        - tree (ast.Module, optional): AST tree to annotate with tokens
        - filename (str): Filename for error reporting
        - tokens (Iterable, optional): Pre-generated tokens
        """
    
    @property
    def text(self) -> str:
        """The source code passed to constructor."""
    
    @property  
    def tokens(self) -> List[Token]:
        """List of tokens from source code."""
    
    @property
    def tree(self) -> Optional[ast.Module]:
        """Root AST tree (parsed or provided)."""
    
    @property
    def filename(self) -> str:
        """Filename that was parsed."""
        
    def mark_tokens(self, root_node):
        """
        Mark AST nodes with .first_token and .last_token attributes.
        
        Parameters:
        - root_node (ast.Module): Root of AST tree to mark
        """
        
    def get_text(self, node, padded=True) -> str:
        """
        Get source text corresponding to AST node.
        
        Parameters:
        - node (ast.AST): AST node to get text for
        - padded (bool): Include leading whitespace for multiline statements
        
        Returns:
        str: Source text for the node, empty string for nodes like Load
        """
        
    def get_text_range(self, node, padded=True) -> Tuple[int, int]:
        """
        Get character positions in source text for AST node.
        
        Parameters:
        - node (ast.AST): AST node to get range for  
        - padded (bool): Include leading whitespace for multiline statements
        
        Returns:
        Tuple[int, int]: (startpos, endpos) character offsets
        """
        
    def get_text_positions(self, node, padded) -> Tuple[Tuple[int, int], Tuple[int, int]]:
        """
        Get line/column positions for AST node.
        
        Parameters:
        - node (ast.AST): AST node to get positions for
        - padded (bool): Include leading whitespace for multiline statements
        
        Returns:
        Tuple[Tuple[int, int], Tuple[int, int]]: ((start_line, start_col), (end_line, end_col))
        """

Usage Example

import asttokens
import ast

# Parse and annotate source code
source = "def hello(name):\n    return f'Hello, {name}!'"
atok = asttokens.ASTTokens(source, parse=True)

# Find function definition node
func_node = atok.tree.body[0]
print(atok.get_text(func_node))  # Gets full function text

# Get specific parts
name_param = func_node.args.args[0]
print(atok.get_text(name_param))  # Gets just 'name'

# Access token attributes added by mark_tokens
print(func_node.first_token.string)  # 'def'
print(func_node.last_token.string)   # "'"

ASTText Class

Performance-optimized alternative that uses AST position information when available, falling back to full tokenization only when necessary.

class ASTText:
    def __init__(self, source_text, tree=None, filename='<unknown>'):
        """
        Initialize ASTText with source code.
        
        Parameters:
        - source_text (str): Source code to analyze
        - tree (ast.Module, optional): Parsed AST tree, will parse if not provided
        - filename (str): Filename for error reporting
        """
    
    @property
    def tree(self) -> ast.Module:
        """AST tree (parsed if not provided)."""
    
    @property
    def asttokens(self) -> ASTTokens:
        """Fallback ASTTokens instance (lazy-loaded)."""
        
    def get_text(self, node, padded=True) -> str:
        """
        Get source text for AST node (same interface as ASTTokens).
        
        Parameters:
        - node (ast.AST): AST node to get text for
        - padded (bool): Include leading whitespace for multiline statements
        
        Returns:
        str: Source text for the node
        """
        
    def get_text_range(self, node, padded=True) -> Tuple[int, int]:
        """
        Get character positions for AST node (same interface as ASTTokens).
        
        Parameters:
        - node (ast.AST): AST node to get range for
        - padded (bool): Include leading whitespace for multiline statements
        
        Returns:
        Tuple[int, int]: (startpos, endpos) character offsets
        """
        
    def get_text_positions(self, node, padded) -> Tuple[Tuple[int, int], Tuple[int, int]]:
        """
        Get line/column positions for AST node (same interface as ASTTokens).
        
        Parameters:
        - node (ast.AST): AST node to get positions for
        - padded (bool): Include leading whitespace for multiline statements
        
        Returns:
        Tuple[Tuple[int, int], Tuple[int, int]]: ((start_line, start_col), (end_line, end_col))
        """

Usage Example

import asttokens
import ast

source = "x = [1, 2, 3]"
tree = ast.parse(source)

# Use ASTText for better performance when supported
astext = asttokens.ASTText(source, tree=tree)

# Same interface as ASTTokens
assign_node = tree.body[0]
print(astext.get_text(assign_node))  # 'x = [1, 2, 3]'

# Falls back to full tokenization when needed
if not asttokens.supports_tokenless(assign_node):
    # Automatically uses .asttokens fallback
    text = astext.get_text(assign_node)

Install with Tessl CLI