AST Parsing and Code Analysis

class Parser:
    """
    Parser for Python source code into Definition objects.
    
    Uses Python's tokenize module to parse source code and construct
    a tree of Definition objects (Module, Class, Function, etc.) with
    extracted docstrings and metadata.
    """
    
    def __call__(self, filelike, filename):
        """
        Parse Python source code from file-like object.
        
        Parameters:
        - filelike: file-like object, source code to parse
        - filename: str, filename for error reporting
        
        Returns:
        Module: Root module definition with children
        """

# Global parser instance
parse = Parser()

from pep257 import parse
from io import StringIO

# Parse source code string
source = '''
"""Module docstring."""

class MyClass:
    """Class docstring."""
    
    def my_method(self):
        """Method docstring."""
        pass

def my_function():
    """Function docstring."""
    return 42
'''

module = parse(StringIO(source), 'example.py')

# Walk all definitions
for definition in module:
    print(f"{definition.kind} {definition.name}: {definition.docstring}")

class TokenStream:
    """
    Wrapper around tokenize.generate_tokens for parsing.
    
    Provides a stream interface over Python tokens with current token
    tracking and movement operations.
    """
    
    def __init__(self, filelike):
        """
        Initialize token stream.
        
        Parameters:
        - filelike: file-like object, source to tokenize
        """
    
    @property
    def current(self):
        """Token: Current token in the stream."""
    
    @property 
    def line(self):
        """int: Current line number."""
    
    def move(self):
        """
        Move to next token in stream.
        
        Returns:
        Token: Previous token before move
        """
    
    def __iter__(self):
        """
        Iterate over all tokens in stream.
        
        Returns:
        generator: Token objects
        """

class Token:
    """
    Represents a tokenized element from Python source.
    
    Attributes:
    - kind: TokenKind, type of token
    - value: str, token string value
    - start: tuple, (line, column) start position
    - end: tuple, (line, column) end position  
    - source: str, source line containing token
    """
    
    def __init__(self, kind, value, start, end, source):
        """Initialize token with tokenize output."""

class TokenKind(int):
    """
    Token type identifier with string representation.
    
    Subclass of int that provides readable __repr__ using token names.
    """
    
    def __repr__(self):
        """str: Human-readable token kind name."""

class Value:
    """
    Base class for value objects with field-based initialization.
    
    Provides common functionality for objects with _fields attribute
    defining the object's structure.
    """
    
    def __init__(self, *args):
        """Initialize object with field values."""
    
    def __hash__(self):
        """int: Hash based on object representation."""
    
    def __eq__(self, other):
        """bool: Equality based on field values."""
    
    def __repr__(self):
        """str: String representation with field values."""

class Definition(Value):
    """
    Base class for all code definitions (modules, classes, functions).
    
    Represents a parsed code structure with name, source location,
    docstring, and hierarchical relationships.
    """
    
    _fields = ('name', '_source', 'start', 'end', 'decorators', 'docstring',
               'children', 'parent')
    
    @property
    def _human(self):
        """str: Human-readable type name."""
    
    @property
    def kind(self):
        """str: Definition kind (module, class, function, etc.)."""
    
    @property
    def module(self):
        """Module: Root module containing this definition."""
    
    @property
    def all(self):
        """list: __all__ list from containing module."""
    
    @property
    def _slice(self):
        """slice: Source line slice for this definition."""
    
    @property
    def source(self):
        """str: Source code for this definition."""
    
    @property
    def _publicity(self):
        """str: 'public' or 'private' based on is_public."""
    
    def __iter__(self):
        """
        Iterate over this definition and all children.
        
        Returns:
        generator: This definition followed by all descendants
        """
    
    def __str__(self):
        """str: Human-readable description of definition location."""

class Module(Definition):
    """
    Represents a Python module.
    
    Root definition type that contains classes, functions, and other
    module-level definitions. Tracks __all__ exports and future imports.
    """
    
    _fields = ('name', '_source', 'start', 'end', 'decorators', 'docstring',
               'children', 'parent', '_all', 'future_imports')
    
    is_public = True
    
    @property
    def module(self):
        """Module: Returns self (modules are their own root)."""
    
    @property
    def all(self):
        """list: __all__ exports list."""
    
    def _nest(self, statement_type):
        """
        Get nested definition class for statement type.
        
        Parameters:
        - statement_type: str, 'def' or 'class'
        
        Returns:
        type: Function or Class definition class
        """
    
    def __str__(self):
        """str: Module location description."""

class Package(Module):
    """
    A package is a __init__.py module.
    
    Special module type representing a Python package's __init__.py file.
    """

class Function(Definition):
    """
    Represents a function definition.
    
    Handles both module-level functions and nested functions with
    appropriate publicity determination.
    """
    
    @property
    def is_public(self):
        """
        bool: True if function is public.
        
        Functions are public if:
        - Listed in __all__ (if __all__ exists), or  
        - Name doesn't start with underscore (if no __all__)
        """
    
    def _nest(self, statement_type):
        """
        Get nested definition class for statement type.
        
        Parameters:
        - statement_type: str, 'def' or 'class'
        
        Returns:
        type: NestedFunction or NestedClass
        """

class NestedFunction(Function):
    """
    Represents a nested function definition.
    
    Functions defined inside other functions or methods.
    Always considered private.
    """
    
    is_public = False

class Method(Function):
    """
    Represents a class method definition.
    
    Methods with special publicity rules considering decorators,
    magic methods, and parent class publicity.
    """
    
    @property
    def is_public(self):
        """
        bool: True if method is public.
        
        Methods are public if:
        - Parent class is public, AND
        - Name doesn't start with underscore OR is magic method OR
          is variadic magic method (__init__, __call__, __new__), AND
        - Not a property setter/deleter method
        """

class Class(Definition):
    """
    Represents a class definition.
    
    Handles both module-level classes and nested classes with
    appropriate nesting behavior.
    """
    
    @property
    def is_public(self):
        """
        bool: True if class is public.
        
        Uses same logic as Function.is_public.
        """
    
    def _nest(self, statement_type):
        """
        Get nested definition class for statement type.
        
        Parameters:
        - statement_type: str, 'def' or 'class'  
        
        Returns:
        type: Method or NestedClass
        """

class NestedClass(Class):
    """
    Represents a nested class definition.
    
    Classes defined inside other classes or functions.
    Always considered private.
    """
    
    is_public = False

class Decorator(Value):
    """
    A decorator for function, method or class.
    
    Represents decorator syntax with name and arguments.
    """
    
    _fields = ('name', 'arguments')
    
    def __init__(self, name, arguments):
        """
        Initialize decorator.
        
        Parameters:
        - name: str, decorator name
        - arguments: str, decorator arguments  
        """

class Parser:
    """Internal parsing methods and state management."""
    
    @property
    def current(self):
        """Token: Current token from stream."""
    
    @property
    def line(self):
        """int: Current line number."""
    
    def consume(self, kind):
        """
        Consume token of expected kind.
        
        Parameters:
        - kind: TokenKind, expected token type
        """
    
    def leapfrog(self, kind, value=None):
        """
        Skip tokens until reaching specified kind/value.
        
        Parameters:
        - kind: TokenKind, token type to find
        - value: str, optional token value to match
        """
    
    def parse_docstring(self):
        """
        Parse a single docstring and return its value.
        
        Returns:
        str or None: Docstring value if found
        """
    
    def parse_decorators(self):
        """
        Parse decorators into accumulated decorators list.
        
        Called after first @ is found. Continues until 'def' or 'class'.
        """
    
    def parse_definitions(self, class_, all=False):
        """
        Parse multiple definitions and yield them.
        
        Parameters:
        - class_: type, definition class for context
        - all: bool, whether to parse __all__ statements
        
        Returns:
        generator: Definition objects
        """
    
    def parse_all(self):
        """
        Parse the __all__ definition in a module.
        
        Evaluates __all__ content and stores in self.all.
        
        Raises:
        AllError: If __all__ cannot be parsed
        """
    
    def parse_module(self):
        """
        Parse a module (and its children) and return Module object.
        
        Returns:
        Module or Package: Parsed module definition
        """
    
    def parse_definition(self, class_):
        """
        Parse a definition and return its value in class_ object.
        
        Parameters:
        - class_: type, definition class to create
        
        Returns:
        Definition: Parsed definition of specified type
        """
    
    def parse_from_import_statement(self):
        """
        Parse a 'from x import y' statement.
        
        Specifically looks for __future__ imports and tracks them
        in self.future_imports.
        """

def humanize(string):
    """
    Convert CamelCase to human readable format.
    
    Parameters:
    - string: str, CamelCase string
    
    Returns:
    str: Human-readable string with spaces
    """

def is_magic(name):
    """
    Check if name is a magic method.
    
    Parameters:
    - name: str, method name
    
    Returns:
    bool: True if magic method (starts/ends with __ but not variadic)
    """

def is_ascii(string):
    """
    Check if string contains only ASCII characters.
    
    Parameters:
    - string: str, string to check
    
    Returns:
    bool: True if all characters are ASCII
    """

def is_blank(string):
    """
    Check if string is blank/whitespace only.
    
    Parameters:
    - string: str, string to check
    
    Returns:
    bool: True if string is empty or whitespace
    """

def leading_space(string):
    """
    Extract leading whitespace from string.
    
    Parameters:
    - string: str, string to analyze
    
    Returns:
    str: Leading whitespace characters
    """

VARIADIC_MAGIC_METHODS = ('__init__', '__call__', '__new__')
"""tuple: Magic methods that can have variable arguments."""

class AllError(Exception):
    """
    Exception for __all__ parsing errors.
    
    Raised when __all__ variable cannot be parsed or evaluated.
    Provides detailed error message with guidance.
    """
    
    def __init__(self, message):
        """
        Initialize with error message.
        
        Parameters:
        - message: str, base error message
        """