Parsing and Model

def get_model(source, **options):
    """
    Parse test data into AST model.
    
    Args:
        source: Path to test file or string content
        **options: Parsing options (language, etc.)
        
    Returns:
        File model object representing parsed content
    """

def get_resource_model(source, **options):
    """
    Parse resource file into AST model.
    
    Args:
        source: Path to resource file or string content
        **options: Parsing options
        
    Returns:
        File model object for resource file
    """

def get_init_model(source, **options):
    """
    Parse suite initialization file into AST model.
    
    Args:
        source: Path to __init__.robot file or string content
        **options: Parsing options
        
    Returns:
        File model object for initialization file
    """

def get_tokens(source, **options):
    """
    Parse test data into tokens.
    
    Args:
        source: Path to test file or string content
        **options: Parsing options
        
    Returns:
        Generator yielding Token objects
    """

def get_resource_tokens(source, **options):
    """Parse resource file into tokens."""

def get_init_tokens(source, **options):
    """Parse initialization file into tokens."""

from robot.api.parsing import get_model, get_tokens

# Parse test file into AST model
model = get_model('tests/example.robot')
print(f"Sections: {len(model.sections)}")

for section in model.sections:
    print(f"Section type: {type(section).__name__}")
    if hasattr(section, 'body'):
        print(f"  Items: {len(section.body)}")

# Parse into tokens for low-level processing
tokens = list(get_tokens('tests/example.robot'))
for token in tokens[:10]:  # First 10 tokens
    print(f"Token: {token.type} = '{token.value}' at {token.lineno}:{token.col_offset}")

# Parse resource file
resource_model = get_resource_model('resources/common.resource')
print(f"Resource keywords: {len(resource_model.keyword_section.body)}")

class Token:
    """
    Represents a single token in Robot Framework syntax.
    
    Attributes:
        type: Token type (string constant)
        value: Token value/content
        lineno: Line number (1-based)
        col_offset: Column offset (0-based)
        error: Error message if token represents an error
    """
    
    # Token types as class attributes
    SETTING_HEADER = 'SETTING HEADER'
    VARIABLE_HEADER = 'VARIABLE HEADER'
    TESTCASE_HEADER = 'TESTCASE HEADER'
    KEYWORD_HEADER = 'KEYWORD HEADER'
    COMMENT_HEADER = 'COMMENT HEADER'
    
    TESTCASE_NAME = 'TESTCASE NAME'
    KEYWORD_NAME = 'KEYWORD NAME'
    
    DOCUMENTATION = 'DOCUMENTATION'
    TAGS = 'TAGS'
    SETUP = 'SETUP'
    TEARDOWN = 'TEARDOWN'
    TIMEOUT = 'TIMEOUT'
    TEMPLATE = 'TEMPLATE'
    ARGUMENTS = 'ARGUMENTS'
    RETURN = 'RETURN'
    
    VARIABLE = 'VARIABLE'
    KEYWORD = 'KEYWORD'
    ASSIGN = 'ASSIGN'
    ARGUMENT = 'ARGUMENT'
    
    FOR = 'FOR'
    FOR_SEPARATOR = 'FOR SEPARATOR'
    END = 'END'
    IF = 'IF'
    INLINE_IF = 'INLINE IF'
    ELSE_IF = 'ELSE IF'
    ELSE = 'ELSE'
    TRY = 'TRY'
    EXCEPT = 'EXCEPT'
    FINALLY = 'FINALLY'
    WHILE = 'WHILE'
    
    COMMENT = 'COMMENT'
    CONTINUATION = 'CONTINUATION'
    EOL = 'EOL'
    EOS = 'EOS'  # End of statement
    EOF = 'EOF'  # End of file

from robot.api.parsing import get_tokens, Token

def analyze_test_structure(source):
    """Analyze test file structure using tokens."""
    
    stats = {
        'test_cases': 0,
        'keywords': 0,
        'variables': 0,
        'comments': 0,
        'for_loops': 0,
        'if_statements': 0
    }
    
    for token in get_tokens(source):
        if token.type == Token.TESTCASE_NAME:
            stats['test_cases'] += 1
        elif token.type == Token.KEYWORD_NAME:
            stats['keywords'] += 1
        elif token.type == Token.VARIABLE:
            stats['variables'] += 1
        elif token.type == Token.COMMENT:
            stats['comments'] += 1
        elif token.type == Token.FOR:
            stats['for_loops'] += 1
        elif token.type == Token.IF:
            stats['if_statements'] += 1
    
    return stats

# Analyze test file
stats = analyze_test_structure('tests/complex.robot')
print(f"Test file contains: {stats}")

class File:
    """
    Root model representing a complete Robot Framework file.
    
    Attributes:
        source: Source file path
        sections: List of section objects
    """

# Section classes
class SettingSection:
    """Settings section (*** Settings ***)."""
    body: List  # Setting statements

class VariableSection:
    """Variables section (*** Variables ***)."""
    body: List  # Variable statements

class TestCaseSection:
    """Test Cases section (*** Test Cases ***)."""
    body: List  # TestCase objects

class KeywordSection:
    """Keywords section (*** Keywords ***)."""  
    body: List  # Keyword objects

class CommentSection:
    """Comments section (*** Comments ***)."""
    body: List  # Comment statements

# Block classes (can contain other blocks and statements)
class TestCase:
    """Individual test case definition."""
    header: TestCaseName
    body: List  # Keyword calls and control structures

class Keyword:
    """Individual keyword definition."""
    header: KeywordName
    body: List  # Keyword calls and control structures

# Control structure classes  
class If:
    """IF/ELSE control structure."""
    header: IfHeader
    body: List
    orelse: List  # ELSE IF and ELSE branches

class For:
    """FOR loop control structure."""  
    header: ForHeader
    body: List

class While:
    """WHILE loop control structure."""
    header: WhileHeader
    body: List

class Try:
    """TRY/EXCEPT control structure."""
    header: TryHeader
    body: List
    next: List  # EXCEPT and FINALLY branches

from robot.api.parsing import get_model

def extract_test_keywords(source):
    """Extract all keywords used in test cases."""
    
    model = get_model(source)
    keywords_used = set()
    
    for section in model.sections:
        if hasattr(section, 'body'):
            for item in section.body:
                if hasattr(item, 'body'):  # Test case or keyword
                    for statement in item.body:
                        if hasattr(statement, 'keyword'):
                            keywords_used.add(statement.keyword)
    
    return sorted(keywords_used)

# Extract keywords from test file
keywords = extract_test_keywords('tests/suite.robot')
print(f"Keywords used: {keywords}")

class ModelVisitor:
    """
    Base class for inspecting model objects.
    Override visit_* methods to process specific node types.
    """
    
    def visit(self, node): ...
    def generic_visit(self, node): ...
    
    # Visit methods for different node types
    def visit_File(self, node): ...
    def visit_Section(self, node): ...
    def visit_TestCase(self, node): ...
    def visit_Keyword(self, node): ...
    def visit_For(self, node): ...
    def visit_If(self, node): ...
    def visit_Statement(self, node): ...

class ModelTransformer:
    """
    Base class for modifying model objects.
    Override visit_* methods to transform specific node types.
    """
    
    def visit(self, node): ...
    def generic_visit(self, node): ...
    
    # Transform methods return modified nodes
    def visit_File(self, node): ...
    def visit_TestCase(self, node): ...
    def visit_Keyword(self, node): ...

from robot.api.parsing import get_model, ModelVisitor, ModelTransformer

class TestCaseAnalyzer(ModelVisitor):
    """Analyze test cases for complexity metrics."""
    
    def __init__(self):
        self.test_stats = {}
        self.current_test = None
    
    def visit_TestCase(self, node):
        self.current_test = node.header.data_tokens[0].value
        self.test_stats[self.current_test] = {
            'keywords': 0,
            'for_loops': 0,
            'if_statements': 0,
            'assignments': 0
        }
        self.generic_visit(node)
    
    def visit_KeywordCall(self, node):
        if self.current_test:
            self.test_stats[self.current_test]['keywords'] += 1
    
    def visit_For(self, node):
        if self.current_test:
            self.test_stats[self.current_test]['for_loops'] += 1
        self.generic_visit(node)
    
    def visit_If(self, node):
        if self.current_test:
            self.test_stats[self.current_test]['if_statements'] += 1
        self.generic_visit(node)

class TagAdder(ModelTransformer):
    """Add tags to test cases based on content analysis."""
    
    def visit_TestCase(self, node):
        # Analyze test content to determine appropriate tags
        tags_to_add = []
        
        # Check if test uses database keywords
        for statement in node.body:
            if hasattr(statement, 'keyword'):
                keyword = statement.keyword.lower()
                if 'database' in keyword or 'sql' in keyword:
                    tags_to_add.append('database')
                elif 'web' in keyword or 'browser' in keyword:  
                    tags_to_add.append('web')
        
        # Add tags statement if new tags found
        if tags_to_add:
            # Create new tags statement
            tags_statement = Tags([Token(Token.TAGS, 'Tags')] + 
                                [Token(Token.ARGUMENT, tag) for tag in tags_to_add])
            node.body.insert(0, tags_statement)
        
        return node

# Use visitors to analyze and transform
model = get_model('tests/suite.robot')

# Analyze complexity
analyzer = TestCaseAnalyzer()
analyzer.visit(model)
print("Test complexity metrics:", analyzer.test_stats)

# Transform by adding tags
transformer = TagAdder()
modified_model = transformer.visit(model)

# Convert back to text
print(str(modified_model))

class SuiteStructure:
    """
    Represents the structure of a test suite directory.
    
    Attributes:
        source: Root directory path
        init_file: Initialization file path (if exists)
        children: Child suite structures and test files
    """

class SuiteStructureBuilder:
    """Build suite structures from file system."""
    
    def build(self, source): ...

class SuiteStructureVisitor:
    """Process suite structures using visitor pattern."""
    
    def visit_directory(self, directory): ...
    def visit_file(self, file): ...

from robot.parsing.suitestructure import SuiteStructureBuilder

# Analyze test suite directory structure
builder = SuiteStructureBuilder()
structure = builder.build('tests/')

def print_structure(structure, indent=0):
    """Print suite structure recursively."""
    prefix = "  " * indent
    print(f"{prefix}{structure.source}")
    
    if structure.init_file:
        print(f"{prefix}  __init__.robot")
    
    for child in structure.children:
        if hasattr(child, 'children'):  # Directory
            print_structure(child, indent + 1)
        else:  # Test file
            print(f"{prefix}  {child.source}")

print_structure(structure)

class SuiteVisitor:
    """
    Abstract base class for processing test suites.
    Can be used as pre-run modifier (--prerunmodifier option).
    """
    
    def start_suite(self, suite): ...
    def end_suite(self, suite): ...
    def start_test(self, test): ...
    def end_test(self, test): ...
    def start_keyword(self, keyword): ...
    def end_keyword(self, keyword): ...
    def visit_suite(self, suite): ...
    def visit_test(self, test): ...
    def visit_keyword(self, keyword): ...

from robot.api import SuiteVisitor

class TestFilter(SuiteVisitor):
    """Filter tests based on custom criteria."""
    
    def __init__(self, min_priority=1):
        self.min_priority = min_priority
    
    def start_suite(self, suite):
        # Filter tests based on priority tag
        original_tests = list(suite.tests)
        suite.tests.clear()
        
        for test in original_tests:
            priority = self._get_priority(test)
            if priority >= self.min_priority:
                suite.tests.append(test)
    
    def _get_priority(self, test):
        """Extract priority from test tags."""
        for tag in test.tags:
            if tag.startswith('priority:'):
                try:
                    return int(tag.split(':')[1])
                except ValueError:
                    pass
        return 0  # Default priority

# Use as pre-run modifier
class TestEnhancer(SuiteVisitor):
    """Enhance tests with additional setup/teardown."""
    
    def start_test(self, test):
        # Add timestamp variable to each test
        test.keywords.create(
            'Set Test Variable', 
            args=['${TEST_START_TIME}', '${CURTIME}'],
            assign=['${START_TIME}']
        ).insert(0)  # Insert at beginning
    
    def end_test(self, test):
        # Add cleanup keyword to each test
        test.teardown.config(
            name='Log Test Duration',
            args=['Test completed in ${CURTIME - START_TIME}']
        )

# Token position information
LineNumber = int      # 1-based line number
ColumnOffset = int    # 0-based column offset

# Model node types
NodeType = Union[
    File, Section, TestCase, Keyword, 
    For, If, While, Try, Statement
]

# Parsing options
ParsingOptions = Dict[str, Any]  # language, etc.

# Statement types
StatementType = Union[
    # Settings
    Documentation, Tags, Setup, Teardown, Timeout, Template, Arguments, Return,
    # Variables  
    Variable,
    # Execution
    KeywordCall, TemplateArguments,
    # Control structures
    IfHeader, ElseIfHeader, ElseHeader, ForHeader, WhileHeader, TryHeader,
    ExceptHeader, FinallyHeader, End,
    # Other
    Comment, EmptyLine, Error
]