0
# Parsing and Model
1

2
Robot Framework provides comprehensive APIs for parsing test data, manipulating AST models, and programmatically working with test structure. These APIs enable custom parsers, test data transformation, and deep integration with Robot Framework's internal data structures.
3

4
## Capabilities
5

6
### Test Data Parsing
7

8
Parse Robot Framework test files into tokens and abstract syntax tree (AST) models.
9

10
```python { .api }
11
def get_model(source, **options):
12
    """
13
    Parse test data into AST model.
14
    
15
    Args:
16
        source: Path to test file or string content
17
        **options: Parsing options (language, etc.)
18
        
19
    Returns:
20
        File model object representing parsed content
21
    """
22

23
def get_resource_model(source, **options):
24
    """
25
    Parse resource file into AST model.
26
    
27
    Args:
28
        source: Path to resource file or string content
29
        **options: Parsing options
30
        
31
    Returns:
32
        File model object for resource file
33
    """
34

35
def get_init_model(source, **options):
36
    """
37
    Parse suite initialization file into AST model.
38
    
39
    Args:
40
        source: Path to __init__.robot file or string content
41
        **options: Parsing options
42
        
43
    Returns:
44
        File model object for initialization file
45
    """
46

47
def get_tokens(source, **options):
48
    """
49
    Parse test data into tokens.
50
    
51
    Args:
52
        source: Path to test file or string content
53
        **options: Parsing options
54
        
55
    Returns:
56
        Generator yielding Token objects
57
    """
58

59
def get_resource_tokens(source, **options):
60
    """Parse resource file into tokens."""
61

62
def get_init_tokens(source, **options):
63
    """Parse initialization file into tokens."""
64
```
65

66
**Usage Examples:**
67

68
```python
69
from robot.api.parsing import get_model, get_tokens
70

71
# Parse test file into AST model
72
model = get_model('tests/example.robot')
73
print(f"Sections: {len(model.sections)}")
74

75
for section in model.sections:
76
    print(f"Section type: {type(section).__name__}")
77
    if hasattr(section, 'body'):
78
        print(f"  Items: {len(section.body)}")
79

80
# Parse into tokens for low-level processing
81
tokens = list(get_tokens('tests/example.robot'))
82
for token in tokens[:10]:  # First 10 tokens
83
    print(f"Token: {token.type} = '{token.value}' at {token.lineno}:{token.col_offset}")
84

85
# Parse resource file
86
resource_model = get_resource_model('resources/common.resource')
87
print(f"Resource keywords: {len(resource_model.keyword_section.body)}")
88
```
89

90
### Token Representation
91

92
Low-level token representation for detailed parsing control.
93

94
```python { .api }
95
class Token:
96
    """
97
    Represents a single token in Robot Framework syntax.
98
    
99
    Attributes:
100
        type: Token type (string constant)
101
        value: Token value/content
102
        lineno: Line number (1-based)
103
        col_offset: Column offset (0-based)
104
        error: Error message if token represents an error
105
    """
106
    
107
    # Token types as class attributes
108
    SETTING_HEADER = 'SETTING HEADER'
109
    VARIABLE_HEADER = 'VARIABLE HEADER'
110
    TESTCASE_HEADER = 'TESTCASE HEADER'
111
    KEYWORD_HEADER = 'KEYWORD HEADER'
112
    COMMENT_HEADER = 'COMMENT HEADER'
113
    
114
    TESTCASE_NAME = 'TESTCASE NAME'
115
    KEYWORD_NAME = 'KEYWORD NAME'
116
    
117
    DOCUMENTATION = 'DOCUMENTATION'
118
    TAGS = 'TAGS'
119
    SETUP = 'SETUP'
120
    TEARDOWN = 'TEARDOWN'
121
    TIMEOUT = 'TIMEOUT'
122
    TEMPLATE = 'TEMPLATE'
123
    ARGUMENTS = 'ARGUMENTS'
124
    RETURN = 'RETURN'
125
    
126
    VARIABLE = 'VARIABLE'
127
    KEYWORD = 'KEYWORD'
128
    ASSIGN = 'ASSIGN'
129
    ARGUMENT = 'ARGUMENT'
130
    
131
    FOR = 'FOR'
132
    FOR_SEPARATOR = 'FOR SEPARATOR'
133
    END = 'END'
134
    IF = 'IF'
135
    INLINE_IF = 'INLINE IF'
136
    ELSE_IF = 'ELSE IF'
137
    ELSE = 'ELSE'
138
    TRY = 'TRY'
139
    EXCEPT = 'EXCEPT'
140
    FINALLY = 'FINALLY'
141
    WHILE = 'WHILE'
142
    
143
    COMMENT = 'COMMENT'
144
    CONTINUATION = 'CONTINUATION'
145
    EOL = 'EOL'
146
    EOS = 'EOS'  # End of statement
147
    EOF = 'EOF'  # End of file
148
```
149

150
**Usage Examples:**
151

152
```python
153
from robot.api.parsing import get_tokens, Token
154

155
def analyze_test_structure(source):
156
    """Analyze test file structure using tokens."""
157
    
158
    stats = {
159
        'test_cases': 0,
160
        'keywords': 0,
161
        'variables': 0,
162
        'comments': 0,
163
        'for_loops': 0,
164
        'if_statements': 0
165
    }
166
    
167
    for token in get_tokens(source):
168
        if token.type == Token.TESTCASE_NAME:
169
            stats['test_cases'] += 1
170
        elif token.type == Token.KEYWORD_NAME:
171
            stats['keywords'] += 1
172
        elif token.type == Token.VARIABLE:
173
            stats['variables'] += 1
174
        elif token.type == Token.COMMENT:
175
            stats['comments'] += 1
176
        elif token.type == Token.FOR:
177
            stats['for_loops'] += 1
178
        elif token.type == Token.IF:
179
            stats['if_statements'] += 1
180
    
181
    return stats
182

183
# Analyze test file
184
stats = analyze_test_structure('tests/complex.robot')
185
print(f"Test file contains: {stats}")
186
```
187

188
### AST Model Classes
189

190
Abstract syntax tree model classes representing different parts of Robot Framework test data.
191

192
```python { .api }
193
class File:
194
    """
195
    Root model representing a complete Robot Framework file.
196
    
197
    Attributes:
198
        source: Source file path
199
        sections: List of section objects
200
    """
201

202
# Section classes
203
class SettingSection:
204
    """Settings section (*** Settings ***)."""
205
    body: List  # Setting statements
206

207
class VariableSection:
208
    """Variables section (*** Variables ***)."""
209
    body: List  # Variable statements
210

211
class TestCaseSection:
212
    """Test Cases section (*** Test Cases ***)."""
213
    body: List  # TestCase objects
214

215
class KeywordSection:
216
    """Keywords section (*** Keywords ***)."""  
217
    body: List  # Keyword objects
218

219
class CommentSection:
220
    """Comments section (*** Comments ***)."""
221
    body: List  # Comment statements
222

223
# Block classes (can contain other blocks and statements)
224
class TestCase:
225
    """Individual test case definition."""
226
    header: TestCaseName
227
    body: List  # Keyword calls and control structures
228

229
class Keyword:
230
    """Individual keyword definition."""
231
    header: KeywordName
232
    body: List  # Keyword calls and control structures
233

234
# Control structure classes  
235
class If:
236
    """IF/ELSE control structure."""
237
    header: IfHeader
238
    body: List
239
    orelse: List  # ELSE IF and ELSE branches
240

241
class For:
242
    """FOR loop control structure."""  
243
    header: ForHeader
244
    body: List
245

246
class While:
247
    """WHILE loop control structure."""
248
    header: WhileHeader
249
    body: List
250

251
class Try:
252
    """TRY/EXCEPT control structure."""
253
    header: TryHeader
254
    body: List
255
    next: List  # EXCEPT and FINALLY branches
256
```
257

258
**Usage Examples:**
259

260
```python
261
from robot.api.parsing import get_model
262

263
def extract_test_keywords(source):
264
    """Extract all keywords used in test cases."""
265
    
266
    model = get_model(source)
267
    keywords_used = set()
268
    
269
    for section in model.sections:
270
        if hasattr(section, 'body'):
271
            for item in section.body:
272
                if hasattr(item, 'body'):  # Test case or keyword
273
                    for statement in item.body:
274
                        if hasattr(statement, 'keyword'):
275
                            keywords_used.add(statement.keyword)
276
    
277
    return sorted(keywords_used)
278

279
# Extract keywords from test file
280
keywords = extract_test_keywords('tests/suite.robot')
281
print(f"Keywords used: {keywords}")
282
```
283

284
### Model Visitor Pattern  
285

286
Process and traverse AST models using the visitor pattern.
287

288
```python { .api }
289
class ModelVisitor:
290
    """
291
    Base class for inspecting model objects.
292
    Override visit_* methods to process specific node types.
293
    """
294
    
295
    def visit(self, node): ...
296
    def generic_visit(self, node): ...
297
    
298
    # Visit methods for different node types
299
    def visit_File(self, node): ...
300
    def visit_Section(self, node): ...
301
    def visit_TestCase(self, node): ...
302
    def visit_Keyword(self, node): ...
303
    def visit_For(self, node): ...
304
    def visit_If(self, node): ...
305
    def visit_Statement(self, node): ...
306

307
class ModelTransformer:
308
    """
309
    Base class for modifying model objects.
310
    Override visit_* methods to transform specific node types.
311
    """
312
    
313
    def visit(self, node): ...
314
    def generic_visit(self, node): ...
315
    
316
    # Transform methods return modified nodes
317
    def visit_File(self, node): ...
318
    def visit_TestCase(self, node): ...
319
    def visit_Keyword(self, node): ...
320
```
321

322
**Usage Examples:**
323

324
```python
325
from robot.api.parsing import get_model, ModelVisitor, ModelTransformer
326

327
class TestCaseAnalyzer(ModelVisitor):
328
    """Analyze test cases for complexity metrics."""
329
    
330
    def __init__(self):
331
        self.test_stats = {}
332
        self.current_test = None
333
    
334
    def visit_TestCase(self, node):
335
        self.current_test = node.header.data_tokens[0].value
336
        self.test_stats[self.current_test] = {
337
            'keywords': 0,
338
            'for_loops': 0,
339
            'if_statements': 0,
340
            'assignments': 0
341
        }
342
        self.generic_visit(node)
343
    
344
    def visit_KeywordCall(self, node):
345
        if self.current_test:
346
            self.test_stats[self.current_test]['keywords'] += 1
347
    
348
    def visit_For(self, node):
349
        if self.current_test:
350
            self.test_stats[self.current_test]['for_loops'] += 1
351
        self.generic_visit(node)
352
    
353
    def visit_If(self, node):
354
        if self.current_test:
355
            self.test_stats[self.current_test]['if_statements'] += 1
356
        self.generic_visit(node)
357

358
class TagAdder(ModelTransformer):
359
    """Add tags to test cases based on content analysis."""
360
    
361
    def visit_TestCase(self, node):
362
        # Analyze test content to determine appropriate tags
363
        tags_to_add = []
364
        
365
        # Check if test uses database keywords
366
        for statement in node.body:
367
            if hasattr(statement, 'keyword'):
368
                keyword = statement.keyword.lower()
369
                if 'database' in keyword or 'sql' in keyword:
370
                    tags_to_add.append('database')
371
                elif 'web' in keyword or 'browser' in keyword:  
372
                    tags_to_add.append('web')
373
        
374
        # Add tags statement if new tags found
375
        if tags_to_add:
376
            # Create new tags statement
377
            tags_statement = Tags([Token(Token.TAGS, 'Tags')] + 
378
                                [Token(Token.ARGUMENT, tag) for tag in tags_to_add])
379
            node.body.insert(0, tags_statement)
380
        
381
        return node
382

383
# Use visitors to analyze and transform
384
model = get_model('tests/suite.robot')
385

386
# Analyze complexity
387
analyzer = TestCaseAnalyzer()
388
analyzer.visit(model)
389
print("Test complexity metrics:", analyzer.test_stats)
390

391
# Transform by adding tags
392
transformer = TagAdder()
393
modified_model = transformer.visit(model)
394

395
# Convert back to text
396
print(str(modified_model))
397
```
398

399
### Suite Structure Analysis
400

401
Analyze and work with test suite directory structures.
402

403
```python { .api }
404
class SuiteStructure:
405
    """
406
    Represents the structure of a test suite directory.
407
    
408
    Attributes:
409
        source: Root directory path
410
        init_file: Initialization file path (if exists)
411
        children: Child suite structures and test files
412
    """
413

414
class SuiteStructureBuilder:
415
    """Build suite structures from file system."""
416
    
417
    def build(self, source): ...
418

419
class SuiteStructureVisitor:
420
    """Process suite structures using visitor pattern."""
421
    
422
    def visit_directory(self, directory): ...
423
    def visit_file(self, file): ...
424
```
425

426
**Usage Examples:**
427

428
```python
429
from robot.parsing.suitestructure import SuiteStructureBuilder
430

431
# Analyze test suite directory structure
432
builder = SuiteStructureBuilder()
433
structure = builder.build('tests/')
434

435
def print_structure(structure, indent=0):
436
    """Print suite structure recursively."""
437
    prefix = "  " * indent
438
    print(f"{prefix}{structure.source}")
439
    
440
    if structure.init_file:
441
        print(f"{prefix}  __init__.robot")
442
    
443
    for child in structure.children:
444
        if hasattr(child, 'children'):  # Directory
445
            print_structure(child, indent + 1)
446
        else:  # Test file
447
            print(f"{prefix}  {child.source}")
448

449
print_structure(structure)
450
```
451

452
### Suite Processing
453

454
Process executable test suites before execution.
455

456
```python { .api }
457
class SuiteVisitor:
458
    """
459
    Abstract base class for processing test suites.
460
    Can be used as pre-run modifier (--prerunmodifier option).
461
    """
462
    
463
    def start_suite(self, suite): ...
464
    def end_suite(self, suite): ...
465
    def start_test(self, test): ...
466
    def end_test(self, test): ...
467
    def start_keyword(self, keyword): ...
468
    def end_keyword(self, keyword): ...
469
    def visit_suite(self, suite): ...
470
    def visit_test(self, test): ...
471
    def visit_keyword(self, keyword): ...
472
```
473

474
**Usage Examples:**
475

476
```python
477
from robot.api import SuiteVisitor
478

479
class TestFilter(SuiteVisitor):
480
    """Filter tests based on custom criteria."""
481
    
482
    def __init__(self, min_priority=1):
483
        self.min_priority = min_priority
484
    
485
    def start_suite(self, suite):
486
        # Filter tests based on priority tag
487
        original_tests = list(suite.tests)
488
        suite.tests.clear()
489
        
490
        for test in original_tests:
491
            priority = self._get_priority(test)
492
            if priority >= self.min_priority:
493
                suite.tests.append(test)
494
    
495
    def _get_priority(self, test):
496
        """Extract priority from test tags."""
497
        for tag in test.tags:
498
            if tag.startswith('priority:'):
499
                try:
500
                    return int(tag.split(':')[1])
501
                except ValueError:
502
                    pass
503
        return 0  # Default priority
504

505
# Use as pre-run modifier
506
class TestEnhancer(SuiteVisitor):
507
    """Enhance tests with additional setup/teardown."""
508
    
509
    def start_test(self, test):
510
        # Add timestamp variable to each test
511
        test.keywords.create(
512
            'Set Test Variable', 
513
            args=['${TEST_START_TIME}', '${CURTIME}'],
514
            assign=['${START_TIME}']
515
        ).insert(0)  # Insert at beginning
516
    
517
    def end_test(self, test):
518
        # Add cleanup keyword to each test
519
        test.teardown.config(
520
            name='Log Test Duration',
521
            args=['Test completed in ${CURTIME - START_TIME}']
522
        )
523
```
524

525
## Types
526

527
```python { .api }
528
# Token position information
529
LineNumber = int      # 1-based line number
530
ColumnOffset = int    # 0-based column offset
531

532
# Model node types
533
NodeType = Union[
534
    File, Section, TestCase, Keyword, 
535
    For, If, While, Try, Statement
536
]
537

538
# Parsing options
539
ParsingOptions = Dict[str, Any]  # language, etc.
540

541
# Statement types
542
StatementType = Union[
543
    # Settings
544
    Documentation, Tags, Setup, Teardown, Timeout, Template, Arguments, Return,
545
    # Variables  
546
    Variable,
547
    # Execution
548
    KeywordCall, TemplateArguments,
549
    # Control structures
550
    IfHeader, ElseIfHeader, ElseHeader, ForHeader, WhileHeader, TryHeader,
551
    ExceptHeader, FinallyHeader, End,
552
    # Other
553
    Comment, EmptyLine, Error
554
]
555
```