0
# Testing and Debugging
1
2
Testing utilities and debugging tools for parser development. PyParsing provides comprehensive facilities for testing parser behavior, debugging parsing issues, and validating grammar correctness through built-in test runners and diagnostic tools.
3
4
## Capabilities
5
6
### Testing Utilities
7
8
The `pyparsing_test` class provides methods for testing and validating parser behavior.
9
10
```python { .api }
11
class pyparsing_test:
12
"""Testing utilities for pyparsing expressions."""
13
14
@staticmethod
15
def with_line_numbers(s: str, start_line: int = 1) -> str:
16
"""Add line numbers to a string for easier debugging."""
17
18
@staticmethod
19
def replace_htmlentity(t: ParseResults) -> str:
20
"""Replace HTML entities in parsed results."""
21
```
22
23
### Parse Action Debugging
24
25
Functions for tracing and debugging parse actions.
26
27
```python { .api }
28
def trace_parse_action(f: callable) -> callable:
29
"""Decorator to trace parse action execution."""
30
31
def null_debug_action(*args) -> None:
32
"""No-op debug action for testing."""
33
```
34
35
**Usage examples:**
36
```python
37
# Trace parse action execution
38
@trace_parse_action
39
def convert_to_int(tokens):
40
return int(tokens[0])
41
42
number_parser = Word(nums).set_parse_action(convert_to_int)
43
44
# Use null action for testing
45
test_parser = Word(alphas).set_parse_action(null_debug_action)
46
```
47
48
### Parser Element Debugging
49
50
Built-in debugging capabilities for parser elements.
51
52
```python { .api }
53
class ParserElement:
54
def set_debug(self, flag: bool = True) -> ParserElement:
55
"""Enable/disable debug output for this element."""
56
57
def run_tests(self, tests: str,
58
parse_all: bool = True,
59
comment: str = '#',
60
full_dump: bool = True,
61
print_results: bool = True,
62
failure_tests: bool = False) -> tuple:
63
"""Run a series of test strings against this parser."""
64
```
65
66
**Usage examples:**
67
```python
68
# Enable debugging for specific parser elements
69
number = Word(nums).set_debug()
70
operator = oneOf("+ - * /").set_debug()
71
expr = number + operator + number
72
73
# Run comprehensive tests
74
test_cases = """
75
# Valid expressions
76
5 + 3
77
10 - 2
78
7 * 4
79
80
# Invalid expressions (should fail)
81
5 +
82
* 3
83
5 & 3
84
"""
85
86
results = expr.run_tests(test_cases, failure_tests=True)
87
```
88
89
### Test Running and Validation
90
91
Methods for running systematic tests against parsers.
92
93
**Basic test running:**
94
```python
95
# Simple test cases
96
parser = Word(alphas) + Word(nums)
97
98
test_strings = """
99
hello 123
100
world 456
101
test 789
102
"""
103
104
# Run tests with automatic validation
105
results = parser.run_tests(test_strings)
106
print(f"Passed: {results[0]}, Failed: {results[1]}")
107
```
108
109
**Advanced test configuration:**
110
```python
111
# Comprehensive test setup
112
email_parser = Regex(r'[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}')
113
114
email_tests = """
115
# Valid email addresses
116
user@example.com
117
test.email+tag@domain.co.uk
118
user123@test-domain.com
119
120
# Invalid email addresses (failure tests)
121
@example.com
122
user@
123
user..name@example.com
124
"""
125
126
# Run with custom settings
127
results = email_parser.run_tests(
128
email_tests,
129
parse_all=True, # Require complete string match
130
comment='#', # Comment character
131
full_dump=True, # Show full parse results
132
print_results=True, # Print results to console
133
failure_tests=True # Include tests that should fail
134
)
135
```
136
137
### Diagnostic Configuration
138
139
Global diagnostic settings for debugging parser behavior.
140
141
```python { .api }
142
class __diag__:
143
"""Diagnostic configuration for pyparsing."""
144
145
warn_multiple_tokens_in_named_alternation: bool
146
warn_ungrouped_named_tokens_in_collection: bool
147
warn_name_set_on_empty_Forward: bool
148
warn_on_parse_using_empty_Forward: bool
149
warn_on_assignment_to_Forward: bool
150
warn_on_multiple_string_args_to_oneof: bool
151
enable_debug_on_named_expressions: bool
152
153
def enable_diag(diag_enum) -> None:
154
"""Enable specific diagnostic option."""
155
156
def disable_diag(diag_enum) -> None:
157
"""Disable specific diagnostic option."""
158
```
159
160
**Usage examples:**
161
```python
162
# Enable specific diagnostics
163
from pyparsing import __diag__, enable_diag, disable_diag
164
165
# Enable warning for multiple tokens in named alternations
166
enable_diag(__diag__.warn_multiple_tokens_in_named_alternation)
167
168
# Enable debug output for all named expressions
169
enable_diag(__diag__.enable_debug_on_named_expressions)
170
```
171
172
### Interactive Testing and Development
173
174
Utilities for interactive parser development and testing.
175
176
**Interactive testing pattern:**
177
```python
178
def test_parser_interactively(parser, name="parser"):
179
"""Interactive testing function for development."""
180
print(f"Testing {name}. Enter 'quit' to exit.")
181
182
while True:
183
try:
184
test_input = input(f"{name}> ")
185
if test_input.lower() == 'quit':
186
break
187
188
result = parser.parse_string(test_input, parse_all=True)
189
print(f"SUCCESS: {result}")
190
print(f"Type: {type(result)}")
191
192
except ParseException as pe:
193
print(f"PARSE ERROR: {pe}")
194
print(f"Location: Line {pe.lineno}, Col {pe.col}")
195
print(f"Context: {pe.mark_input_line()}")
196
except Exception as e:
197
print(f"ERROR: {e}")
198
199
# Usage
200
arithmetic_parser = infix_notation(Word(nums), [
201
('+', 2, opAssoc.LEFT),
202
('*', 3, opAssoc.LEFT),
203
])
204
205
# test_parser_interactively(arithmetic_parser, "arithmetic")
206
```
207
208
### Performance Testing
209
210
Methods for testing parser performance and optimization.
211
212
**Performance measurement:**
213
```python
214
import time
215
from pyparsing import *
216
217
def benchmark_parser(parser, test_data, iterations=1000):
218
"""Benchmark parser performance."""
219
220
start_time = time.time()
221
222
for _ in range(iterations):
223
for test_string in test_data:
224
try:
225
parser.parse_string(test_string)
226
except ParseException:
227
pass # Ignore parsing failures for benchmarking
228
229
end_time = time.time()
230
total_time = end_time - start_time
231
232
print(f"Parser benchmarked:")
233
print(f" Total time: {total_time:.4f} seconds")
234
print(f" Iterations: {iterations}")
235
print(f" Test strings: {len(test_data)}")
236
print(f" Average per parse: {total_time / (iterations * len(test_data)) * 1000:.4f} ms")
237
238
# Example usage
239
csv_parser = delimited_list(Word(alphanums + "._-"))
240
test_data = [
241
"apple,banana,cherry",
242
"one,two,three,four,five",
243
"test.file,data.csv,results.txt"
244
]
245
246
benchmark_parser(csv_parser, test_data)
247
```
248
249
### Debugging Complex Grammars
250
251
Strategies for debugging complex recursive grammars.
252
253
**Grammar debugging pattern:**
254
```python
255
def debug_grammar():
256
"""Example of debugging a complex grammar."""
257
258
# Enable comprehensive debugging
259
enable_diag(__diag__.enable_debug_on_named_expressions)
260
261
# Create grammar with meaningful names
262
expr = Forward().set_name("expression")
263
term = Forward().set_name("term")
264
factor = Forward().set_name("factor")
265
266
number = Word(nums).set_name("number")
267
identifier = Word(alphas).set_name("identifier")
268
269
factor <<= (number | identifier | ("(" + expr + ")")).set_name("factor_def")
270
term <<= (factor + ZeroOrMore(("*" | "/") + factor)).set_name("term_def")
271
expr <<= (term + ZeroOrMore(("+" | "-") + term)).set_name("expr_def")
272
273
# Enable debugging on key elements
274
expr.set_debug()
275
term.set_debug()
276
factor.set_debug()
277
278
# Test with problematic input
279
test_input = "2 + 3 * (4 - 1)"
280
281
try:
282
result = expr.parse_string(test_input)
283
print(f"Parse successful: {result}")
284
except ParseException as pe:
285
print(f"Parse failed: {pe}")
286
print(pe.explain())
287
288
# debug_grammar()
289
```
290
291
### Unit Testing Integration
292
293
Integration with Python's unittest framework.
294
295
```python
296
import unittest
297
from pyparsing import *
298
299
class TestMyParsers(unittest.TestCase):
300
"""Unit tests for custom parsers."""
301
302
def setUp(self):
303
"""Set up test parsers."""
304
self.number_parser = Word(nums).set_parse_action(lambda t: int(t[0]))
305
self.email_parser = Regex(r'[^@]+@[^@]+\.[^@]+')
306
307
def test_number_parsing(self):
308
"""Test number parser."""
309
result = self.number_parser.parse_string("123")
310
self.assertEqual(result[0], 123)
311
self.assertIsInstance(result[0], int)
312
313
def test_number_parsing_failure(self):
314
"""Test number parser failure cases."""
315
with self.assertRaises(ParseException):
316
self.number_parser.parse_string("abc")
317
318
def test_email_parsing(self):
319
"""Test email parser."""
320
result = self.email_parser.parse_string("user@example.com")
321
self.assertEqual(result[0], "user@example.com")
322
323
def test_multiple_test_cases(self):
324
"""Test multiple cases efficiently."""
325
test_cases = [
326
("123", [123]),
327
("456", [456]),
328
("789", [789]),
329
]
330
331
for input_str, expected in test_cases:
332
with self.subTest(input_str=input_str):
333
result = self.number_parser.parse_string(input_str)
334
self.assertEqual(result.as_list(), expected)
335
336
# Run tests
337
if __name__ == '__main__':
338
unittest.main()
339
```