0
# Low-level Processing
1
2
Token-based JSON processing for fine-grained control over parsing and custom JSON processing workflows.
3
4
## Capabilities
5
6
### JsonLexer
7
8
Streaming JSON lexer that tokenizes JSON input for low-level processing and custom parsing scenarios.
9
10
```java { .api }
11
/**
12
* Streaming JSON token lexer implementing Iterator<JsonToken>
13
*/
14
public class JsonLexer implements Iterator<JsonToken> {
15
/** Constructor with Reader input */
16
public JsonLexer(Reader reader);
17
18
/** Get the underlying LineColumnReader */
19
public LineColumnReader getReader();
20
21
/** Get the next JSON token */
22
public JsonToken nextToken();
23
24
/** Skip whitespace characters and return count */
25
public int skipWhitespace();
26
27
/** Iterator interface: check if more tokens available */
28
public boolean hasNext();
29
30
/** Iterator interface: get next token */
31
public JsonToken next();
32
33
/** Iterator interface: remove operation (throws UnsupportedOperationException) */
34
public void remove();
35
36
/** Unescape JSON string content */
37
public static String unescape(String input);
38
}
39
```
40
41
**Usage Examples:**
42
43
```java
44
import groovy.json.JsonLexer;
45
import groovy.json.JsonToken;
46
import groovy.json.JsonTokenType;
47
import java.io.StringReader;
48
49
// Basic tokenization
50
String jsonInput = '{"name":"Alice","age":30,"active":true}';
51
JsonLexer lexer = new JsonLexer(new StringReader(jsonInput));
52
53
while (lexer.hasNext()) {
54
JsonToken token = lexer.next();
55
System.out.println("Type: " + token.getType() +
56
", Value: " + token.getValue() +
57
", Position: " + token.getStartLine() + ":" + token.getStartColumn());
58
}
59
60
// Custom JSON validator
61
public boolean isValidJson(String json) {
62
try {
63
JsonLexer lexer = new JsonLexer(new StringReader(json));
64
int depth = 0;
65
boolean inObject = false, inArray = false;
66
67
while (lexer.hasNext()) {
68
JsonToken token = lexer.next();
69
switch (token.getType()) {
70
case OPEN_BRACE:
71
depth++;
72
inObject = true;
73
break;
74
case CLOSE_BRACE:
75
depth--;
76
if (depth < 0) return false;
77
break;
78
case OPEN_BRACKET:
79
depth++;
80
inArray = true;
81
break;
82
case CLOSE_BRACKET:
83
depth--;
84
if (depth < 0) return false;
85
break;
86
}
87
}
88
return depth == 0;
89
} catch (Exception e) {
90
return false;
91
}
92
}
93
94
// Extract specific values by path
95
public Object extractJsonValue(String json, String[] path) {
96
JsonLexer lexer = new JsonLexer(new StringReader(json));
97
// Implementation for path-based extraction
98
// ... custom logic using token stream
99
}
100
```
101
102
### JsonToken
103
104
Represents a single JSON token with value, type, and position information.
105
106
```java { .api }
107
/**
108
* Represents a single JSON token with position and value information
109
*/
110
public class JsonToken {
111
/** Get the parsed value of the token (Boolean, Number, String, or null) */
112
public Object getValue();
113
114
/** Get the token type */
115
public JsonTokenType getType();
116
public void setType(JsonTokenType type);
117
118
/** Get/set the raw text content */
119
public String getText();
120
public void setText(String text);
121
122
/** Position information - line numbers */
123
public long getStartLine();
124
public void setStartLine(long startLine);
125
public long getEndLine();
126
public void setEndLine(long endLine);
127
128
/** Position information - column numbers */
129
public long getStartColumn();
130
public void setStartColumn(long startColumn);
131
public long getEndColumn();
132
public void setEndColumn(long endColumn);
133
134
/** String representation of the token */
135
public String toString();
136
}
137
```
138
139
**Usage Examples:**
140
141
```java
142
// Token inspection and debugging
143
JsonToken token = lexer.nextToken();
144
if (token.getType() == JsonTokenType.STRING) {
145
System.out.println("String token: '" + token.getValue() + "' at " +
146
token.getStartLine() + ":" + token.getStartColumn());
147
}
148
149
// Error reporting with position
150
if (token.getType() == JsonTokenType.STRING && !isValidEmail((String) token.getValue())) {
151
throw new ValidationException("Invalid email at line " + token.getStartLine() +
152
", column " + token.getStartColumn());
153
}
154
```
155
156
### JsonTokenType
157
158
Enumeration of all possible JSON token types with validation and matching capabilities.
159
160
```java { .api }
161
/**
162
* Enumeration of all JSON token types
163
*/
164
public enum JsonTokenType {
165
/** Structural tokens */
166
OPEN_CURLY, // {
167
CLOSE_CURLY, // }
168
OPEN_BRACKET, // [
169
CLOSE_BRACKET, // ]
170
COLON, // :
171
COMMA, // ,
172
173
/** Literal value tokens */
174
NULL, // null
175
TRUE, // true
176
FALSE, // false
177
NUMBER, // numeric values
178
STRING, // string values
179
180
/** Check if input matches this token type pattern */
181
public boolean matching(String input);
182
183
/** Get descriptive label for this token type */
184
public String getLabel();
185
186
/** Get validator (String, Pattern, or Closure) */
187
public Object getValidator();
188
189
/** Find token type that starts with given character */
190
public static JsonTokenType startingWith(char c);
191
}
192
```
193
194
**Usage Examples:**
195
196
```java
197
// Token type checking
198
JsonToken token = lexer.nextToken();
199
switch (token.getType()) {
200
case OPEN_BRACE:
201
System.out.println("Starting object");
202
break;
203
case OPEN_BRACKET:
204
System.out.println("Starting array");
205
break;
206
case STRING:
207
System.out.println("String value: " + token.getValue());
208
break;
209
case NUMBER:
210
System.out.println("Numeric value: " + token.getValue());
211
break;
212
case TRUE:
213
case FALSE:
214
System.out.println("Boolean value: " + token.getValue());
215
break;
216
case NULL:
217
System.out.println("Null value");
218
break;
219
}
220
221
// Token type pattern matching
222
String input = "true";
223
if (JsonTokenType.TRUE.matching(input)) {
224
System.out.println("Input matches TRUE token");
225
}
226
227
// Find token type by starting character
228
JsonTokenType type = JsonTokenType.startingWith('{');
229
System.out.println("Token starting with '{': " + type); // OPEN_BRACE
230
```
231
232
## Advanced Low-level Processing
233
234
### Custom JSON Parser Implementation
235
236
```java
237
import groovy.json.JsonLexer;
238
import groovy.json.JsonToken;
239
import groovy.json.JsonTokenType;
240
241
public class CustomJsonParser {
242
private JsonLexer lexer;
243
private JsonToken currentToken;
244
245
public CustomJsonParser(Reader reader) {
246
this.lexer = new JsonLexer(reader);
247
advance();
248
}
249
250
private void advance() {
251
currentToken = lexer.hasNext() ? lexer.next() : null;
252
}
253
254
private void expect(JsonTokenType expectedType) {
255
if (currentToken == null || currentToken.getType() != expectedType) {
256
throw new RuntimeException("Expected " + expectedType +
257
" but found " +
258
(currentToken != null ? currentToken.getType() : "EOF"));
259
}
260
advance();
261
}
262
263
public Object parseValue() {
264
if (currentToken == null) {
265
throw new RuntimeException("Unexpected end of input");
266
}
267
268
switch (currentToken.getType()) {
269
case TRUE:
270
advance();
271
return true;
272
case FALSE:
273
advance();
274
return false;
275
case NULL:
276
advance();
277
return null;
278
case STRING:
279
String stringValue = (String) currentToken.getValue();
280
advance();
281
return stringValue;
282
case NUMBER:
283
Number numberValue = (Number) currentToken.getValue();
284
advance();
285
return numberValue;
286
case OPEN_BRACE:
287
return parseObject();
288
case OPEN_BRACKET:
289
return parseArray();
290
default:
291
throw new RuntimeException("Unexpected token: " + currentToken.getType());
292
}
293
}
294
295
private Map<String, Object> parseObject() {
296
Map<String, Object> object = new LinkedHashMap<>();
297
expect(JsonTokenType.OPEN_BRACE);
298
299
if (currentToken != null && currentToken.getType() == JsonTokenType.CLOSE_BRACE) {
300
advance();
301
return object;
302
}
303
304
do {
305
if (currentToken == null || currentToken.getType() != JsonTokenType.STRING) {
306
throw new RuntimeException("Expected string key");
307
}
308
String key = (String) currentToken.getValue();
309
advance();
310
311
expect(JsonTokenType.COLON);
312
Object value = parseValue();
313
object.put(key, value);
314
315
if (currentToken != null && currentToken.getType() == JsonTokenType.COMMA) {
316
advance();
317
} else {
318
break;
319
}
320
} while (true);
321
322
expect(JsonTokenType.CLOSE_BRACE);
323
return object;
324
}
325
326
private List<Object> parseArray() {
327
List<Object> array = new ArrayList<>();
328
expect(JsonTokenType.OPEN_BRACKET);
329
330
if (currentToken != null && currentToken.getType() == JsonTokenType.CLOSE_BRACKET) {
331
advance();
332
return array;
333
}
334
335
do {
336
array.add(parseValue());
337
338
if (currentToken != null && currentToken.getType() == JsonTokenType.COMMA) {
339
advance();
340
} else {
341
break;
342
}
343
} while (true);
344
345
expect(JsonTokenType.CLOSE_BRACKET);
346
return array;
347
}
348
}
349
```
350
351
### JSON Stream Processor
352
353
```java
354
// Process large JSON streams without loading entire document
355
public class JsonStreamProcessor {
356
public void processJsonStream(Reader reader, StreamHandler handler) {
357
JsonLexer lexer = new JsonLexer(reader);
358
Stack<String> path = new Stack<>();
359
360
while (lexer.hasNext()) {
361
JsonToken token = lexer.next();
362
363
switch (token.getType()) {
364
case OPEN_BRACE:
365
handler.startObject(getCurrentPath(path));
366
break;
367
case CLOSE_BRACE:
368
handler.endObject(getCurrentPath(path));
369
if (!path.isEmpty()) path.pop();
370
break;
371
case OPEN_BRACKET:
372
handler.startArray(getCurrentPath(path));
373
break;
374
case CLOSE_BRACKET:
375
handler.endArray(getCurrentPath(path));
376
if (!path.isEmpty()) path.pop();
377
break;
378
case STRING:
379
if (isPropertyName(lexer)) {
380
path.push((String) token.getValue());
381
} else {
382
handler.stringValue(getCurrentPath(path), (String) token.getValue());
383
}
384
break;
385
case NUMBER:
386
handler.numberValue(getCurrentPath(path), (Number) token.getValue());
387
break;
388
case TRUE:
389
case FALSE:
390
handler.booleanValue(getCurrentPath(path), (Boolean) token.getValue());
391
break;
392
case NULL:
393
handler.nullValue(getCurrentPath(path));
394
break;
395
}
396
}
397
}
398
399
private String getCurrentPath(Stack<String> path) {
400
return String.join(".", path);
401
}
402
403
interface StreamHandler {
404
void startObject(String path);
405
void endObject(String path);
406
void startArray(String path);
407
void endArray(String path);
408
void stringValue(String path, String value);
409
void numberValue(String path, Number value);
410
void booleanValue(String path, Boolean value);
411
void nullValue(String path);
412
}
413
}
414
```
415
416
### JSON Validation with Error Location
417
418
```java
419
public class JsonValidator {
420
public static class ValidationError {
421
private final String message;
422
private final long line;
423
private final long column;
424
425
public ValidationError(String message, JsonToken token) {
426
this.message = message;
427
this.line = token.getStartLine();
428
this.column = token.getStartColumn();
429
}
430
431
public String getMessage() { return message; }
432
public long getLine() { return line; }
433
public long getColumn() { return column; }
434
435
@Override
436
public String toString() {
437
return String.format("%s at line %d, column %d", message, line, column);
438
}
439
}
440
441
public static List<ValidationError> validate(String json) {
442
List<ValidationError> errors = new ArrayList<>();
443
444
try {
445
JsonLexer lexer = new JsonLexer(new StringReader(json));
446
Stack<JsonTokenType> stack = new Stack<>();
447
448
while (lexer.hasNext()) {
449
JsonToken token = lexer.next();
450
451
switch (token.getType()) {
452
case OPEN_BRACE:
453
stack.push(JsonTokenType.OPEN_BRACE);
454
break;
455
case CLOSE_BRACE:
456
if (stack.isEmpty() || stack.pop() != JsonTokenType.OPEN_BRACE) {
457
errors.add(new ValidationError("Unmatched closing brace", token));
458
}
459
break;
460
case OPEN_BRACKET:
461
stack.push(JsonTokenType.OPEN_BRACKET);
462
break;
463
case CLOSE_BRACKET:
464
if (stack.isEmpty() || stack.pop() != JsonTokenType.OPEN_BRACKET) {
465
errors.add(new ValidationError("Unmatched closing bracket", token));
466
}
467
break;
468
}
469
}
470
471
if (!stack.isEmpty()) {
472
errors.add(new ValidationError("Unclosed " +
473
(stack.peek() == JsonTokenType.OPEN_BRACE ? "object" : "array"), null));
474
}
475
476
} catch (Exception e) {
477
errors.add(new ValidationError("Parse error: " + e.getMessage(), null));
478
}
479
480
return errors;
481
}
482
}
483
```
484
485
## Performance Considerations
486
487
### Memory-Efficient Processing
488
489
```java
490
// Process large JSON files without loading into memory
491
public void processLargeJsonFile(File jsonFile) throws IOException {
492
try (FileReader reader = new FileReader(jsonFile);
493
BufferedReader buffered = new BufferedReader(reader)) {
494
495
JsonLexer lexer = new JsonLexer(buffered);
496
497
// Process tokens as they're read
498
while (lexer.hasNext()) {
499
JsonToken token = lexer.next();
500
processToken(token);
501
502
// Periodically yield to prevent blocking
503
if (Thread.interrupted()) {
504
throw new InterruptedException("Processing interrupted");
505
}
506
}
507
}
508
}
509
```
510
511
### Token Filtering and Optimization
512
513
```java
514
// Skip unwanted tokens for performance
515
public List<String> extractStringValues(String json) {
516
List<String> strings = new ArrayList<>();
517
JsonLexer lexer = new JsonLexer(new StringReader(json));
518
519
while (lexer.hasNext()) {
520
JsonToken token = lexer.next();
521
if (token.getType() == JsonTokenType.STRING) {
522
strings.add((String) token.getValue());
523
}
524
// Skip other token types for performance
525
}
526
527
return strings;
528
}
529
```