evals
scenario-1
scenario-10
scenario-2
scenario-3
scenario-4
scenario-5
scenario-6
scenario-7
scenario-8
scenario-9
{
"context": "This criteria evaluates how well the engineer leverages oboe.js's error resilience and isolation capabilities when transformation errors occur in callbacks. The focus is on proper use of streaming JSON parsing with .node() callbacks, implementing error handling that allows the stream to continue despite exceptions, and tracking both successful results and errors.",
"type": "weighted_checklist",
"checklist": [
{
"name": "Uses oboe() function",
"description": "Correctly initializes oboe with the provided URL using the oboe() function to establish the streaming JSON parser.",
"max_score": 15
},
{
"name": "Uses .node() method",
"description": "Uses the .node() method with appropriate JSONPath pattern (e.g., '!.users.*' or '.users.*') to register callbacks for processing user records as they arrive in the stream.",
"max_score": 20
},
{
"name": "Handles callback errors",
"description": "Implements error handling (e.g., try-catch) inside the .node() callback to catch transformation errors (like .toUpperCase() on undefined) without stopping the stream, demonstrating understanding of error resilience.",
"max_score": 20
},
{
"name": "Continues stream on errors",
"description": "Ensures the oboe stream continues processing subsequent records even after transformation errors occur in previous callbacks, leveraging oboe's error isolation mechanism that prevents callback failures from crashing the parser.",
"max_score": 15
},
{
"name": "Tracks successful results",
"description": "Collects successfully processed records in an array during the .node() callback execution, properly accumulating all valid records.",
"max_score": 10
},
{
"name": "Tracks error information",
"description": "Maintains an errors array that captures information about failed records (such as index and error message) alongside successful results.",
"max_score": 10
},
{
"name": "Uses .done() method",
"description": "Uses the .done() method to register a completion callback that is invoked when the entire JSON stream has been parsed, regardless of individual callback errors.",
"max_score": 10
}
]
}