evals
scenario-1
scenario-2
scenario-3
scenario-4
scenario-5
scenario-6
scenario-7
scenario-8
scenario-9
scenario-10
{
"context": "This evaluation assesses how well the engineer uses the @mdx-js/mdx package to create reusable processors for efficient batch compilation of MDX files. The focus is on correct usage of processor creation APIs and efficient processor reuse.",
"type": "weighted_checklist",
"checklist": [
{
"name": "createProcessor usage",
"description": "Uses the createProcessor() function from @mdx-js/mdx to create a reusable processor instance rather than using compile() or compileSync() for each file",
"max_score": 30
},
{
"name": "Processor reuse",
"description": "Reuses the same processor instance across multiple files by calling processor.process() or processor.processSync() on the processor returned by createProcessor(), rather than creating a new processor for each file",
"max_score": 25
},
{
"name": "VFile handling",
"description": "Properly handles VFile objects by passing file content and path information to the processor, and extracts the compiled code from the VFile result (e.g., accessing result.value or result.toString())",
"max_score": 20
},
{
"name": "Error handling",
"description": "Implements try-catch blocks or error handling for processor.process() calls to gracefully handle compilation errors and return structured results with success/error status",
"max_score": 15
},
{
"name": "Format detection",
"description": "Leverages the processor's built-in format detection capability by providing file paths with extensions (e.g., .mdx, .md) so the processor can automatically determine the appropriate format",
"max_score": 10
}
]
}