docs
evals
scenario-1
scenario-10
scenario-2
scenario-3
scenario-4
scenario-5
scenario-6
scenario-7
scenario-8
scenario-9
{
"context": "This evaluation assesses how well the engineer uses Karma's plugin system and file injection mechanisms to automatically inject framework files into the test environment. The focus is on proper usage of Karma's factory pattern, file list manipulation, and file pattern configuration APIs.",
"type": "weighted_checklist",
"checklist": [
{
"name": "Plugin Registration",
"description": "Uses Karma's factory pattern correctly by exporting an object with the framework name as key (e.g., 'framework:testframework') and ['factory', functionName] as value",
"max_score": 15
},
{
"name": "Module Resolution",
"description": "Uses require.resolve() to locate framework files from node_modules (e.g., require.resolve('mock-framework/dist/framework.js'))",
"max_score": 20
},
{
"name": "File List Injection",
"description": "Correctly uses files.unshift() method to inject files into Karma's file list at the beginning",
"max_score": 15
},
{
"name": "File Pattern Objects",
"description": "Creates proper file pattern objects with the pattern property containing the resolved file path",
"max_score": 15
},
{
"name": "Pattern Configuration",
"description": "Configures file patterns with correct properties: included: true, served: true, watched: false, and nocache: true",
"max_score": 20
},
{
"name": "JavaScript Injection",
"description": "Injects the framework JavaScript library file from node_modules into the files array",
"max_score": 5
},
{
"name": "CSS Injection",
"description": "Injects the framework CSS file from node_modules into the files array",
"max_score": 5
},
{
"name": "Adapter Injection",
"description": "Injects a local adapter script file into the files array",
"max_score": 5
}
]
}