evals
scenario-1
scenario-10
scenario-2
scenario-3
scenario-4
scenario-5
scenario-6
scenario-7
scenario-8
scenario-9
{
"context": "This criteria evaluates how effectively the engineer uses picomatch's token-level pattern analysis capabilities, specifically the scan() and parse() functions, to implement a glob pattern analyzer.",
"type": "weighted_checklist",
"checklist": [
{
"name": "Uses picomatch.scan()",
"description": "The analyzePattern function uses picomatch.scan() to analyze pattern structure and return metadata about the pattern.",
"max_score": 25
},
{
"name": "Returns scan metadata",
"description": "The analyzePattern function correctly returns the relevant scan metadata fields: isGlob, base, glob, negated, isGlobstar, isExtglob, and isBrace.",
"max_score": 20
},
{
"name": "Uses picomatch.parse()",
"description": "The parsePattern function uses picomatch.parse() to perform full pattern parsing and tokenization.",
"max_score": 25
},
{
"name": "Returns parse state",
"description": "The parsePattern function returns the detailed state object from picomatch.parse() containing token array and regex output information.",
"max_score": 20
},
{
"name": "Correct picomatch import",
"description": "The implementation correctly imports or requires the picomatch package.",
"max_score": 10
}
]
}