docs
evals
scenario-1
scenario-10
scenario-2
scenario-3
scenario-4
scenario-5
scenario-6
scenario-7
scenario-8
scenario-9
{
"context": "This criteria evaluates how well the engineer uses js-yaml and glob packages to implement catalog-based version management functionality. The focus is on proper usage of YAML parsing for workspace configuration, glob pattern matching for package discovery, and correct implementation of catalog reference resolution.",
"type": "weighted_checklist",
"checklist": [
{
"name": "YAML parsing",
"description": "Uses js-yaml (or equivalent YAML library) to parse the workspace configuration file. Should use functions like yaml.load() or yaml.parse() to read the YAML content.",
"max_score": 20
},
{
"name": "Glob pattern matching",
"description": "Uses glob package to match workspace packages from patterns. Should use glob() or glob.sync() to resolve package paths from patterns specified in the configuration.",
"max_score": 20
},
{
"name": "Catalog extraction",
"description": "Correctly extracts and stores the catalog object from the parsed workspace configuration as a key-value mapping of package names to versions.",
"max_score": 15
},
{
"name": "Catalog reference detection",
"description": "Identifies dependencies in package.json files that use catalog references (values starting with 'catalog:'). Correctly distinguishes these from regular version specifications.",
"max_score": 15
},
{
"name": "Version replacement",
"description": "Replaces catalog: references with actual versions from the catalog object. Correctly handles the string manipulation to extract package name from 'catalog:' format and look up the corresponding version.",
"max_score": 15
},
{
"name": "Report generation",
"description": "Generates and returns a synchronization report containing counts of packages processed, dependencies updated, and lists any unresolved catalog references.",
"max_score": 10
},
{
"name": "Error handling",
"description": "Properly handles missing catalog entries without crashing. Reports unresolved references and continues processing other dependencies.",
"max_score": 5
}
]
}