docs
evals
scenario-1
scenario-10
scenario-2
scenario-3
scenario-4
scenario-5
scenario-6
scenario-7
scenario-8
scenario-9
{
"context": "This criteria evaluates how well the engineer uses pnpm-specific configuration keys for resolution strategies, specifically focusing on the resolution-mode, registry-supports-time-field, and minimum-release-age settings in .npmrc files.",
"type": "weighted_checklist",
"checklist": [
{
"name": "resolution-mode configuration",
"description": "Correctly uses the 'resolution-mode' configuration key to set the resolution strategy to 'highest', 'time-based', or 'lowest-direct' in the .npmrc file",
"max_score": 30
},
{
"name": "registry-supports-time-field setting",
"description": "Correctly uses the 'registry-supports-time-field' configuration key to enable or disable time-based resolution support in the .npmrc file",
"max_score": 20
},
{
"name": "minimum-release-age configuration",
"description": "Correctly uses the 'minimum-release-age' or 'min-release-age' configuration key to set the minimum package age in days in the .npmrc file",
"max_score": 20
},
{
"name": ".npmrc file handling",
"description": "Properly reads from and writes to .npmrc files, preserving existing non-resolution configuration while updating resolution-specific settings",
"max_score": 15
},
{
"name": "Mode validation",
"description": "Validates that the resolution mode is one of the three supported values (highest, time-based, lowest-direct) and provides appropriate error messages for invalid inputs",
"max_score": 10
},
{
"name": "Configuration display",
"description": "Reads and displays the current resolution-mode, registry-supports-time-field, and minimum-release-age settings from the .npmrc file",
"max_score": 5
}
]
}