docs
evals
scenario-1
scenario-10
scenario-2
scenario-3
scenario-4
scenario-5
scenario-6
scenario-7
scenario-8
scenario-9
{
"context": "This evaluation assesses how effectively the engineer uses pnpm's peer dependency resolution capabilities to implement a package installation tool. The focus is on proper usage of pnpm APIs for installing packages, handling peer dependencies, and detecting conflicts.",
"type": "weighted_checklist",
"checklist": [
{
"name": "pnpm Installation API",
"description": "Uses pnpm's programmatic API (e.g., pnpm.install() or equivalent) to install packages rather than spawning shell commands",
"max_score": 25
},
{
"name": "Peer Dependency Detection",
"description": "Correctly uses pnpm's APIs to detect and retrieve peer dependency requirements (e.g., through package resolution or metadata inspection)",
"max_score": 20
},
{
"name": "Auto-install Peers Option",
"description": "Implements the autoInstallPeers option using pnpm's --auto-install-peers or equivalent configuration",
"max_score": 20
},
{
"name": "Strict Peer Validation",
"description": "Implements strict peer dependency checking using pnpm's --strict-peer-dependencies or equivalent configuration",
"max_score": 15
},
{
"name": "Conflict Detection",
"description": "Uses pnpm's resolution capabilities to identify peer dependency conflicts between packages",
"max_score": 15
},
{
"name": "Installation Result Handling",
"description": "Properly captures and returns installation results, including which packages were installed and any peer-related warnings or errors",
"max_score": 5
}
]
}