docs
evals
scenario-1
scenario-10
scenario-2
scenario-3
scenario-4
scenario-5
scenario-6
scenario-7
scenario-8
scenario-9
{
"context": "This evaluation assesses how well the engineer understands and implements pnpm's workspace filtering and package selection capabilities. The focus is on proper use of name-based filtering with glob pattern support, dependency graph traversal for including dependencies and dependents, and exclusion filter application.",
"type": "weighted_checklist",
"checklist": [
{
"name": "Name-based filtering",
"description": "Implements exact name matching that can identify packages by their exact package name from the workspace array. The implementation should iterate through workspace packages and match against the provided name filters.",
"max_score": 20
},
{
"name": "Glob pattern matching",
"description": "Implements glob pattern matching for package names (e.g., '@scope/*' matching all packages under a scope). Should use a glob matching library or implement pattern matching logic that handles wildcards and scope patterns common in pnpm workspaces.",
"max_score": 20
},
{
"name": "Dependency inclusion",
"description": "Implements forward dependency traversal to include all dependencies of matched packages. Should recursively follow the dependencies array of each matched package to build a complete set of direct and transitive dependencies.",
"max_score": 20
},
{
"name": "Dependent inclusion",
"description": "Implements reverse dependency traversal to include all packages that depend on matched packages. Should build a reverse dependency graph by examining which packages list the matched packages in their dependencies array.",
"max_score": 20
},
{
"name": "Exclusion filters",
"description": "Implements exclusion logic that removes specified packages from the result set after inclusion filters are applied. Should handle both exact names and glob patterns for exclusions.",
"max_score": 15
},
{
"name": "Result sorting",
"description": "Returns results as a sorted array of package names in alphabetical order, ensuring consistent and predictable output regardless of the order of filter application or workspace package ordering.",
"max_score": 5
}
]
}