evals
scenario-1
scenario-10
scenario-2
scenario-3
scenario-4
scenario-5
scenario-6
scenario-7
scenario-8
scenario-9
{
"context": "This criteria evaluates how well the engineer uses Nx's release and changelog generation capabilities to build a command-line tool. The focus is on proper usage of Nx's release commands, handling of conventional commits, and correct implementation of versioning and dry-run features.",
"type": "weighted_checklist",
"checklist": [
{
"name": "Nx release command",
"description": "Uses `nx release changelog` command or the programmatic equivalent from Nx's API to generate changelogs",
"max_score": 25
},
{
"name": "Version handling",
"description": "Correctly passes the version parameter to Nx release commands using the --version flag or equivalent API option",
"max_score": 15
},
{
"name": "Dry-run implementation",
"description": "Implements dry-run mode using Nx's --dry-run flag or equivalent API option to preview changes without modifying files",
"max_score": 15
},
{
"name": "Git reference range",
"description": "Uses --from and --to flags (or equivalent API options) to specify git reference ranges for changelog generation",
"max_score": 15
},
{
"name": "Workspace integration",
"description": "Uses Nx workspace APIs (@nx/devkit) such as readProjectConfiguration, getProjects, or similar to access project information",
"max_score": 10
},
{
"name": "Conventional commits",
"description": "Leverages Nx's built-in conventional commits parsing rather than implementing custom commit parsing logic",
"max_score": 10
},
{
"name": "Error handling",
"description": "Properly handles Nx command failures and edge cases (e.g., no changes, invalid git references) using Nx's error responses",
"max_score": 10
}
]
}