evals
scenario-1
scenario-10
scenario-2
scenario-3
scenario-4
scenario-5
scenario-6
scenario-7
scenario-8
scenario-9
{
"context": "This criteria evaluates how effectively the engineer utilizes the czg package for Git hook integration, specifically focusing on proper use of the --hook flag, configuration loading, and the core API for interactive commit message generation.",
"type": "weighted_checklist",
"checklist": [
{
"name": "czg invocation",
"description": "Uses czg as the primary tool for commit message generation. The implementation should invoke czg programmatically or via CLI rather than reimplementing commit message prompts from scratch.",
"max_score": 25
},
{
"name": "--hook flag usage",
"description": "Correctly implements hook integration mode by passing the --hook flag to czg when running in hook mode, which causes czg to write to .git/COMMIT_EDITMSG instead of executing git commit directly.",
"max_score": 25
},
{
"name": "Configuration integration",
"description": "Properly integrates with czg's configuration system by passing or referencing .czrc configuration, allowing custom types and scopes to be used. May use czg's built-in config loading or pass config via command-line options.",
"max_score": 20
},
{
"name": "Mode switching",
"description": "Correctly implements different behavior for hook mode vs normal mode - in hook mode czg writes to .git/COMMIT_EDITMSG, in normal mode czg executes the commit. This should leverage czg's built-in capabilities rather than manual file handling.",
"max_score": 20
},
{
"name": "Package dependency",
"description": "Includes czg as a dependency in package.json and properly sets up the project to use it (e.g., npm scripts, proper installation, correct import/require if using programmatically).",
"max_score": 10
}
]
}