evals
scenario-1
scenario-10
scenario-2
scenario-3
scenario-4
scenario-5
scenario-6
scenario-7
scenario-8
scenario-9
{
"context": "This criteria evaluates how well the engineer leverages cz-git's issue tracking integration capabilities to build a commit footer generator. The focus is on proper use of the package's issue prefix configuration features, handling of custom prefixes, multiple issue formatting, and adherence to the conventional commit footer format.",
"type": "weighted_checklist",
"checklist": [
{
"name": "Issue prefix configuration",
"description": "Uses cz-git's issuePrefixes configuration option to define available prefix choices with value and name properties",
"max_score": 25
},
{
"name": "Custom prefix support",
"description": "Implements allowCustomIssuePrefix configuration to enable user-defined prefixes beyond the predefined list",
"max_score": 20
},
{
"name": "Empty prefix handling",
"description": "Implements allowEmptyIssuePrefix configuration to support issue references without a prefix",
"max_score": 15
},
{
"name": "Multiple issues formatting",
"description": "Correctly formats multiple issue references with a single prefix, joining them with commas and spaces (e.g., 'closes #100, #101, #102')",
"max_score": 20
},
{
"name": "Prefix validation",
"description": "Validates prefixes against the configuration, checking if a prefix is in the predefined list or if custom/empty prefixes are allowed",
"max_score": 20
}
]
}