evals
scenario-1
scenario-10
scenario-2
scenario-3
scenario-4
scenario-5
scenario-6
scenario-7
scenario-8
scenario-9
{
"context": "This criteria evaluates how well the engineer leverages the czg package to implement Conventional Commits validation. The focus is on utilizing czg's built-in validation capabilities, configuration system, and type definitions rather than implementing validation from scratch.",
"type": "weighted_checklist",
"checklist": [
{
"name": "Uses czg configuration",
"description": "Uses czg's configuration system (types from CommitizenGitOptions or UserConfig) to define valid commit types and validation rules",
"max_score": 25
},
{
"name": "Leverages type validation",
"description": "Utilizes czg's built-in commit type validation (feat, fix, docs, style, refactor, perf, test, build, ci, chore) rather than implementing custom type checking",
"max_score": 20
},
{
"name": "Parses commit format",
"description": "Uses czg's commit message parsing capabilities or follows its Conventional Commits format specification for extracting type, scope, and subject",
"max_score": 20
},
{
"name": "Handles breaking changes",
"description": "Implements breaking change detection using czg's markBreakingChangeMode or allowBreakingChanges configuration options to validate ! indicators",
"max_score": 15
},
{
"name": "Validates scope format",
"description": "Uses czg's scope validation features (scopeOverrides, scopes configuration, or scope regex patterns) to ensure proper scope formatting",
"max_score": 10
},
{
"name": "Returns structured result",
"description": "Returns validation results in a format consistent with czg's validation approach (Answers type or similar structured object with valid/error information)",
"max_score": 10
}
]
}