docs
evals
scenario-1
scenario-10
scenario-2
scenario-3
scenario-4
scenario-5
scenario-6
scenario-7
scenario-8
scenario-9
{
"context": "This criteria evaluates how effectively the engineer uses the cli package's option definition and parsing capabilities, specifically focusing on defining options with short and long names, setting appropriate types and defaults, and properly accessing parsed values.",
"type": "weighted_checklist",
"checklist": [
{
"name": "Uses cli.parse()",
"description": "The solution uses cli.parse() method to define and parse command-line options rather than implementing custom argument parsing",
"max_score": 20
},
{
"name": "Defines short/long names",
"description": "Each option is defined with both a short name (single letter) and long name (full word) using the cli package's option definition format, e.g., ['i', 'input', ...]",
"max_score": 25
},
{
"name": "Specifies option types",
"description": "Options are defined with appropriate type specifications (e.g., 'file', 'string', 'bool') using the cli package's type system",
"max_score": 15
},
{
"name": "Sets default values",
"description": "Options with defaults (like encoding) are properly defined with default values in the option definition array",
"max_score": 10
},
{
"name": "Accesses parsed options",
"description": "Parsed option values are accessed using cli.options object or the return value from cli.parse()",
"max_score": 15
},
{
"name": "Handles required options",
"description": "Required options (input/output files) are properly validated, checking for their presence in the parsed options",
"max_score": 10
},
{
"name": "Array-based enum validation",
"description": "The format option uses array-based validation to restrict values to the allowed set (uppercase, lowercase, titlecase)",
"max_score": 5
}
]
}