evals
scenario-1
scenario-10
scenario-2
scenario-3
scenario-4
scenario-5
scenario-6
scenario-7
scenario-8
scenario-9
{
"context": "This criteria evaluates how well the engineer uses the openapi-types package's version-agnostic union types to implement a document analyzer that works across OpenAPI 2.0, 3.0, and 3.1 specifications.",
"type": "weighted_checklist",
"checklist": [
{
"name": "OpenAPI.Document usage",
"description": "Uses the OpenAPI.Document union type to accept any version of OpenAPI document in the function signature or type guards",
"max_score": 30
},
{
"name": "Version detection",
"description": "Correctly extracts the version string from the document by accessing appropriate version fields (swagger for v2.0, openapi for v3.0/3.1)",
"max_score": 20
},
{
"name": "Operation counting",
"description": "Correctly iterates through paths and HTTP methods to count all operations regardless of OpenAPI version",
"max_score": 25
},
{
"name": "Parameter counting",
"description": "Correctly counts parameters across all operations, handling both OpenAPI 2.0 and 3.0+ parameter structures",
"max_score": 25
}
]
}