evals
scenario-1
scenario-10
scenario-2
scenario-3
scenario-4
scenario-5
scenario-6
scenario-7
scenario-8
scenario-9
{
"context": "This criteria evaluates how effectively an engineer uses the openapi-types package to work with OpenAPI 3.0 request and response type definitions. The focus is on proper usage of type annotations for ParameterObject, RequestBodyObject, ResponseObject, and MediaTypeObject.",
"type": "weighted_checklist",
"checklist": [
{
"name": "Import OpenAPIV3 types",
"description": "Correctly imports OpenAPIV3 namespace from openapi-types package to access OpenAPI 3.0 type definitions.",
"max_score": 10
},
{
"name": "Type ParameterObject",
"description": "Uses OpenAPIV3.ParameterObject type annotation when working with operation parameters, demonstrating understanding of the parameter structure including name, in, and schema properties.",
"max_score": 20
},
{
"name": "Type RequestBodyObject",
"description": "Uses OpenAPIV3.RequestBodyObject type annotation when handling request body definitions, showing knowledge of the content and required properties.",
"max_score": 20
},
{
"name": "Type ResponseObject",
"description": "Uses OpenAPIV3.ResponseObject type annotation when processing response definitions, demonstrating understanding of description, headers, content, and links properties.",
"max_score": 20
},
{
"name": "Type MediaTypeObject",
"description": "Uses OpenAPIV3.MediaTypeObject type annotation when extracting media types from content objects, showing knowledge of schema, example, examples, and encoding properties.",
"max_score": 15
},
{
"name": "Type ReferenceObject handling",
"description": "Properly handles OpenAPIV3.ReferenceObject union types (e.g., ReferenceObject | ParameterObject, ReferenceObject | ResponseObject) when working with components that may be references.",
"max_score": 10
},
{
"name": "Type OperationObject",
"description": "Uses OpenAPIV3.OperationObject type annotation for the input parameter, demonstrating comprehensive understanding of the operation structure including parameters, requestBody, and responses properties.",
"max_score": 5
}
]
}