evals
scenario-1
scenario-10
scenario-2
scenario-3
scenario-4
scenario-5
scenario-6
scenario-7
scenario-8
scenario-9
{
"context": "This evaluation assesses how well the engineer uses the openapi-types package to work with OpenAPI security requirements, specifically focusing on the SecurityRequirementObject type and proper handling of security requirement combinations with AND/OR logic.",
"type": "weighted_checklist",
"checklist": [
{
"name": "OpenAPIV3.Document type usage",
"description": "Uses the OpenAPIV3.Document type (or OpenAPIV3.Document<T>) from openapi-types to type the OpenAPI document parameter",
"max_score": 15
},
{
"name": "SecurityRequirementObject type usage",
"description": "Uses OpenAPIV3.SecurityRequirementObject type (or imports and references it) when working with security requirement objects from the document",
"max_score": 20
},
{
"name": "Document-level security access",
"description": "Correctly accesses document.security property to retrieve document-level security requirements, which is of type SecurityRequirementObject[] | undefined",
"max_score": 15
},
{
"name": "Operation-level security access",
"description": "Correctly accesses operation.security property from the PathItemObject and OperationObject types to retrieve operation-specific security requirements",
"max_score": 15
},
{
"name": "Paths and operations navigation",
"description": "Uses OpenAPIV3.PathsObject and OpenAPIV3.PathItemObject types to navigate to the specific operation, demonstrating understanding of the document structure types",
"max_score": 10
},
{
"name": "SecurityRequirementObject structure",
"description": "Correctly interprets SecurityRequirementObject as a map/record of security scheme names to string arrays (scopes), accessing both keys (scheme names) and values (scope arrays)",
"max_score": 15
},
{
"name": "Security array semantics",
"description": "Correctly implements OR logic by treating the security array (SecurityRequirementObject[]) as alternative requirements where any one array item satisfies the security requirement",
"max_score": 10
}
]
}