evals
scenario-1
scenario-10
scenario-2
scenario-3
scenario-4
scenario-5
scenario-6
scenario-7
scenario-8
scenario-9
{
"context": "This criteria evaluates how effectively the engineer uses the openapi-types package to work with OpenAPI 3.0 security types. The focus is on proper usage of OpenAPIV3 security-related types including SecuritySchemeObject, SecurityRequirementObject, and the various specific security scheme types (HttpSecurityScheme, ApiKeySecurityScheme, OAuth2SecurityScheme, OpenIdSecurityScheme).",
"type": "weighted_checklist",
"checklist": [
{
"name": "OpenAPIV3.Document type usage",
"description": "Uses OpenAPIV3.Document type for function parameters that accept OpenAPI 3.0 documents, properly importing it from 'openapi-types'",
"max_score": 15
},
{
"name": "SecuritySchemeObject type usage",
"description": "Uses OpenAPIV3.SecuritySchemeObject or its specific variants (HttpSecurityScheme, ApiKeySecurityScheme, OAuth2SecurityScheme, OpenIdSecurityScheme) when working with security scheme definitions from document.components.securitySchemes",
"max_score": 25
},
{
"name": "SecurityRequirementObject type usage",
"description": "Uses OpenAPIV3.SecurityRequirementObject when working with security requirements from document.security or operation.security arrays",
"max_score": 20
},
{
"name": "OperationObject type usage",
"description": "Uses OpenAPIV3.OperationObject type for function parameters that accept operations, properly typed to access security property",
"max_score": 15
},
{
"name": "Type-safe security scheme access",
"description": "Accesses security scheme properties in a type-safe manner using the proper types (e.g., checking 'type' field, accessing type-specific properties like 'flows' for OAuth2, 'scheme' for HTTP, 'in' and 'name' for API key)",
"max_score": 15
},
{
"name": "ComponentsObject usage",
"description": "Properly accesses document.components.securitySchemes using type-safe navigation through OpenAPIV3.ComponentsObject structure",
"max_score": 10
}
]
}