evals
scenario-1
scenario-10
scenario-2
scenario-3
scenario-4
scenario-5
scenario-6
scenario-7
scenario-8
scenario-9
{
"context": "This criteria evaluates how effectively the engineer uses the IJsonSchema type from the openapi-types package to implement JSON Schema validation and analysis utilities. The focus is on correct usage of the type definitions to ensure type safety when working with schema objects.",
"type": "weighted_checklist",
"checklist": [
{
"name": "IJsonSchema Import",
"description": "Correctly imports the IJsonSchema type from the openapi-types package",
"max_score": 10
},
{
"name": "Type Annotations",
"description": "Uses IJsonSchema as the type annotation for schema parameters in all four functions (isValidSchema, getRequiredFields, analyzeComplexity, getPropertyType)",
"max_score": 25
},
{
"name": "Schema Properties Access",
"description": "Correctly accesses IJsonSchema properties such as type, properties, items, required, allOf, anyOf, oneOf, $ref when implementing validation logic",
"max_score": 20
},
{
"name": "Validation Constraints",
"description": "Properly checks IJsonSchema validation constraint properties (minLength, maxLength, pattern, minimum, maximum, enum, minItems, maxItems, uniqueItems, etc.) in the analyzeComplexity function",
"max_score": 20
},
{
"name": "Properties Navigation",
"description": "Correctly navigates the properties object of IJsonSchema (which is a Record<string, IJsonSchema>) when implementing getPropertyType and checking for nested objects",
"max_score": 15
},
{
"name": "Type Field Handling",
"description": "Properly handles the type field which can be either a string or string[] according to IJsonSchema definition, particularly in the getPropertyType function",
"max_score": 10
}
]
}