docs
evals
scenario-1
scenario-10
scenario-2
scenario-3
scenario-4
scenario-5
scenario-6
scenario-7
scenario-8
scenario-9
{
"context": "This criteria evaluates how well the engineer uses the jsons package to serialize and deserialize a defaultdict with proper type hints, preserving the default_factory behavior for automatic list initialization.",
"type": "weighted_checklist",
"checklist": [
{
"name": "DefaultDict Usage",
"description": "Uses collections.defaultdict with list as the default_factory to store student grades",
"max_score": 20
},
{
"name": "Type Hint",
"description": "Uses typing.DefaultDict[str, List[float]] or DefaultDict[str, list[float]] type hint for the grades data structure",
"max_score": 20
},
{
"name": "Serialization",
"description": "Uses jsons.dump() or jsons.dumps() to serialize the defaultdict in the to_json method",
"max_score": 15
},
{
"name": "Deserialization",
"description": "Uses jsons.load() or jsons.loads() with the DefaultDict type hint to deserialize in the from_json method",
"max_score": 20
},
{
"name": "Type Preservation",
"description": "Passes the correct type (DefaultDict[str, List[float]] or similar) as the cls parameter to jsons.load/loads to preserve the default_factory",
"max_score": 15
},
{
"name": "Functional Correctness",
"description": "All test cases pass: automatic initialization works both before and after deserialization",
"max_score": 10
}
]
}