docs
evals
scenario-1
scenario-10
scenario-2
scenario-3
scenario-4
scenario-5
scenario-6
scenario-7
scenario-8
scenario-9
{
"context": "This evaluation assesses how effectively the engineer uses the jsons library to serialize and deserialize Python collections (lists, dictionaries, sets) with proper type preservation.",
"type": "weighted_checklist",
"checklist": [
{
"name": "jsons.dump usage",
"description": "Uses jsons.dump() or jsons.dumps() to serialize the Inventory object and nested collections to dictionary format",
"max_score": 20
},
{
"name": "jsons.load usage",
"description": "Uses jsons.load() or jsons.loads() to deserialize dictionary data back to Inventory object with proper type reconstruction",
"max_score": 20
},
{
"name": "List[Product] typing",
"description": "Properly uses List[Product] type hints to ensure jsons correctly deserializes list elements as Product objects rather than plain dictionaries",
"max_score": 15
},
{
"name": "Dict[str, List[Product]] typing",
"description": "Properly uses Dict[str, List[Product]] type hints for the categories mapping to ensure nested list deserialization works correctly",
"max_score": 15
},
{
"name": "Set[str] handling",
"description": "Properly uses Set[str] type hints to ensure tags are deserialized as sets rather than lists, maintaining set semantics",
"max_score": 15
},
{
"name": "Dataclass compatibility",
"description": "Implements Product as a dataclass (or compatible structure) that works seamlessly with jsons serialization without custom serializers",
"max_score": 10
},
{
"name": "Type preservation",
"description": "Ensures all collection types (list, dict, set) and their element types are correctly preserved through the serialize-deserialize cycle using jsons type handling",
"max_score": 5
}
]
}