docs
evals
scenario-1
scenario-10
scenario-2
scenario-3
scenario-4
scenario-5
scenario-6
scenario-7
scenario-8
scenario-9
{
"context": "This evaluation assesses how effectively the engineer uses the jsons library to handle Union type deserialization. The focus is on using jsons.load() with Union type hints to deserialize polymorphic data structures, including Optional types and lists with mixed types.",
"type": "weighted_checklist",
"checklist": [
{
"name": "jsons.load() usage",
"description": "Uses jsons.load() function to deserialize dictionary data into Python objects with proper type hints for Union types",
"max_score": 30
},
{
"name": "Union type hint",
"description": "Correctly uses Union[EmailNotification, SMSNotification, PushNotification] type hint with jsons.load() to enable automatic type selection during deserialization",
"max_score": 25
},
{
"name": "Optional type handling",
"description": "Properly handles Optional[int] for the badge_count field using jsons with appropriate type hints, allowing None values",
"max_score": 15
},
{
"name": "List deserialization",
"description": "Uses jsons to deserialize a list of dictionaries where each element could be any of the Union types, leveraging jsons' support for generic List types with Union elements",
"max_score": 20
},
{
"name": "Dataclass integration",
"description": "Properly defines dataclasses with type annotations that work seamlessly with jsons' automatic type inspection and Union type resolution",
"max_score": 10
}
]
}