evals
scenario-1
scenario-10
scenario-2
scenario-3
scenario-4
scenario-5
scenario-6
scenario-7
scenario-8
scenario-9
{
"context": "This criteria evaluates the engineer's understanding and implementation of Apache Superset's multi-level cache hierarchy pattern. It assesses how well they implement timeout resolution following the query → chart → dataset → database → system hierarchy, and proper cache key generation for query result caching.",
"type": "weighted_checklist",
"checklist": [
{
"name": "Timeout hierarchy implementation",
"description": "The resolve_cache_timeout function correctly implements the hierarchical timeout resolution: checks query_timeout first, then chart_timeout, then dataset_timeout, then database_timeout, and finally returns system_default. Uses proper conditional logic to check for None values in the correct order.",
"max_score": 30
},
{
"name": "Cache key generation",
"description": "The generate_cache_key function uses a cryptographic hash function (MD5, SHA-256, or similar) from hashlib to create a deterministic cache key. Includes all required parameters (sql, database_name, schema, user_id) in the hash computation, with optional parameters handled correctly.",
"max_score": 25
},
{
"name": "CacheConfigHelper class structure",
"description": "The CacheConfigHelper class is properly defined with __init__ method that accepts and stores system_default_timeout. Implements get_effective_timeout and create_cache_key methods with correct signatures matching the specification.",
"max_score": 15
},
{
"name": "get_effective_timeout method",
"description": "The get_effective_timeout method in CacheConfigHelper delegates to or reimplements the hierarchical timeout resolution logic, correctly handling all five levels of the cache hierarchy and returning the appropriate timeout value.",
"max_score": 15
},
{
"name": "create_cache_key method",
"description": "The create_cache_key method in CacheConfigHelper delegates to or reimplements the cache key generation logic, properly hashing query parameters to produce consistent, deterministic cache keys.",
"max_score": 10
},
{
"name": "Type annotations usage",
"description": "Code uses proper Python type hints including Optional[int] for nullable timeout parameters, Optional[str] for optional string parameters, and appropriate return type annotations (int for timeout, str for cache keys).",
"max_score": 5
}
]
}