evals
scenario-1
scenario-10
scenario-2
scenario-3
scenario-4
scenario-5
scenario-6
scenario-7
scenario-8
scenario-9
{
"context": "This criteria evaluates how well the engineer uses Apache Superset's SQL Lab API components to implement query execution, history management, and editor state persistence. The focus is on correct usage of Superset's models and execution logic.",
"type": "weighted_checklist",
"checklist": [
{
"name": "Query Model Usage",
"description": "Uses superset.models.sql_lab.Query model to track query execution with appropriate fields (sql, status, start_time, end_time, database, schema, user_id)",
"max_score": 20
},
{
"name": "Sync Execution Logic",
"description": "Implements synchronous query execution using superset.sql_lab execution logic with timeout handling and row limiting",
"max_score": 15
},
{
"name": "Async Execution Logic",
"description": "Implements asynchronous query execution using superset.sql_lab with background task submission and query ID tracking",
"max_score": 15
},
{
"name": "Query Status Tracking",
"description": "Properly tracks and updates query status (SUCCESS, FAILED, STOPPED, RUNNING, PENDING) using the Query model's status field",
"max_score": 10
},
{
"name": "Progress Tracking",
"description": "Implements query progress tracking for async queries using Query model's progress field (0-100%)",
"max_score": 10
},
{
"name": "Query Cancellation",
"description": "Implements query cancellation using superset.sql_lab query stopping functionality with proper status updates",
"max_score": 10
},
{
"name": "TabState Model Usage",
"description": "Uses superset.models.sql_lab.TabState model to persist editor state with tab_id, user_id, sql, database_id, and schema fields",
"max_score": 10
},
{
"name": "Query History Retrieval",
"description": "Implements query history retrieval by querying saved Query model instances with filtering by user_id and proper ordering",
"max_score": 5
},
{
"name": "TabState Persistence",
"description": "Implements tab state save and retrieval operations using TabState model CRUD operations",
"max_score": 5
}
]
}