evals
scenario-1
scenario-10
scenario-2
scenario-3
scenario-4
scenario-5
scenario-6
scenario-7
scenario-8
scenario-9
{
"context": "This criteria evaluates how well the engineer uses Apache Superset's dataset certification and documentation APIs to implement a certification manager. The focus is on proper usage of certification-related models, methods, and patterns from the superset.connectors.sqla.models and superset.datasets.api modules.",
"type": "weighted_checklist",
"checklist": [
{
"name": "Dataset model usage",
"description": "Uses superset.connectors.sqla.models.SqlaTable or BaseDatasource for dataset representation and management",
"max_score": 25
},
{
"name": "Certification property implementation",
"description": "Properly implements certification status tracking using is_certified property and certification metadata fields (certified_by, certification_details)",
"max_score": 25
},
{
"name": "Dataset API integration",
"description": "Uses superset.datasets.api.DatasetRestApi or equivalent API methods for dataset CRUD operations and certification management",
"max_score": 20
},
{
"name": "Timestamp handling",
"description": "Correctly manages certification timestamps using SQLAlchemy datetime fields or similar date handling mechanisms",
"max_score": 15
},
{
"name": "Query and filtering",
"description": "Implements proper querying/filtering of datasets by certification status using SQLAlchemy query methods or REST API filters",
"max_score": 15
}
]
}