evals
scenario-1
scenario-10
scenario-2
scenario-3
scenario-4
scenario-5
scenario-6
scenario-7
scenario-8
scenario-9
{
"context": "This evaluation assesses how well the engineer uses Apache Superset's authentication and security APIs to implement a multi-provider authentication configuration system. The focus is exclusively on proper usage of authentication configuration methods and understanding Superset's security patterns.",
"type": "weighted_checklist",
"checklist": [
{
"name": "Database Auth Config",
"description": "Uses configure_database_auth method to return a dictionary with 'password_scheme' and 'min_password_length' keys. Returns correct default values ('bcrypt' and 10) when no arguments provided, and accepts custom values when specified.",
"max_score": 20
},
{
"name": "LDAP Auth Config",
"description": "Uses configure_ldap_auth method to return a dictionary containing all required keys: 'server', 'bind_user', 'bind_password', 'base_dn', and 'search_filter'. Properly handles LDAP server URLs and DN strings.",
"max_score": 25
},
{
"name": "OAuth Provider Config",
"description": "Uses configure_oauth_provider method to return a dictionary with all OAuth2 configuration keys: 'provider_name', 'client_id', 'client_secret', 'authorize_url', and 'token_url'. Stores provider-specific configurations correctly.",
"max_score": 25
},
{
"name": "File-Based Config",
"description": "Implements load_from_file method that reads JSON configuration files and populates authentication settings. Correctly parses 'auth_type' and associated configuration parameters from the JSON structure.",
"max_score": 20
},
{
"name": "Auth Type Retrieval",
"description": "Implements get_active_auth_type method that returns the current authentication type as a string ('database', 'ldap', or 'oauth'). Returns the correct type based on the most recently configured or loaded authentication method.",
"max_score": 10
}
]
}