evals
scenario-1
scenario-2
scenario-3
scenario-4
scenario-5
scenario-6
scenario-7
scenario-8
scenario-9
scenario-10
{
"context": "Evaluates how the solution uses the notion-client package to retrieve user records by ID and translate them into the required profile shape. Focuses on correct invocation of the client's user-fetch API, option passthrough, mapping of returned fields, and explicit handling of missing users while preserving input order.",
"type": "weighted_checklist",
"checklist": [
{
"name": "User fetch call",
"description": "Creates a notion-client NotionAPI (or equivalent) instance and calls its getUsers/user-record endpoint with the provided IDs rather than reimplementing HTTP requests.",
"max_score": 40
},
{
"name": "Dedup & order",
"description": "Deduplicates IDs before calling getUsers to avoid redundant requests while returning results in the same order as the original input list.",
"max_score": 15
},
{
"name": "Profile mapping",
"description": "Builds each found profile from the getUsers response using fields like given_name/family_name (or name fallback) for fullName, email for contact, and profile_photo for avatarUrl, setting status to \"found\".",
"max_score": 20
},
{
"name": "Missing users",
"description": "Detects IDs absent from the getUsers results and returns entries for them with status \"not_found\" and null email/avatarUrl while keeping other entries intact.",
"max_score": 15
},
{
"name": "Option passthrough",
"description": "Forwards provided requestOptions into the getUsers kyOptions argument (or equivalent) so custom headers/timeouts reach the underlying HTTP call.",
"max_score": 10
}
]
}