evals
scenario-1
scenario-2
scenario-3
scenario-4
scenario-5
scenario-6
scenario-7
scenario-8
scenario-9
scenario-10
{
"context": "Evaluates how well the solution uses notion-client to fetch a page record map and obtain signed URLs for workspace-hosted files, media, and covers as defined in the spec. Emphasizes correct use of NotionAPI page loading and signing helpers instead of ad-hoc HTTP calls. Focuses solely on package API usage for discovering signable sources, honoring options, and mapping results.",
"type": "weighted_checklist",
"checklist": [
{
"name": "Page fetch",
"description": "Uses NotionAPI.getPage (or equivalent) to load the page record map before signing and forwards any provided request overrides.",
"max_score": 15
},
{
"name": "Signable discovery",
"description": "Detects signable Notion-hosted sources (secure.notion-static.com, prod-files-secure, attachment URLs) across supported file/image/video/audio/pdf blocks and prepares permission records keyed by block id before signing.",
"max_score": 20
},
{
"name": "Signing call",
"description": "Retrieves signed URLs via notion-client helpers like addSignedUrls or getSignedFileUrls rather than manual signing, and correctly maps returned URLs back to their originating block IDs.",
"max_score": 25
},
{
"name": "External skip",
"description": "Excludes external HTTP(S) file sources from signing requests and leaves them out of signed URL results.",
"max_score": 10
},
{
"name": "Subset handling",
"description": "When blockIds is provided, limits signing requests to that subset and avoids processing other signable blocks.",
"max_score": 15
},
{
"name": "Cover toggle",
"description": "Conditionally includes page cover signing when includeCovers is true by forming a cover-specific permission record, and omits cover signing when false.",
"max_score": 15
}
]
}