docs
evals
scenario-1
scenario-10
scenario-2
scenario-3
scenario-4
scenario-5
scenario-6
scenario-7
scenario-8
scenario-9
{
"context": "Evaluates how well the solution uses @pulumi/pulumi's inline dynamic resource support to manage the token lifecycle described in the spec. Focuses on correct dynamic.ResourceProvider wiring, lifecycle methods (check/diff/create/update/read/delete), replacements, and output/secret handling rather than general code quality.",
"type": "weighted_checklist",
"checklist": [
{
"name": "Inline dynamic setup",
"description": "Uses a dynamic.Resource (or subclass) backed by an inline dynamic.ResourceProvider from @pulumi/pulumi, wiring the provider through super(..., { provider }) instead of relying on an external plugin.",
"max_score": 15
},
{
"name": "Check & diff rules",
"description": "Implements provider.check and provider.diff to validate inputs, return detailed failures when arguments are missing/invalid, and signal replacements via DiffResult.replaces when identifier changes while treating ttlSeconds/metadata as in-place updates.",
"max_score": 15
},
{
"name": "Create outputs",
"description": "Provider.create generates a 32-byte base64 token, writes the JSON file, initializes version=1 and lastRotated to ISO now, returns the file path as id, and sets outputs that match the on-disk state.",
"max_score": 15
},
{
"name": "Update lifecycle",
"description": "Provider.update persists ttlSeconds/metadata changes without replacing the resource, keeps id/token stable, increments version, refreshes lastRotated, and returns updated outputs using olds/news correctly.",
"max_score": 15
},
{
"name": "Read/import support",
"description": "Provider.read handles opts.id to adopt an existing file, loading and returning outputs without mutation, and participates in import flows by preserving token/version/lastRotated for future updates.",
"max_score": 10
},
{
"name": "Delete cleanup",
"description": "Provider.delete removes the old file during destroy or replacement (using deleteBeforeReplace or explicit delete) so identifier changes do not leave orphaned state.",
"max_score": 10
},
{
"name": "Outputs & secrets",
"description": "Resource registers outputs via registerOutputs, exposes properties as pulumi.Output values, and marks sensitive data (token) as secret using pulumi.secret or additionalSecretOutputs to preserve encryption.",
"max_score": 20
}
]
}