docs
evals
scenario-1
scenario-10
scenario-2
scenario-3
scenario-4
scenario-5
scenario-6
scenario-7
scenario-8
scenario-9
{
"context": "Evaluates how well the solution uses Pulumi's Automation API to drive stack lifecycle: initializing stacks with inline programs, applying configuration (including secrets), previewing before updates, deploying while capturing outputs/change summaries, and cleaning up stacks when requested. Scoring focuses purely on correct use of the automation primitives, not general code quality.",
"type": "weighted_checklist",
"checklist": [
{
"name": "Stack init",
"description": "Initializes the stack through @pulumi/pulumi/automation using LocalWorkspace.createOrSelectStack (or createStack/selectStack equivalents) with project and stack names plus the inline program, relying on project/stack settings rather than manual file edits.",
"max_score": 20
},
{
"name": "Config apply",
"description": "Uses stack.setAllConfig or setConfig to apply the provided config map before preview/up, marking ConfigValue.secret=true for secret entries and avoiding hard-coded defaults in the program.",
"max_score": 20
},
{
"name": "Preview first",
"description": "Calls stack.preview() prior to any stack.up(), surfaces PreviewResult.changeSummary and summary.result into the returned PreviewSummary, and stops the deployment path when preview indicates errors.",
"max_score": 20
},
{
"name": "Inline program",
"description": "Passes the PulumiFn via automation APIs (e.g., LocalWorkspace.createOrSelectStack({ program })) and inside it reads stack config with new pulumi.Config(), exporting outputs keyed to the config names to feed the DeploymentResult outputs.",
"max_score": 15
},
{
"name": "Update outputs",
"description": "Runs stack.up() for the update and returns UpResult.outputs preserving both value and secret flags, also exposing UpResult.summary.resourceChanges (or equivalent) as updateChanges in the result.",
"max_score": 15
},
{
"name": "Destroy flag",
"description": "When destroyOnComplete is true, invokes stack.destroy() (and optionally workspace.removeStack) after a successful update and sets performedDestroy accordingly; leaves the stack intact when the flag is false.",
"max_score": 10
}
]
}