docs
evals
scenario-1
scenario-10
scenario-2
scenario-3
scenario-4
scenario-5
scenario-6
scenario-7
scenario-8
scenario-9
{
"context": "Evaluates how well the solution uses @pulumi/pulumi's Input/Output graph and secret utilities to build connection details without breaking dependency or secrecy semantics. Scoring prioritizes proper composition of outputs, preservation of secrets, and correct handling of unknown values during previews.",
"type": "weighted_checklist",
"checklist": [
{
"name": "Graph outputs",
"description": "Builds the URI and related values with pulumi.output/pulumi.all and Output.apply or pulumi.interpolate instead of awaiting raw inputs, keeping dependency tracking intact.",
"max_score": 25
},
{
"name": "Secret propagation",
"description": "Marks sensitive credentials with pulumi.secret and ensures derived URI and credential bundles stay secret (verifiable via Output.isSecret).",
"max_score": 30
},
{
"name": "Redacted summary",
"description": "Produces a non-secret redacted summary using pulumi.unsecret or equivalent, omitting the password and confirming the summary output is not secret.",
"max_score": 15
},
{
"name": "Unknown handling",
"description": "Uses Output.isKnown checks or guarded apply chains so that if any input is unknown in preview, the composed URI remains unknown rather than substituting placeholders while retaining secret flags when set.",
"max_score": 20
},
{
"name": "Safe serialization",
"description": "Avoids direct toString/JSON on unresolved outputs; combines values through Output.apply or pulumi.interpolate so secrets and dependencies are preserved.",
"max_score": 10
}
]
}