docs
evals
scenario-1
scenario-10
scenario-2
scenario-3
scenario-4
scenario-5
scenario-6
scenario-7
scenario-8
scenario-9
{
"context": "This criteria evaluates how well the engineer uses pixelmatch's transparent diff masking capability (diffMask option) to generate overlay images where only differences are visible with full opacity while matching pixels remain fully transparent.",
"type": "weighted_checklist",
"checklist": [
{
"name": "diffMask option usage",
"description": "Uses the diffMask option set to true when calling pixelmatch to enable transparent background mode where matching pixels are not drawn (fully transparent)",
"max_score": 30
},
{
"name": "Output buffer allocation",
"description": "Properly allocates the output buffer (Uint8Array, Uint8ClampedArray, or Buffer) with correct size (width * height * 4 bytes) for RGBA format to hold the diff mask",
"max_score": 15
},
{
"name": "diffColor configuration",
"description": "Sets the diffColor option to [255, 0, 0] (red RGB values) to highlight differing pixels in the specified red color",
"max_score": 15
},
{
"name": "Image buffer preparation",
"description": "Correctly reads PNG files and extracts image data buffers in RGBA format suitable for pixelmatch (e.g., using pngjs to get img1.data and img2.data)",
"max_score": 15
},
{
"name": "Dimension parameters",
"description": "Passes correct width and height parameters to pixelmatch that match the actual image dimensions",
"max_score": 10
},
{
"name": "PNG output generation",
"description": "Writes the diff mask output buffer back to a PNG file with correct dimensions and RGBA format (e.g., using pngjs)",
"max_score": 10
},
{
"name": "Return value handling",
"description": "Returns the mismatch count from pixelmatch as specified in the API (number of differing pixels)",
"max_score": 5
}
]
}