docs
evals
scenario-1
scenario-10
scenario-2
scenario-3
scenario-4
scenario-5
scenario-6
scenario-7
scenario-8
scenario-9
{
"context": "This evaluation assesses how well the engineer implements a custom ConnectionTransport for Puppeteer to monitor and log Chrome DevTools Protocol traffic. The focus is on correct usage of Puppeteer's transport interface and connection APIs.",
"type": "weighted_checklist",
"checklist": [
{
"name": "ConnectionTransport implementation",
"description": "Implements a class or object that satisfies the ConnectionTransport interface with required methods: send(), onmessage, onclose, and close()",
"max_score": 25
},
{
"name": "WebSocket integration",
"description": "Creates and properly manages a WebSocket connection to the browserWSEndpoint, handling connection lifecycle events",
"max_score": 20
},
{
"name": "Message forwarding",
"description": "Correctly forwards outgoing messages via send() method and incoming messages via onmessage callback, maintaining message structure and order",
"max_score": 20
},
{
"name": "Browser connection",
"description": "Uses puppeteer.connect() with the custom transport via the transport or browserWSEndpoint option to establish browser connection",
"max_score": 15
},
{
"name": "CDP message logging",
"description": "Logs both outgoing CDP commands and incoming responses/events with sufficient detail to identify message types and content",
"max_score": 10
},
{
"name": "Connection closure",
"description": "Properly implements close() method and onclose callback to handle connection termination and cleanup",
"max_score": 10
}
]
}