evals
scenario-1
scenario-2
scenario-3
scenario-4
scenario-5
scenario-6
scenario-7
{
"context": "Evaluates how well the solution uses the portfinder package to retrieve multiple sequential free TCP ports. Emphasizes forwarding range and host options and surfacing portfinder-provided errors without masking them.",
"type": "weighted_checklist",
"checklist": [
{
"name": "Portfinder API",
"description": "Uses portfinder.getPorts or portfinder.getPortsPromise to fetch multiple ports in one call rather than manual socket probing or repeated single-port lookups.",
"max_score": 35
},
{
"name": "Count handling",
"description": "Passes the requested count directly into getPorts/getPortsPromise and returns the package-provided ordered results without truncation or extra ports.",
"max_score": 20
},
{
"name": "Range options",
"description": "Forwards start/stop boundaries via portfinder options (e.g., port/startPort and stopPort) or setBasePort/setHighestPort so the search respects the requested window.",
"max_score": 15
},
{
"name": "Host forwarding",
"description": "Propagates the host option into getPorts/getPortsPromise so selection is performed for the requested interface rather than ignoring the host.",
"max_score": 15
},
{
"name": "Error propagation",
"description": "Surfaces portfinder errors (such as exhaustion of the range or invalid hosts) directly to callers without suppressing or replacing them with unrelated messages.",
"max_score": 15
}
]
}