evals
scenario-1
scenario-10
scenario-2
scenario-3
scenario-4
scenario-5
scenario-6
scenario-7
scenario-8
scenario-9
{
"context": "Evaluates how the solution leverages JupyterLab notebook and services APIs to build, execute, and persist notebooks for the requested workflow. Checks focus on constructing notebook models, starting kernel-backed sessions, sequencing execution, capturing results, and saving the executed document.",
"type": "weighted_checklist",
"checklist": [
{
"name": "Notebook model",
"description": "Creates the notebook using @jupyterlab/notebook primitives (e.g., NotebookModel or NotebookPanel) bound to the requested kernel name instead of manually crafting nbformat JSON.",
"max_score": 20
},
{
"name": "Cell insertion",
"description": "Populates markdown and code cells in order through notebook APIs such as NotebookModel.sharedModel.insertCell or NotebookActions.insertBelow/insertAbove rather than editing raw arrays.",
"max_score": 20
},
{
"name": "Session setup",
"description": "Starts and manages the kernel session via @jupyterlab/services (ServiceManager.sessions.startNew or SessionContext.initialize/ready) using the provided kernelName and reuses the session for all cell executions.",
"max_score": 20
},
{
"name": "Sequential execution",
"description": "Runs code cells sequentially through the kernel connection (kernel.requestExecute or sessionContext.session?.kernel?.requestExecute), awaits future.done, collects IOPub output messages, and stops or continues based on allowErrors.",
"max_score": 25
},
{
"name": "Saving outputs",
"description": "Persists the executed notebook to outputPath with captured outputs using notebook-aware saving (ContentsManager.save or context.save()) and surfaces the saved path in the returned result.",
"max_score": 15
}
]
}