evals
scenario-1
scenario-10
scenario-2
scenario-3
scenario-4
scenario-5
scenario-6
scenario-7
scenario-8
scenario-9
{
"context": "Evaluates how well the solution leverages JupyterLab's kernel-backed text editing APIs to run selections, manage per-document sessions, and surface results without reimplementing platform features. Checks emphasize correct use of JupyterLab services, editor models, and rendering pipelines tied to text documents.",
"type": "weighted_checklist",
"checklist": [
{
"name": "Session context",
"description": "Creates and manages one SessionContext per document using @jupyterlab/services kernel/session managers, with clean shutdown when a document closes.",
"max_score": 25
},
{
"name": "Editor access",
"description": "Pulls selection or cursor text via CodeEditor.IEditor APIs (e.g., getSelection, getOffsetAt/positionAt) from the document model instead of manual string parsing.",
"max_score": 20
},
{
"name": "Kernel execute",
"description": "Dispatches code through KernelConnection.requestExecute (or SessionContext.session?.requestExecute) and listens to IExecuteReplyMsg/IOPub streams to capture stdout/stderr for history.",
"max_score": 20
},
{
"name": "History storage",
"description": "Maintains per-document output history with OutputAreaModel/IOutputAreaModel (or RenderMimeRegistry-backed models) rather than ad-hoc arrays, keeping ordering across runs.",
"max_score": 15
},
{
"name": "Preview toggle",
"description": "Toggles a markdown/preview panel using JupyterLab renderers (e.g., RenderMimeRegistry or MarkdownDocument widgets) and refreshes it when content or run results change.",
"max_score": 10
},
{
"name": "Lifecycle cleanup",
"description": "Hooks into document/context disposal to dispose editor bindings, output models, and futures to avoid leaked kernel requests or preview widgets.",
"max_score": 10
}
]
}