evals
scenario-1
scenario-10
scenario-2
scenario-3
scenario-4
scenario-5
scenario-6
scenario-7
scenario-8
scenario-9
{
"context": "Evaluates how the solution uses JupyterLab's Language Server Protocol service and the jupyterlab-lsp extension to activate servers, wire LSP-aware features, and drive diagnostics, completion, and definition queries.",
"type": "weighted_checklist",
"checklist": [
{
"name": "Server config",
"description": "Registers Python and Markdown entries in LanguageServerManager.language_servers with the provided argv and language ids instead of spawning ad-hoc processes.",
"max_score": 25
},
{
"name": "Service toggle",
"description": "Turns on the LSP service through package settings (e.g., @jupyterlab/lsp:language-server activate flag or LanguageServerManager.enabled) before requesting any features.",
"max_score": 20
},
{
"name": "Diagnostics wiring",
"description": "Uses jupyterlab-lsp diagnostics plumbing (ILSPFeatureManager diagnostics feature or DocumentConnectionManager diagnostics stream) to populate DiagnosticSummary rather than manual parsing.",
"max_score": 20
},
{
"name": "LSP completions",
"description": "Requests completions through an LSP connection (DocumentConnectionManager connection + textDocument/completion or ILSPFeatureManager completion feature) and surfaces LSP-sourced labels like 'print'.",
"max_score": 20
},
{
"name": "Definition lookup",
"description": "Uses the LSP textDocument/definition request via an LSP connection (e.g., from DocumentConnectionManager) to resolve definition URIs instead of text searching.",
"max_score": 15
}
]
}