evals
scenario-1
scenario-10
scenario-2
scenario-3
scenario-4
scenario-5
scenario-6
scenario-7
scenario-8
scenario-9
{
"context": "Evaluates how the solution leverages JupyterLab's extension manager APIs to reconcile allow/block directives. Checks that code reads extension state from the manager model, performs installs/enables/disables/removals through manager methods, and builds summaries from those results.",
"type": "weighted_checklist",
"checklist": [
{
"name": "Manager API only",
"description": "Obtains the `IExtensionManager` token from `@jupyterlab/extensionmanager` and routes all extension operations through it instead of shelling out to pip/npm or editing settings files directly.",
"max_score": 20
},
{
"name": "Read state via model",
"description": "Uses the manager's model (e.g., `manager.model` / `ListModel`) to enumerate installed extensions and their enabled flags before applying directives, rather than relying on hardcoded lists.",
"max_score": 20
},
{
"name": "Install allowlist",
"description": "For missing allow-list entries, calls `manager.install(name)` and then `manager.enable(name)` to pull from the default index and activate the extension when installation succeeds.",
"max_score": 20
},
{
"name": "Disable/remove blocklist",
"description": "For blocked extensions that are present, invokes `manager.disable(name)` and, when removal is requested, `manager.uninstall(name)` to ensure they no longer load.",
"max_score": 20
},
{
"name": "Plan reflects results",
"description": "Builds the returned plan/warnings directly from `IExtensionManager` outcomes (successful installs/enables/disables/uninstalls or thrown errors) instead of synthesizing statuses without manager feedback.",
"max_score": 20
}
]
}