evals
scenario-1
scenario-10
scenario-2
scenario-3
scenario-4
scenario-5
scenario-6
scenario-7
scenario-8
scenario-9
{
"context": "Evaluates how well the solution configures CKEditor 5 marker-based mention feeds using @ckeditor/ckeditor5-mention. Focuses on correct mention.feeds setup, commitKeys, dropdown limits, itemRenderer use, and MentionCommand text handling for the @ and # feeds, including async feed hygiene.",
"type": "weighted_checklist",
"checklist": [
{
"name": "Plugin setup",
"description": "Registers the @ckeditor/ckeditor5-mention plugin and builds the editor configuration with a mention object that includes mention.feeds.",
"max_score": 20
},
{
"name": "@ feed config",
"description": "Configures a feed entry with marker '@' using the provided users as a static feed array, enforces marker-prefixed ids/text, sets dropdownLimit to 5, and relies on the package's single-character marker validation.",
"max_score": 15
},
{
"name": "# feed async",
"description": "Defines the '#' feed with an async feed callback returning a Promise of mention items, sets minimumCharacters to 2, dropdownLimit to 8, and ensures returned ids start with '#' to satisfy MentionCommand validation.",
"max_score": 20
},
{
"name": "Commit keys",
"description": "Sets mention.commitKeys to include Enter and Tab and applies a global mention.dropdownLimit of 6 that per-feed limits override.",
"max_score": 10
},
{
"name": "Renderers",
"description": "Uses itemRenderer for both feeds to show display data (name+role for users, number+title for issues) via the mention plugin's DOM/string renderer contract.",
"max_score": 15
},
{
"name": "Inserted text",
"description": "Uses the mention item text property so MentionCommand inserts marker-prefixed handles/issue keys, preserving trailing space defaults and the built-in suppression before closing brackets.",
"max_score": 10
},
{
"name": "Async hygiene",
"description": "Relies on mention feed debouncing (~100ms) and stale-response dropping by returning a fresh Promise per query so only the latest '#' results populate the dropdown.",
"max_score": 10
}
]
}