Pragmatic Testing Framework for Python with BDD-style syntax and pluggable architecture
{
"context": "Evaluates whether the harness uses Vedro's built-in console and JSON reporters together and wires output capture flags/limits correctly. Scoring focuses on Director reporter configuration, use of RichReporter/JsonReporter, and proper use of capture toggles from the Vedro runner/CLI.",
"type": "weighted_checklist",
"checklist": [
{
"name": "Dual reporters",
"description": "Director.default_reporters (or -r/--reporter flags) includes both \"rich\" and \"json\" so runs emit console output via RichReporter and a JSON artifact via JsonReporter without custom logging.",
"max_score": 30
},
{
"name": "Rich reporter",
"description": "Enables the built-in vedro.plugins.director.rich.RichReporter (either in Config.Plugins.RichReporter or via --reporter rich) for the console stream instead of a custom reporter.",
"max_score": 15
},
{
"name": "JSON reporter path",
"description": "Enables vedro.plugins.director.json.JsonReporter and routes its output to the requested json_report_path (e.g., through JsonReporter config or artifacts_dir) rather than writing JSON manually.",
"max_score": 25
},
{
"name": "Capture toggle",
"description": "Uses Vedro's output capture controls (OutputCapturer or --capture-output/-C) to switch between captured and passthrough stdout/stderr based on the capture_enabled argument, instead of ad-hoc prints.",
"max_score": 15
},
{
"name": "Capture limit",
"description": "Applies the Vedro capture limit option (--capture-limit or OutputCapturer capture_limit) so captured streams truncate at the configured byte/character limit with the package-provided truncation behavior.",
"max_score": 15
}
]
}tessl i tessl/pypi-vedro@1.14.0docs
evals
scenario-1
scenario-2
scenario-3
scenario-4
scenario-5
scenario-6
scenario-7
scenario-8
scenario-9
scenario-10