Pragmatic Testing Framework for Python with BDD-style syntax and pluggable architecture
{
"context": "Evaluates whether the CLI helper actually delegates execution to Vedro while wiring its slicing and ordering controls so workers can run deterministic shards. Scoring prioritizes correct use of Vedro's CLI/config flags over custom scheduling logic.",
"type": "weighted_checklist",
"checklist": [
{
"name": "Vedro entrypoint",
"description": "Uses Vedro's runner (`vedro run` shell command or `vedro.run()` entrypoint) to execute scenarios instead of implementing a bespoke test loop.",
"max_score": 25
},
{
"name": "Slice forwarding",
"description": "Passes the user-provided slice string directly into Vedro via `--slice <index>/<total>` or `Config.Plugins.Slicer.slice`, ensuring only that shard executes.",
"max_score": 25
},
{
"name": "Ordering modes",
"description": "Maps CLI flags to Vedro's orderer options (`--order declaration|reversed|random` or `Config.Plugins.Orderer.orderer`) so the requested ordering mode reaches the framework.",
"max_score": 20
},
{
"name": "Random seed",
"description": "When random ordering is chosen, forwards the seed through Vedro (`--seed <value>` or `Config.Plugins.Seeder.seed`) to keep slice order reproducible across runs.",
"max_score": 20
},
{
"name": "Arg passthrough",
"description": "Extras after `--` are preserved when invoking Vedro (e.g., `vedro run ... -- <args>`), guaranteeing downstream options reach the test runner unchanged.",
"max_score": 10
}
]
}tessl i tessl/pypi-vedro@1.14.0docs
evals
scenario-1
scenario-2
scenario-3
scenario-4
scenario-5
scenario-6
scenario-7
scenario-8
scenario-9
scenario-10