Pytest plugin to create CodSpeed benchmarks
—
Precise measurement control with setup/teardown functions, multiple rounds, and custom iterations for complex benchmarking scenarios. The pedantic API provides fine-grained control over benchmark execution.
Advanced benchmarking interface that provides complete control over setup, execution, and teardown phases with configurable rounds and iterations.
def benchmark.pedantic(
target: Callable[..., T],
args: tuple = (),
kwargs: dict = {},
setup: Callable | None = None,
teardown: Callable | None = None,
rounds: int = 1,
warmup_rounds: int = 0,
iterations: int = 1
) -> T:
"""
Advanced benchmarking with precise control over execution phases.
Parameters:
- target: Function to benchmark
- args: Arguments tuple to pass to target function
- kwargs: Keyword arguments dict to pass to target function
- setup: Function called before each round to prepare state
- teardown: Function called after each round to cleanup state
- rounds: Number of measurement rounds to perform
- warmup_rounds: Number of warmup rounds before measurement
- iterations: Number of target calls per round
Returns:
The return value of the final target function call
Raises:
RuntimeError: If benchmark fixture already used in test
ValueError: If invalid parameter combinations provided
"""def test_with_setup_teardown(benchmark):
data = []
def setup():
data.clear()
data.extend(range(1000))
def teardown():
data.clear()
def sort_data():
return sorted(data)
result = benchmark.pedantic(
sort_data,
setup=setup,
teardown=teardown,
rounds=10,
warmup_rounds=2
)
assert len(result) == 1000When setup returns a value, it replaces the args/kwargs for the target function:
def test_setup_returns_args(benchmark):
def setup():
# Setup returns arguments for the target function
import random
return [random.randint(1, 100) for _ in range(1000)]
def find_max(numbers):
return max(numbers)
# setup() return value becomes argument to find_max
result = benchmark.pedantic(
find_max,
setup=setup,
rounds=5
)
assert isinstance(result, int)def test_multiple_iterations(benchmark):
counter = {'value': 0}
def increment():
counter['value'] += 1
return counter['value']
def reset():
counter['value'] = 0
# Each round calls increment() 5 times
result = benchmark.pedantic(
increment,
setup=reset,
rounds=3,
iterations=5
)
assert result == 5 # Last iteration resultComplex parameter combinations and validation rules for pedantic benchmarking.
# Parameter validation rules:
# - rounds must be >= 1
# - warmup_rounds must be >= 0
# - iterations must be >= 1
# - setup cannot be used with iterations > 1
# - setup cannot return value when args/kwargs provided# Valid configurations
benchmark.pedantic(target, rounds=5) # Basic multiple rounds
benchmark.pedantic(target, warmup_rounds=3, rounds=5) # With warmup
benchmark.pedantic(target, iterations=10) # Multiple iterations per round
benchmark.pedantic(target, setup=setup_func, rounds=5) # Setup with multiple rounds
# Invalid configurations - will raise ValueError
benchmark.pedantic(target, rounds=0) # rounds must be >= 1
benchmark.pedantic(target, warmup_rounds=-1) # warmup_rounds must be >= 0
benchmark.pedantic(target, iterations=0) # iterations must be >= 1
benchmark.pedantic(target, setup=setup_func, iterations=5) # setup conflicts with iterations > 1
benchmark.pedantic(target, args=(1,), setup=lambda: (2,)) # setup return conflicts with argsThe pedantic API behaves differently depending on the active measurement mode.
# In walltime mode, all parameters are respected
result = benchmark.pedantic(
target_function,
rounds=10, # 10 measurement rounds
iterations=5, # 5 calls per round
warmup_rounds=3 # 3 warmup rounds before measurement
)
# Total function calls: 3 * 5 (warmup) + 10 * 5 (measurement) = 65 calls# In instrumentation mode, rounds/iterations are ignored with warning
result = benchmark.pedantic(
target_function,
rounds=10, # Ignored - warning issued
iterations=5, # Ignored - warning issued
warmup_rounds=3 # May be respected for perf trampoline warmup
)
# Actual behavior: Single measurement call after optional warmupCommon error conditions and their handling in pedantic benchmarking.
# Multiple benchmark calls in one test - RuntimeError
def test_invalid_multiple_calls(benchmark):
benchmark(some_function) # First call OK
benchmark.pedantic(other_function) # RuntimeError: already used
# Invalid parameter combinations - ValueError
def test_invalid_setup_with_iterations(benchmark):
benchmark.pedantic(
target,
setup=setup_func,
iterations=5 # ValueError: setup cannot be used with iterations > 1
)
# Setup return value conflicts with provided args - ValueError
def test_setup_return_with_args(benchmark):
benchmark.pedantic(
target,
args=(1, 2), # Providing args
setup=lambda: (3,) # ValueError: setup cannot return value when args provided
)Pedantic options interact with global configuration and marker options.
# Marker options still apply to pedantic calls
@pytest.mark.benchmark(group="advanced", max_time=30.0)
def test_pedantic_with_marker_options(benchmark):
# Pedantic settings + marker max_time limit
result = benchmark.pedantic(
expensive_function,
rounds=100, # May be limited by max_time=30.0
iterations=10
)Install with Tessl CLI
npx tessl i tessl/pypi-pytest-codspeed