tessl/pypi-pyperf
Python module to run and analyze benchmarks with high precision and statistical rigor
—
Pending
data-management.mddocs/
Data Management and Analysis
Comprehensive data structures for storing, analyzing, and managing benchmark results. Provides statistical analysis, metadata handling, and serialization capabilities for persistent storage and result sharing.
Capabilities
Run Class
Represents a single benchmark run containing timing values and execution metadata.
class Run:
def __init__(self, values, warmups=None, metadata=None, collect_metadata=True):
"""
Create a benchmark run.
Args:
values: Sequence of timing measurements (numbers > 0)
warmups: Optional sequence of (loops, value) tuples for warmup measurements
metadata: Optional dictionary of metadata
collect_metadata: Whether to automatically collect system metadata
"""
def get_metadata(self) -> dict:
"""Get run metadata including system information."""
def get_loops(self) -> int:
"""Get number of loops per measurement."""
def get_inner_loops(self) -> int:
"""
Get inner loop count from metadata.
Returns:
Inner loop count (default: 1 if not specified)
"""
def get_total_loops(self) -> int:
"""
Get total loops (loops * inner_loops).
Returns:
Total number of loop iterations per measurement
"""
@property
def warmups(self) -> tuple:
"""Tuple of warmup measurements."""
@property
def values(self) -> tuple:
"""Tuple of timing values."""Benchmark Class
Collection of benchmark runs with comprehensive statistical analysis capabilities.
class Benchmark:
def __init__(self, runs):
"""
Create a benchmark from a sequence of Run objects.
Args:
runs: Non-empty sequence of Run objects
"""
def get_name(self) -> str:
"""Get benchmark name."""
def get_metadata(self) -> dict:
"""Get metadata common to all runs."""
def get_runs(self) -> list:
"""Get list of Run objects."""
def get_values(self) -> tuple:
"""Get all measurement values from all runs."""
def get_nrun(self) -> int:
"""Get number of runs."""
def get_nvalue(self) -> int:
"""Get total number of measurement values."""
def get_unit(self) -> str:
"""Get measurement unit (e.g., 'second', 'byte')."""
def get_total_duration(self) -> float:
"""Get total execution time for all runs."""
def get_dates(self) -> tuple:
"""Get (start_date, end_date) timestamps."""
def mean(self) -> float:
"""Mean of all measurement values (cached property)."""
def stdev(self) -> float:
"""Standard deviation of all measurement values (cached property)."""
def median(self) -> float:
"""Median of all measurement values (cached property)."""
def median_abs_dev(self) -> float:
"""Median absolute deviation of all measurement values (cached property)."""
def get_dates(self) -> tuple:
"""
Get benchmark execution date range.
Returns:
Tuple of (start_date, end_date) as datetime objects,
or None if no date information available
"""
def required_nprocesses(self) -> int:
"""
Get recommended number of processes for stable results.
Returns:
Number of processes needed for 95% certainty with <1% variance,
or None if insufficient data
"""
def percentile(self, p: float) -> float:
"""
Get percentile value.
Args:
p: Percentile (0-100)
Returns:
Value at the specified percentile
"""
def add_run(self, run: Run):
"""
Add a new run to this benchmark.
Args:
run: Run object to add
"""
def add_runs(self, benchmark: 'Benchmark'):
"""
Merge runs from another benchmark.
Args:
benchmark: Benchmark object to merge runs from
"""
def format_value(self, value: float) -> str:
"""
Format a single value for display.
Args:
value: Value to format
Returns:
Formatted string representation
"""
def format_values(self, values: tuple) -> list:
"""
Format multiple values for display.
Args:
values: Tuple of values to format
Returns:
List of formatted string representations
"""
def update_metadata(self, metadata: dict):
"""
Update benchmark metadata.
Args:
metadata: Dictionary of metadata to merge
"""
@staticmethod
def load(file) -> 'Benchmark':
"""
Load benchmark from JSON file.
Args:
file: File path or file object
Returns:
Benchmark object loaded from file
"""
@staticmethod
def loads(string: str) -> 'Benchmark':
"""
Load benchmark from JSON string.
Args:
string: JSON string representation
Returns:
Benchmark object parsed from string
"""
def dump(self, file, compact=True, replace=False):
"""
Save benchmark to JSON file.
Args:
file: Output file path or file object
compact: Whether to use compact JSON format
replace: Whether to replace existing file
"""BenchmarkSuite Class
Collection of benchmarks that can be managed and saved together.
class BenchmarkSuite:
def __init__(self, benchmarks, filename=None):
"""
Create a benchmark suite.
Args:
benchmarks: Non-empty sequence of Benchmark objects
filename: Optional filename for reference
"""
def get_benchmarks(self) -> list:
"""Get list of Benchmark objects."""
def get_benchmark(self, name: str) -> Benchmark:
"""
Get benchmark by name.
Args:
name: Benchmark name
Returns:
Benchmark object with matching name
"""
def get_benchmark_names(self) -> list:
"""Get list of benchmark names."""
def get_metadata(self) -> dict:
"""Get suite-wide metadata."""
def add_benchmark(self, benchmark: Benchmark):
"""
Add a benchmark to this suite.
Args:
benchmark: Benchmark object to add
"""
def add_runs(self, result):
"""
Add runs from Benchmark or BenchmarkSuite.
Args:
result: Benchmark or BenchmarkSuite object
"""
def get_total_duration(self) -> float:
"""Get total execution time for all benchmarks."""
def get_dates(self) -> tuple:
"""Get (start_date, end_date) timestamps for entire suite."""
@classmethod
def load(cls, file) -> 'BenchmarkSuite':
"""
Load benchmark suite from JSON file.
Args:
file: File path or file object
Returns:
BenchmarkSuite object loaded from file
"""
@classmethod
def loads(cls, string: str) -> 'BenchmarkSuite':
"""
Load benchmark suite from JSON string.
Args:
string: JSON string representation
Returns:
BenchmarkSuite object parsed from string
"""
def dump(self, file, compact=True, replace=False):
"""
Save benchmark suite to JSON file.
Args:
file: Output file path or file object
compact: Whether to use compact JSON format
replace: Whether to replace existing file
"""Suite Management Functions
Utility functions for working with benchmark files and data merging.
def add_runs(filename: str, result):
"""
Add benchmark results to existing JSON file.
Args:
filename: Path to JSON file
result: Benchmark or BenchmarkSuite object to add
"""Usage Examples
Creating and Managing Runs
import pyperf
# Create runs manually
values = [0.001, 0.0012, 0.0011, 0.001, 0.0013]
run1 = pyperf.Run(values, metadata={'version': '1.0'})
run2 = pyperf.Run([0.0009, 0.001, 0.0011], metadata={'version': '1.0'})
# Create benchmark from runs
benchmark = pyperf.Benchmark([run1, run2])
print(f"Mean: {benchmark.mean():.6f} seconds")
print(f"Median: {benchmark.median():.6f} seconds")
print(f"Standard deviation: {benchmark.stdev():.6f} seconds")Statistical Analysis
import pyperf
# Load benchmark from file
benchmark = pyperf.Benchmark.load('results.json')
# Basic statistics
print(f"Number of runs: {benchmark.get_nrun()}")
print(f"Total values: {benchmark.get_nvalue()}")
print(f"Mean: {benchmark.mean():.6f} ± {benchmark.stdev():.6f} seconds")
# Percentile analysis
print(f"Min (0th percentile): {benchmark.percentile(0):.6f}")
print(f"25th percentile: {benchmark.percentile(25):.6f}")
print(f"Median (50th percentile): {benchmark.percentile(50):.6f}")
print(f"75th percentile: {benchmark.percentile(75):.6f}")
print(f"Max (100th percentile): {benchmark.percentile(100):.6f}")
# Median absolute deviation
print(f"Median absolute deviation: {benchmark.median_abs_dev():.6f}")Data Persistence
import pyperf
# Save single benchmark
runner = pyperf.Runner()
benchmark = runner.timeit('test', '[i for i in range(100)]')
benchmark.dump('single_benchmark.json')
# Load and modify
loaded = pyperf.Benchmark.load('single_benchmark.json')
loaded.update_metadata({'test_type': 'list_comprehension'})
loaded.dump('modified_benchmark.json')
# Create and save benchmark suite
suite = pyperf.BenchmarkSuite([benchmark])
suite.dump('benchmark_suite.json')
# Add results to existing file
new_benchmark = runner.timeit('test2', 'sum(range(100))')
pyperf.add_runs('benchmark_suite.json', new_benchmark)Working with Metadata
import pyperf
benchmark = pyperf.Benchmark.load('results.json')
# Examine metadata
metadata = benchmark.get_metadata()
print(f"Python version: {metadata.get('python_version')}")
print(f"CPU model: {metadata.get('cpu_model_name')}")
print(f"Platform: {metadata.get('platform')}")
# Get run-specific metadata
for i, run in enumerate(benchmark.get_runs()):
run_metadata = run.get_metadata()
print(f"Run {i}: {run_metadata.get('date')}")Benchmark Merging
import pyperf
# Load multiple benchmark files
bench1 = pyperf.Benchmark.load('results1.json')
bench2 = pyperf.Benchmark.load('results2.json')
# Merge benchmarks (must have compatible metadata)
bench1.add_runs(bench2)
print(f"Combined runs: {bench1.get_nrun()}")
# Create suite from multiple benchmarks
suite = pyperf.BenchmarkSuite([bench1, bench2])
print(f"Suite benchmarks: {suite.get_benchmark_names()}")Install with Tessl CLI
npx tessl i tessl/pypi-pyperf