A py.test plugin to validate Jupyter notebooks
npx @tessl/cli install tessl/pypi-nbval@0.11.0A py.test plugin to validate Jupyter notebooks by executing notebook cells and comparing their outputs against stored results. NBVal enables automated testing of notebook content to ensure consistency and functionality, supporting features like output sanitization through regex patterns, parallel execution compatibility, coverage integration, and flexible kernel selection.
pip install nbvalimport nbvalFor accessing plugin functionality directly:
from nbval.plugin import IPyNbFile, IPyNbCell, NbCellError
from nbval.kernel import RunningKernel, start_new_kernelNBVal works as a pytest plugin, activated through command-line flags:
# Run notebooks validating all outputs
pytest --nbval
# Run notebooks only validating marked cells
pytest --nbval-lax
# Test a specific notebook
pytest --nbval my_notebook.ipynb
# Use output sanitization
pytest --nbval my_notebook.ipynb --nbval-sanitize-with sanitize_config.cfg
# Use current environment kernel
pytest --nbval --nbval-current-env
# Use specific kernel
pytest --nbval --nbval-kernel-name python3Control cell behavior using comment markers or metadata tags:
# NBVAL_IGNORE_OUTPUT - Skip output validation
print("This output won't be checked")
# NBVAL_CHECK_OUTPUT - Force output validation (useful with --nbval-lax)
print("This output will be validated")
# NBVAL_RAISES_EXCEPTION - Expect cell to raise exception
raise ValueError("Expected error")
# NBVAL_SKIP - Skip cell execution entirely
# This cell won't runCreate a sanitization file to clean outputs before comparison:
[timestamps]
regex: \d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}
replace: TIMESTAMP
[memory_addresses]
regex: 0x[0-9a-fA-F]+
replace: 0xADDRESSAccess to package version information.
__version__: strCore pytest integration hooks for notebook collection and configuration.
def pytest_addoption(parser):
"""
Add nbval command-line options to pytest.
Args:
parser: pytest argument parser
"""
def pytest_configure(config):
"""
Configure nbval plugin based on options.
Args:
config: pytest configuration object
"""
def pytest_collect_file(file_path, parent):
"""
Collect .ipynb files for testing.
Args:
file_path: Path to potential test file
parent: Parent collector
Returns:
IPyNbFile instance for .ipynb files, None otherwise
"""Pytest collector for handling notebook files as test suites.
class IPyNbFile(pytest.File):
"""
Pytest collector for notebook files.
"""
def setup(self):
"""Set up notebook execution environment."""
def collect(self):
"""
Collect notebook cells as test items.
Returns:
List of IPyNbCell instances
"""
def teardown(self):
"""Clean up notebook execution environment."""
def get_kernel_message(self, timeout=None):
"""
Get message from running kernel.
Args:
timeout: Message timeout in seconds
Returns:
Kernel message dictionary
"""
def setup_sanitize_files(self):
"""Set up output sanitization configuration."""
def get_sanitize_files(self):
"""
Get sanitization configuration.
Returns:
List of sanitization patterns
"""Pytest item representing individual notebook cells for execution and validation.
class IPyNbCell(pytest.Item):
"""
Pytest item representing a notebook cell.
"""
def runtest(self):
"""Execute cell and compare outputs."""
def repr_failure(self, excinfo):
"""
Represent test failure for reporting.
Args:
excinfo: Exception information
Returns:
Formatted failure representation
"""
def reportinfo(self):
"""
Get test location information.
Returns:
Tuple of (file_path, line_number, test_name)
"""
def compare_outputs(self, test, ref, skip_compare=None):
"""
Compare cell outputs against reference.
Args:
test: Actual execution output
ref: Reference output from notebook
skip_compare: Output types to skip comparison
Returns:
True if outputs match, False otherwise
"""
def setup(self):
"""Set up cell execution environment."""
def raise_cell_error(self, **kwargs):
"""
Raise formatted cell execution error.
Args:
**kwargs: Error details
Raises:
NbCellError: Formatted cell error
"""
def sanitize(self, s):
"""
Apply output sanitization patterns.
Args:
s: String to sanitize
Returns:
Sanitized string
"""
def sanitize_outputs(self, outputs):
"""
Apply sanitization to output list.
Args:
outputs: List of output dictionaries
Returns:
List of sanitized outputs
"""Custom exception for notebook cell execution errors.
class NbCellError(Exception):
"""
Custom exception for cell execution errors.
Attributes:
cell_num: Cell number that failed
source: Cell source code
inner_traceback: Original exception traceback
"""
def __init__(self, cell_num, msg, source, traceback=None, *args, **kwargs):
"""
Initialize cell error.
Args:
cell_num: Failed cell number
msg: Error message
source: Cell source code
traceback: Original traceback
*args: Additional arguments
**kwargs: Additional keyword arguments
"""Jupyter kernel execution and lifecycle management.
class RunningKernel:
"""
Manages Jupyter kernel execution.
"""
def __init__(self, kernel_name, cwd=None, startup_timeout=60):
"""
Initialize kernel manager.
Args:
kernel_name: Name of kernel to start
cwd: Working directory for kernel
startup_timeout: Kernel startup timeout in seconds
"""
def execute_cell_input(self, cell_input, allow_stdin=True):
"""
Execute cell input in kernel.
Args:
cell_input: Cell source code to execute
allow_stdin: Whether to allow stdin
Returns:
Execution message ID
"""
def get_message(self, timeout=None):
"""
Get message from kernel.
Args:
timeout: Message timeout in seconds
Returns:
Kernel message dictionary
"""
def await_reply(self, msg_id, timeout=None):
"""
Wait for execution reply.
Args:
msg_id: Message ID to wait for
timeout: Reply timeout in seconds
Returns:
Reply message dictionary
"""
def await_idle(self, msg_id, timeout=None):
"""
Wait for kernel to become idle.
Args:
msg_id: Message ID to wait for
timeout: Idle timeout in seconds
"""
def is_alive(self):
"""
Check if kernel is alive.
Returns:
True if kernel is running, False otherwise
"""
def restart(self):
"""Restart the kernel."""
def interrupt(self):
"""Interrupt kernel execution."""
def stop(self):
"""Stop and cleanup kernel."""
@property
def language(self):
"""
Get kernel language.
Returns:
Kernel language name
"""
def start_new_kernel(startup_timeout=60, kernel_name=None, **kwargs):
"""
Start new Jupyter kernel.
Args:
startup_timeout: Kernel startup timeout in seconds
kernel_name: Name of kernel to start
**kwargs: Additional kernel arguments
Returns:
Tuple of (kernel_manager, kernel_client)
"""
class NbvalKernelspecManager(KernelSpecManager):
"""Custom kernel spec manager for nbval."""Helper functions for notebook processing and output handling.
def find_comment_markers(cellsource):
"""
Find special comment markers in cell source.
Args:
cellsource: Cell source code
Returns:
Dictionary of found markers
"""
def find_metadata_tags(cell_metadata):
"""
Find nbval tags in cell metadata.
Args:
cell_metadata: Cell metadata dictionary
Returns:
Dictionary of found tags
"""
def coalesce_streams(outputs):
"""
Merge stream outputs for consistent results.
Args:
outputs: List of output dictionaries
Returns:
List of coalesced outputs
"""
def transform_streams_for_comparison(outputs):
"""
Transform stream outputs for comparison.
Args:
outputs: List of output dictionaries
Returns:
List of transformed outputs
"""
def get_sanitize_patterns(string):
"""
Parse regex patterns from sanitize config.
Args:
string: Configuration string
Returns:
List of (regex, replacement) tuples
"""
def hash_string(s):
"""
Create MD5 hash of string.
Args:
s: String to hash
Returns:
MD5 hash hexdigest
"""
def _trim_base64(s):
"""
Trim and hash base64 strings for cleaner output display.
Args:
s: String to trim if it's base64
Returns:
Trimmed string with hash if base64, original string otherwise
"""
def _indent(s, indent=' '):
"""
Indent each line with specified indentation.
Args:
s: String to indent
indent: Indentation string (default: ' ')
Returns:
Indented string
"""Coverage reporting setup and teardown for notebook testing.
def setup_coverage(config, kernel, floc, output_loc):
"""
Set up coverage reporting.
Args:
config: pytest configuration
kernel: Running kernel instance
floc: File location
output_loc: Output location
"""
def teardown_coverage(config, kernel):
"""
Tear down coverage reporting.
Args:
config: pytest configuration
kernel: Running kernel instance
"""Reporter for visual diff display of notebook failures.
class NbdimeReporter:
"""Pytest reporter for nbdime integration."""
def __init__(self, config, file=None):
"""
Initialize nbdime reporter.
Args:
config: pytest configuration
file: Output file handle
"""
def pytest_runtest_logreport(self, report):
"""
Handle test log reports.
Args:
report: pytest test report
"""
def pytest_collectreport(self, report):
"""
Handle collection reports.
Args:
report: pytest collection report
"""
def pytest_sessionfinish(self, exitstatus):
"""
Handle session finish.
Args:
exitstatus: pytest exit status
"""
def make_report(self, outcome):
"""
Create nbdime report.
Args:
outcome: Test outcome
Returns:
Report data
"""NBVal adds the following options to pytest:
--nbval: Run notebooks validating all outputs--nbval-lax: Run notebooks only validating marked cells--nbval-sanitize-with FILE: File with regex patterns to sanitize outputs--nbval-current-env: Use current environment kernel--nbval-kernel-name KERNEL: Force specific kernel name--nbval-cell-timeout SECONDS: Cell execution timeout--nbval-kernel-startup-timeout SECONDS: Kernel startup timeout--nbdime: View failed cells with nbdimeUse in cell source code:
# NBVAL_IGNORE_OUTPUT: Skip output validation# NBVAL_CHECK_OUTPUT: Force output validation# NBVAL_RAISES_EXCEPTION: Expect cell to raise exception# NBVAL_SKIP: Skip cell execution# PYTEST_VALIDATE_IGNORE_OUTPUT: Legacy markerUse in cell metadata.tags:
nbval-ignore-output, nbval-check-output, nbval-raises-exception, nbval-skip, raises-exceptionCURRENT_ENV_KERNEL_NAME: str
comment_markers: dict
metadata_tags: dict
class bcolors:
"""Color codes for terminal output."""
class nocolors:
"""No-color version of bcolors."""version_info: tuple