tessl/pypi-hdmf
A hierarchical data modeling framework for modern science data standards
—
Pending
specification.mddocs/
Specification System
HDMF's specification system provides schema definition and management for data models. It enables the creation of structured schemas that define data organization, validation rules, and metadata requirements, serving as the foundation for HDMF's specification-driven architecture.
Capabilities
Specification Catalogs
Core catalog classes for managing collections of specifications and namespaces.
class SpecCatalog:
"""
Catalog for managing data type specifications.
Provides registration, retrieval, and organization of specifications
for different data types within a namespace.
"""
def __init__(self):
"""Initialize empty specification catalog."""
def register_spec(self, spec, source_file: str = None):
"""
Register a specification in the catalog.
Args:
spec: Specification object to register
source_file: Source file path for the specification
"""
def get_spec(self, neurodata_type: str) -> 'BaseStorageSpec':
"""
Get specification by data type name.
Args:
neurodata_type: Name of the data type
Returns:
Specification object for the data type
Raises:
KeyError: If specification not found
"""
def get_hierarchy(self, neurodata_type: str) -> list:
"""
Get inheritance hierarchy for a data type.
Args:
neurodata_type: Name of the data type
Returns:
List of parent types in inheritance order
"""
class NamespaceCatalog:
"""
Catalog for managing namespaces and their associated specifications.
Organizes multiple specification catalogs under different namespaces
and handles namespace resolution and dependencies.
"""
def __init__(self, *args, **kwargs):
"""Initialize namespace catalog."""
def add_namespace(self, namespace: str, spec_catalog: SpecCatalog):
"""
Add namespace with its specification catalog.
Args:
namespace: Namespace identifier
spec_catalog: Specification catalog for the namespace
"""
def get_namespace(self, namespace: str) -> 'SpecNamespace':
"""
Get namespace by name.
Args:
namespace: Namespace identifier
Returns:
SpecNamespace object
"""
def load_namespaces(self, namespace_path: str, **kwargs):
"""
Load namespaces from file or directory.
Args:
namespace_path: Path to namespace files
"""Namespace Definitions
Classes for defining and managing specification namespaces.
class SpecNamespace:
"""
Specification namespace containing metadata and dependencies.
Defines a namespace with version information, documentation,
and relationships to other namespaces.
"""
def __init__(self, doc: str, name: str, **kwargs):
"""
Initialize specification namespace.
Args:
doc: Documentation for the namespace
name: Namespace identifier
**kwargs: Additional namespace properties:
- version: Namespace version
- author: Namespace author(s)
- contact: Contact information
- dependencies: List of dependency namespaces
"""
@property
def name(self) -> str:
"""Namespace name."""
@property
def version(self) -> str:
"""Namespace version."""
@property
def doc(self) -> str:
"""Namespace documentation."""
@property
def dependencies(self) -> list:
"""List of namespace dependencies."""
class NamespaceBuilder:
"""
Builder for creating specification namespaces programmatically.
Provides a fluent interface for constructing namespaces with
specifications, dependencies, and metadata.
"""
def __init__(self, doc: str, name: str, **kwargs):
"""
Initialize namespace builder.
Args:
doc: Documentation for the namespace
name: Namespace identifier
"""
def include_namespace(self, namespace: str):
"""
Include another namespace as dependency.
Args:
namespace: Namespace to include
Returns:
Self for method chaining
"""
def include_type(self, type_name: str, source_file: str = None):
"""
Include a data type in the namespace.
Args:
type_name: Name of the data type
source_file: Source file containing the type
Returns:
Self for method chaining
"""
def export(self, namespace_path: str, **kwargs):
"""
Export namespace to file.
Args:
namespace_path: Path where to export namespace
"""Data Type Specifications
Core specification classes for defining data structures and validation rules.
class GroupSpec:
"""
Specification for group (container) data types.
Defines hierarchical containers that can hold datasets, other groups,
attributes, and links with validation rules and metadata.
"""
def __init__(self, doc: str, name: str = None, **kwargs):
"""
Initialize group specification.
Args:
doc: Documentation for the group
name: Name of the group (None for flexible naming)
**kwargs: Additional specification properties:
- neurodata_type_def: Data type being defined
- neurodata_type_inc: Data type being inherited from
- default_name: Default name for the group
- linkable: Whether group can be linked
- attributes: List of attribute specifications
- datasets: List of dataset specifications
- groups: List of nested group specifications
"""
def add_attribute(self, attr_spec: 'AttributeSpec'):
"""
Add attribute specification to the group.
Args:
attr_spec: Attribute specification to add
"""
def add_dataset(self, dataset_spec: 'DatasetSpec'):
"""
Add dataset specification to the group.
Args:
dataset_spec: Dataset specification to add
"""
def add_group(self, group_spec: 'GroupSpec'):
"""
Add nested group specification.
Args:
group_spec: Group specification to add
"""
class DatasetSpec:
"""
Specification for dataset data types.
Defines data arrays with shape constraints, data type requirements,
and associated metadata validation rules.
"""
def __init__(self, doc: str, name: str = None, **kwargs):
"""
Initialize dataset specification.
Args:
doc: Documentation for the dataset
name: Name of the dataset (None for flexible naming)
**kwargs: Dataset properties:
- neurodata_type_def: Data type being defined
- neurodata_type_inc: Data type being inherited from
- dtype: Data type specification
- shape: Shape constraints
- dims: Dimension names
- default_name: Default name for the dataset
- linkable: Whether dataset can be linked
- attributes: List of attribute specifications
"""
def add_attribute(self, attr_spec: 'AttributeSpec'):
"""
Add attribute specification to the dataset.
Args:
attr_spec: Attribute specification to add
"""
class AttributeSpec:
"""
Specification for metadata attributes.
Defines key-value metadata with type constraints and validation rules.
"""
def __init__(self, name: str, doc: str, **kwargs):
"""
Initialize attribute specification.
Args:
name: Name of the attribute
doc: Documentation for the attribute
**kwargs: Attribute properties:
- dtype: Data type specification
- value: Fixed value for the attribute
- default_value: Default value
- required: Whether attribute is required
- shape: Shape constraints for array attributes
"""
class LinkSpec:
"""
Specification for links between data elements.
Defines relationships and references between different parts
of the hierarchical data structure.
"""
def __init__(self, doc: str, **kwargs):
"""
Initialize link specification.
Args:
doc: Documentation for the link
**kwargs: Link properties:
- target_type: Target data type for the link
- allow_subclasses: Allow subclasses of target type
"""
class RefSpec:
"""
Specification for object references.
Defines references to other objects within the data hierarchy
with type constraints and validation rules.
"""
def __init__(self, target_type: str, reftype: str, **kwargs):
"""
Initialize reference specification.
Args:
target_type: Target data type for references
reftype: Type of reference ('object', 'region')
"""Data Type Specifications
Classes for defining and managing data types within specifications.
class DtypeSpec:
"""
Specification for data types with validation and constraints.
Defines allowable data types, value constraints, and conversion rules
for datasets and attributes.
"""
def __init__(self, name: str, doc: str, **kwargs):
"""
Initialize data type specification.
Args:
name: Name of the data type
doc: Documentation for the data type
**kwargs: Data type properties:
- dtype: Base data type
- constraints: Value constraints
- default_value: Default value
"""
class DtypeHelper:
"""
Helper utilities for working with data type specifications.
Provides validation, conversion, and analysis functions for
data types used in specifications.
"""
@staticmethod
def check_dtype(dtype_spec, value) -> bool:
"""
Check if value matches data type specification.
Args:
dtype_spec: Data type specification
value: Value to check
Returns:
True if value matches specification
"""
@staticmethod
def convert_dtype(dtype_spec, value):
"""
Convert value to match data type specification.
Args:
dtype_spec: Target data type specification
value: Value to convert
Returns:
Converted value
"""Specification I/O
Classes for reading and writing specifications to different formats.
class SpecReader:
"""
Reader for loading specifications from files.
Supports multiple file formats including YAML and JSON for
specification definitions and namespace declarations.
"""
def __init__(self, **kwargs):
"""Initialize specification reader."""
def read_spec(self, spec_file: str) -> dict:
"""
Read specification from file.
Args:
spec_file: Path to specification file
Returns:
Dictionary containing specification data
"""
def read_namespace(self, namespace_file: str) -> dict:
"""
Read namespace from file.
Args:
namespace_file: Path to namespace file
Returns:
Dictionary containing namespace data
"""
class SpecWriter:
"""
Writer for saving specifications to files.
Exports specifications and namespaces to YAML or JSON format
for sharing and version control.
"""
def __init__(self, **kwargs):
"""Initialize specification writer."""
def write_spec(self, spec, spec_file: str):
"""
Write specification to file.
Args:
spec: Specification object to write
spec_file: Output file path
"""
def write_namespace(self, namespace, namespace_file: str):
"""
Write namespace to file.
Args:
namespace: Namespace object to write
namespace_file: Output file path
"""Specification Utilities
Utility functions and constants for working with specifications.
def export_spec(namespace_builder: NamespaceBuilder, export_path: str, **kwargs):
"""
Export namespace and specifications to directory.
Args:
namespace_builder: NamespaceBuilder to export
export_path: Directory path for export
**kwargs: Export options
"""
# Constants
NAME_WILDCARD = '*' # Wildcard for flexible naming in specificationsUsage Examples
Creating Basic Specifications
from hdmf.spec import GroupSpec, DatasetSpec, AttributeSpec, NamespaceBuilder
# Create attribute specification
name_attr = AttributeSpec(
name='name',
doc='Name of the experimental subject',
dtype='text',
required=True
)
# Create dataset specification
data_spec = DatasetSpec(
doc='Neural recording data',
name='data',
dtype='float64',
shape=(None, None), # Variable dimensions
dims=['time', 'channels'],
attributes=[name_attr]
)
# Create group specification
recording_spec = GroupSpec(
doc='Neural recording container',
neurodata_type_def='Recording',
default_name='recording',
datasets=[data_spec],
attributes=[
AttributeSpec('sampling_rate', 'Sampling rate in Hz', dtype='float64')
]
)Building and Exporting Namespaces
from hdmf.spec import NamespaceBuilder
# Create namespace builder
ns_builder = NamespaceBuilder(
doc='Experimental neuroscience data standard',
name='neuro-experiment',
version='1.0.0',
author='Neuroscience Lab',
contact='lab@university.edu'
)
# Include HDMF common namespace
ns_builder.include_namespace('hdmf-common')
# Add custom data types
ns_builder.include_type('Recording', source_file='recording.yaml')
ns_builder.include_type('Stimulus', source_file='stimulus.yaml')
# Export namespace and specifications
ns_builder.export('./specs/', overwrite=True)Loading Existing Specifications
from hdmf.spec import NamespaceCatalog, SpecCatalog
from hdmf.common import load_namespaces
# Load HDMF common specifications
load_namespaces()
# Load custom namespace
namespace_catalog = NamespaceCatalog()
namespace_catalog.load_namespaces('./custom_specs/namespace.yaml')
# Get specification catalog for a namespace
spec_catalog = namespace_catalog.get_namespace('custom-namespace').catalog
# Retrieve specific specifications
recording_spec = spec_catalog.get_spec('Recording')
print(f"Recording spec: {recording_spec.doc}")
# Get inheritance hierarchy
hierarchy = spec_catalog.get_hierarchy('Recording')
print(f"Inheritance: {hierarchy}")Creating Complex Specifications with Inheritance
from hdmf.spec import GroupSpec, DatasetSpec
# Base container specification
base_container = GroupSpec(
doc='Base container for all experimental data',
neurodata_type_def='BaseContainer',
attributes=[
AttributeSpec('description', 'Description of the container', dtype='text'),
AttributeSpec('created_on', 'Creation timestamp', dtype='text')
]
)
# Specialized recording container inheriting from base
recording_container = GroupSpec(
doc='Container for neural recordings',
neurodata_type_def='RecordingContainer',
neurodata_type_inc='BaseContainer', # Inherit from BaseContainer
datasets=[
DatasetSpec(
doc='Raw neural data',
name='data',
dtype='int16',
shape=(None, None),
dims=['time', 'channels']
)
],
attributes=[
AttributeSpec('sampling_rate', 'Sampling rate in Hz', dtype='float64')
]
)Advanced Data Type Specifications
from hdmf.spec import DatasetSpec, DtypeSpec
# Complex data type with constraints
constrained_dtype = DtypeSpec(
name='bounded_float',
doc='Float value between 0 and 1',
dtype='float64',
constraints={'min': 0.0, 'max': 1.0}
)
# Dataset with complex shape and type constraints
timeseries_spec = DatasetSpec(
doc='Time series data with metadata',
name='timeseries',
dtype=[
{'name': 'timestamp', 'dtype': 'float64', 'doc': 'Time in seconds'},
{'name': 'value', 'dtype': constrained_dtype, 'doc': 'Normalized value'},
{'name': 'quality', 'dtype': 'uint8', 'doc': 'Data quality flag'}
],
shape=(None,), # Variable length
dims=['time']
)Programmatic Specification Creation
from hdmf.spec import GroupSpec, DatasetSpec, AttributeSpec
def create_experiment_spec(experiment_name: str) -> GroupSpec:
"""Create specification for a specific experiment type."""
# Common attributes for all experiments
common_attrs = [
AttributeSpec('experiment_id', 'Unique experiment identifier', dtype='text'),
AttributeSpec('start_time', 'Experiment start time', dtype='text'),
AttributeSpec('duration', 'Experiment duration in seconds', dtype='float64')
]
# Experiment-specific datasets
if experiment_name == 'electrophysiology':
datasets = [
DatasetSpec('voltage_traces', 'Voltage recordings', dtype='float64',
shape=(None, None), dims=['time', 'channels']),
DatasetSpec('spike_times', 'Detected spike timestamps', dtype='float64',
shape=(None,), dims=['spikes'])
]
elif experiment_name == 'behavior':
datasets = [
DatasetSpec('position', 'Animal position over time', dtype='float64',
shape=(None, 2), dims=['time', 'coordinates']),
DatasetSpec('speed', 'Movement speed', dtype='float64',
shape=(None,), dims=['time'])
]
else:
datasets = []
# Create and return specification
return GroupSpec(
doc=f'Container for {experiment_name} experiments',
neurodata_type_def=f'{experiment_name.title()}Experiment',
default_name=experiment_name,
attributes=common_attrs,
datasets=datasets
)
# Create specifications for different experiment types
ephys_spec = create_experiment_spec('electrophysiology')
behavior_spec = create_experiment_spec('behavior')Install with Tessl CLI
npx tessl i tessl/pypi-hdmf