0
# Build System
1

2
HDMF's build system converts container objects to storage representations and manages type mappings between specifications and Python classes. It provides the bridge between HDMF's object-oriented container system and the storage backend formats, enabling serialization and deserialization with validation.
3

4
## Capabilities
5

6
### Builder Classes
7

8
Core builder classes for creating storage representations of different data elements.
9

10
```python { .api }
11
class Builder:
12
    """
13
    Abstract base class for all builders.
14
    
15
    Builders create storage representations that can be written to
16
    different backends (HDF5, Zarr, etc.) while maintaining metadata
17
    and structural information.
18
    """
19
    
20
    def __init__(self, name: str, **kwargs):
21
        """
22
        Initialize builder.
23
        
24
        Args:
25
            name: Name of the data element
26
            **kwargs: Additional builder properties:
27
                - attributes: Dictionary of attributes
28
                - object_id: Unique identifier
29
                - parent: Parent builder
30
        """
31
    
32
    @property
33
    def name(self) -> str:
34
        """Name of the data element."""
35
    
36
    @property
37
    def attributes(self) -> dict:
38
        """Dictionary of attributes."""
39
    
40
    def set_attribute(self, name: str, value):
41
        """
42
        Set attribute value.
43
        
44
        Args:
45
            name: Attribute name
46
            value: Attribute value
47
        """
48

49
class GroupBuilder(Builder):
50
    """
51
    Builder for group (container) structures.
52
    
53
    Creates hierarchical storage representations that can contain
54
    datasets, nested groups, and links with associated metadata.
55
    """
56
    
57
    def __init__(self, name: str, **kwargs):
58
        """
59
        Initialize group builder.
60
        
61
        Args:
62
            name: Name of the group
63
            **kwargs: Group builder properties:
64
                - groups: Dictionary of nested group builders
65
                - datasets: Dictionary of dataset builders
66
                - links: Dictionary of link builders
67
        """
68
    
69
    def set_group(self, builder: 'GroupBuilder'):
70
        """
71
        Add nested group builder.
72
        
73
        Args:
74
            builder: Group builder to add
75
        """
76
    
77
    def set_dataset(self, builder: 'DatasetBuilder'):
78
        """
79
        Add dataset builder.
80
        
81
        Args:
82
            builder: Dataset builder to add
83
        """
84
    
85
    def set_link(self, builder: 'LinkBuilder'):
86
        """
87
        Add link builder.
88
        
89
        Args:
90
            builder: Link builder to add
91
        """
92
    
93
    @property
94
    def groups(self) -> dict:
95
        """Dictionary of nested group builders."""
96
    
97
    @property
98
    def datasets(self) -> dict:
99
        """Dictionary of dataset builders."""
100

101
### Build Error Classes
102

103
Exception classes for build system errors and warnings with detailed error reporting.
104

105
```python { .api }
106
class BuildError(Exception):
107
    """
108
    Base exception for build system errors.
109
    
110
    Raised when build operations fail due to invalid specifications,
111
    missing dependencies, or incompatible data structures.
112
    """
113
    pass
114

115
class OrphanContainerBuildError(BuildError):
116
    """
117
    Error for orphaned containers during build process.
118
    
119
    Raised when containers lack proper parent relationships
120
    required for building storage representations.
121
    """
122
    pass
123

124
class ReferenceTargetNotBuiltError(BuildError):
125
    """
126
    Error for unresolved reference targets during build.
127
    
128
    Raised when referenced objects haven't been built yet
129
    or are missing from the build context.
130
    """
131
    pass
132

133
class ContainerConfigurationError(BuildError):
134
    """
135
    Error for invalid container configurations.
136
    
137
    Raised when container configurations violate specification
138
    requirements or have incompatible settings.
139
    """
140
    pass
141

142
class ConstructError(BuildError):
143
    """
144
    Error during object construction from builders.
145
    
146
    Raised when builders cannot be converted to container objects
147
    due to specification mismatches or invalid data.
148
    """
149
    pass
150

151
# Build warnings
152
class BuildWarning(UserWarning):
153
    """Base warning for build system issues."""
154
    pass
155

156
class MissingRequiredBuildWarning(BuildWarning):
157
    """Warning for missing required build components."""
158
    pass
159

160
class DtypeConversionWarning(BuildWarning):
161
    """Warning for data type conversion during build."""
162
    pass
163

164
class IncorrectQuantityBuildWarning(BuildWarning):
165
    """Warning for incorrect quantities during build."""
166
    pass
167

168
class OrphanContainerWarning(BuildWarning):
169
    """Warning for orphaned containers."""
170
    pass
171
```
172
    
173
    @property
174
    def links(self) -> dict:
175
        """Dictionary of link builders."""
176

177
class DatasetBuilder(Builder):
178
    """
179
    Builder for dataset structures.
180
    
181
    Creates storage representations for data arrays with metadata,
182
    chunking, compression, and other storage-specific properties.
183
    """
184
    
185
    def __init__(self, name: str, data, **kwargs):
186
        """
187
        Initialize dataset builder.
188
        
189
        Args:
190
            name: Name of the dataset
191
            data: Dataset content
192
            **kwargs: Dataset builder properties:
193
                - dtype: Data type
194
                - shape: Data shape
195
                - maxshape: Maximum shape for resizable datasets
196
                - chunks: Chunk configuration
197
                - compression: Compression settings
198
        """
199
    
200
    @property
201
    def data(self):
202
        """Dataset content."""
203
    
204
    @property
205
    def shape(self) -> tuple:
206
        """Shape of the dataset."""
207
    
208
    @property
209
    def dtype(self):
210
        """Data type of the dataset."""
211

212
class LinkBuilder(Builder):
213
    """
214
    Builder for link structures.
215
    
216
    Creates storage representations for links between data elements,
217
    enabling references and relationships within the data hierarchy.
218
    """
219
    
220
    def __init__(self, builder: Builder, name: str, **kwargs):
221
        """
222
        Initialize link builder.
223
        
224
        Args:
225
            builder: Target builder for the link
226
            name: Name of the link
227
        """
228
    
229
    @property
230
    def builder(self) -> Builder:
231
        """Target builder for the link."""
232

233
class ReferenceBuilder(Builder):
234
    """
235
    Builder for object references.
236
    
237
    Creates storage representations for references to other objects
238
    within the data hierarchy, enabling complex relationships and cross-references.
239
    """
240
    
241
    def __init__(self, builder: Builder, **kwargs):
242
        """
243
        Initialize reference builder.
244
        
245
        Args:
246
            builder: Target builder for the reference
247
        """
248
    
249
    @property
250
    def builder(self) -> Builder:
251
        """Target builder for the reference."""
252
```
253
254
### Build Management
255

256
Core management classes for coordinating the build process and type mappings.
257

258
```python { .api }
259
class BuildManager:
260
    """
261
    Manager for building containers into storage representations.
262
    
263
    Coordinates the conversion process from container objects to
264
    builder objects, handling validation, type mapping, and error reporting.
265
    """
266
    
267
    def __init__(self, type_map: 'TypeMap'):
268
        """
269
        Initialize build manager.
270
        
271
        Args:
272
            type_map: Type mapping for container-to-builder conversion
273
        """
274
    
275
    def build(self, container, source: str = None, **kwargs) -> Builder:
276
        """
277
        Build storage representation from container.
278
        
279
        Args:
280
            container: Container object to build
281
            source: Source identifier for tracking
282
            **kwargs: Build options:
283
                - root: Whether this is the root container
284
                - exhaust_dci: Whether to exhaust data chunk iterators
285
        
286
        Returns:
287
            Builder object representing the container
288
        """
289
    
290
    def construct(self, builder: Builder, **kwargs):
291
        """
292
        Construct container from storage representation.
293
        
294
        Args:
295
            builder: Builder object to construct from
296
            **kwargs: Construction options
297
            
298
        Returns:
299
            Container object constructed from builder
300
        """
301
    
302
    def get_builder(self, container) -> Builder:
303
        """
304
        Get existing builder for container if available.
305
        
306
        Args:
307
            container: Container to find builder for
308
            
309
        Returns:
310
            Builder object or None
311
        """
312

313
class TypeMap:
314
    """
315
    Mapping between specifications, container classes, and object mappers.
316
    
317
    Manages the relationships between data type specifications,
318
    Python container classes, and the mappers that convert between them.
319
    """
320
    
321
    def __init__(self, namespaces: 'NamespaceCatalog', type_map=None):
322
        """
323
        Initialize type map.
324
        
325
        Args:
326
            namespaces: Catalog of available namespaces
327
            type_map: Parent type map for inheritance
328
        """
329
    
330
    def register_container_type(self, namespace: str, data_type: str, container_cls):
331
        """
332
        Register container class for a data type.
333
        
334
        Args:
335
            namespace: Namespace containing the data type
336
            data_type: Name of the data type
337
            container_cls: Python class for the container
338
        """
339
    
340
    def register_map(self, container_cls, mapper_cls):
341
        """
342
        Register object mapper for a container class.
343
        
344
        Args:
345
            container_cls: Container class
346
            mapper_cls: Mapper class for serialization
347
        """
348
    
349
    def get_container_cls(self, namespace: str, data_type: str):
350
        """
351
        Get container class for a data type.
352
        
353
        Args:
354
            namespace: Namespace containing the data type
355
            data_type: Name of the data type
356
            
357
        Returns:
358
            Container class
359
        """
360
    
361
    def get_map(self, container):
362
        """
363
        Get object mapper for a container.
364
        
365
        Args:
366
            container: Container object
367
            
368
        Returns:
369
            ObjectMapper instance
370
        """
371
    
372
    def copy(self) -> 'TypeMap':
373
        """
374
        Create a copy of this type map.
375
        
376
        Returns:
377
            New TypeMap instance with same mappings
378
        """
379
```
380

381
### Object Mapping
382

383
Classes for mapping between container objects and builder representations.
384

385
```python { .api }
386
class ObjectMapper:
387
    """
388
    Base class for mapping container objects to/from builders.
389
    
390
    Provides the interface for converting between container objects
391
    and their storage representations with validation and error handling.
392
    """
393
    
394
    def __init__(self, spec):
395
        """
396
        Initialize object mapper.
397
        
398
        Args:
399
            spec: Specification for the data type
400
        """
401
    
402
    def build(self, container, manager: BuildManager, **kwargs) -> Builder:
403
        """
404
        Build storage representation from container.
405
        
406
        Args:
407
            container: Container object to build
408
            manager: Build manager coordinating the process
409
            
410
        Returns:
411
            Builder object
412
        """
413
    
414
    def construct(self, builder: Builder, manager: BuildManager, **kwargs):
415
        """
416
        Construct container from storage representation.
417
        
418
        Args:
419
            builder: Builder object to construct from
420
            manager: Build manager coordinating the process
421
            
422
        Returns:
423
            Container object
424
        """
425
    
426
    @property
427
    def spec(self):
428
        """Specification for this mapper."""
429

430
class CustomClassGenerator:
431
    """
432
    Generator for creating custom container classes from specifications.
433
    
434
    Dynamically creates Python classes that match specification requirements
435
    with proper validation, attributes, and methods.
436
    """
437
    
438
    def __init__(self, **kwargs):
439
        """Initialize custom class generator."""
440
    
441
    def generate_class(self, namespace: str, data_type: str, spec, parent_cls=None):
442
        """
443
        Generate custom container class from specification.
444
        
445
        Args:
446
            namespace: Namespace containing the specification
447
            data_type: Name of the data type
448
            spec: Specification object
449
            parent_cls: Parent class for inheritance
450
            
451
        Returns:
452
            Generated container class
453
        """
454

455
class MCIClassGenerator(CustomClassGenerator):
456
    """
457
    Generator for Multi-Container Interface classes.
458
    
459
    Creates classes that can hold multiple containers of the same type
460
    with auto-generated methods for access and manipulation.
461
    """
462
    
463
    def generate_class(self, namespace: str, data_type: str, spec, parent_cls=None):
464
        """
465
        Generate MCI container class from specification.
466
        
467
        Args:
468
            namespace: Namespace containing the specification
469
            data_type: Name of the data type
470
            spec: Specification object
471
            parent_cls: Parent class for inheritance
472
            
473
        Returns:
474
            Generated MCI container class
475
        """
476
```
477

478
### Build Errors and Warnings
479

480
Exception and warning classes for build process error handling.
481

482
```python { .api }
483
class BuildError(Exception):
484
    """Base exception for build process errors."""
485
    pass
486

487
class OrphanContainerBuildError(BuildError):
488
    """Exception raised when trying to build orphaned containers."""
489
    
490
    def __init__(self, container):
491
        """
492
        Initialize with orphaned container.
493
        
494
        Args:
495
            container: Container that lacks proper parent relationships
496
        """
497

498
class ReferenceTargetNotBuiltError(BuildError):
499
    """Exception raised when reference target hasn't been built yet."""
500
    
501
    def __init__(self, reference, target):
502
        """
503
        Initialize with reference details.
504
        
505
        Args:
506
            reference: Reference that failed
507
            target: Target that hasn't been built
508
        """
509

510
class ContainerConfigurationError(BuildError):
511
    """Exception raised for container configuration problems."""
512
    pass
513

514
class ConstructError(Exception):
515
    """Exception raised during container construction from builders."""
516
    pass
517

518
# Warning classes
519
class BuildWarning(UserWarning):
520
    """Base warning for build process issues."""
521
    pass
522

523
class MissingRequiredBuildWarning(BuildWarning):
524
    """Warning for missing required fields during build."""
525
    pass
526

527
class DtypeConversionWarning(BuildWarning):
528
    """Warning for data type conversions during build."""
529
    pass
530
```
531

532
## Usage Examples
533

534
### Basic Building and Construction
535

536
```python
537
from hdmf.build import BuildManager, TypeMap
538
from hdmf.common import get_type_map
539
from hdmf import Container, Data
540
import numpy as np
541

542
# Get type map with HDMF common types
543
type_map = get_type_map()
544
build_manager = BuildManager(type_map)
545

546
# Create container structure
547
data_array = np.random.randn(100, 50)
548
data_container = Data(name='neural_data', data=data_array)
549

550
container = Container(name='experiment')
551
container.add_child(data_container)
552

553
# Build storage representation
554
builder = build_manager.build(container)
555
print(f"Built {builder.name} with {len(builder.datasets)} datasets")
556

557
# Construct container back from builder
558
constructed = build_manager.construct(builder)
559
print(f"Constructed: {constructed.name}")
560
```
561

562
### Custom Type Registration
563

564
```python
565
from hdmf.build import TypeMap, ObjectMapper
566
from hdmf.spec import GroupSpec, AttributeSpec
567
from hdmf import Container
568
from hdmf.common import get_manager
569

570
# Define custom container class
571
class ExperimentSession(Container):
572
    
573
    @docval({'name': 'name', 'type': str, 'doc': 'Name of the session'},
574
            {'name': 'session_id', 'type': str, 'doc': 'Unique session identifier'},
575
            {'name': 'start_time', 'type': str, 'doc': 'Session start time'})
576
    def __init__(self, **kwargs):
577
        name, session_id, start_time = getargs('name', 'session_id', 'start_time', kwargs)
578
        super().__init__(name=name)
579
        self.session_id = session_id
580
        self.start_time = start_time
581

582
# Create specification for the custom type
583
session_spec = GroupSpec(
584
    doc='Container for experimental session data',
585
    neurodata_type_def='ExperimentSession',
586
    attributes=[
587
        AttributeSpec('session_id', 'Unique session identifier', dtype='text'),
588
        AttributeSpec('start_time', 'Session start time', dtype='text')
589
    ]
590
)
591

592
# Create custom mapper
593
class ExperimentSessionMapper(ObjectMapper):
594
    
595
    def __init__(self, spec):
596
        super().__init__(spec)
597
    
598
    def construct(self, builder, manager, **kwargs):
599
        return ExperimentSession(
600
            name=builder.name,
601
            session_id=builder.attributes['session_id'],
602
            start_time=builder.attributes['start_time']
603
        )
604

605
# Register the custom type
606
type_map = get_type_map()
607
type_map.register_container_type('custom', 'ExperimentSession', ExperimentSession)
608
type_map.register_map(ExperimentSession, ExperimentSessionMapper)
609
```
610

611
### Advanced Builder Manipulation
612

613
```python
614
from hdmf.build import GroupBuilder, DatasetBuilder, LinkBuilder
615
import numpy as np
616

617
# Create dataset builder with specific storage properties
618
data_builder = DatasetBuilder(
619
    name='high_res_data',
620
    data=np.random.randn(10000, 1000),
621
    dtype='float64',
622
    chunks=(1000, 100),  # Optimized chunk size
623
    compression='gzip',
624
    compression_opts=9
625
)
626

627
# Create group builder with nested structure
628
analysis_group = GroupBuilder(name='analysis')
629
analysis_group.set_dataset(data_builder)
630

631
# Add metadata attributes
632
analysis_group.set_attribute('analysis_type', 'spike_detection')
633
analysis_group.set_attribute('algorithm_version', '2.1.0')
634
analysis_group.set_attribute('parameters', {
635
    'threshold': -50.0,
636
    'min_interval': 0.001,
637
    'detection_method': 'threshold_crossing'
638
})
639

640
# Create main experiment group
641
experiment_group = GroupBuilder(name='experiment_001')
642
experiment_group.set_group(analysis_group)
643

644
# Create link to reference data from multiple locations
645
data_link = LinkBuilder(data_builder, name='reference_data')
646
experiment_group.set_link(data_link)
647

648
print(f"Built experiment with {len(experiment_group.groups)} groups")
649
print(f"Dataset shape: {data_builder.shape}")
650
```
651

652
### Error Handling and Validation
653

654
```python
655
from hdmf.build import (BuildManager, BuildError, OrphanContainerBuildError,
656
                        ReferenceTargetNotBuiltError)
657
from hdmf import Container
658

659
def safe_build_container(container, build_manager):
660
    """Safely build container with comprehensive error handling."""
661
    
662
    try:
663
        builder = build_manager.build(container)
664
        return builder
665
        
666
    except OrphanContainerBuildError as e:
667
        print(f"Container {e.args[0].name} is orphaned - no parent relationship")
668
        # Fix by adding to a parent or marking as root
669
        return build_manager.build(container, root=True)
670
        
671
    except ReferenceTargetNotBuiltError as e:
672
        print(f"Reference target not built: {e.args[1]}")
673
        # Build target first, then retry
674
        target_builder = build_manager.build(e.args[1])
675
        return build_manager.build(container)
676
        
677
    except BuildError as e:
678
        print(f"Build error: {e}")
679
        return None
680

681
# Usage
682
container = Container(name='test_container')
683
type_map = get_type_map()
684
build_manager = BuildManager(type_map)
685

686
builder = safe_build_container(container, build_manager)
687
if builder:
688
    print("Successfully built container")
689
```
690

691
### Dynamic Class Generation
692

693
```python
694
from hdmf.build import CustomClassGenerator
695
from hdmf.spec import GroupSpec, DatasetSpec, AttributeSpec
696
from hdmf.utils import docval, getargs
697

698
# Create specification for dynamic class
699
recording_spec = GroupSpec(
700
    doc='Neural recording container',
701
    neurodata_type_def='NeuralRecording',
702
    datasets=[
703
        DatasetSpec('data', 'Raw recording data', dtype='int16', 
704
                   shape=(None, None), dims=['time', 'channels']),
705
        DatasetSpec('timestamps', 'Sample timestamps', dtype='float64',
706
                   shape=(None,), dims=['time'])
707
    ],
708
    attributes=[
709
        AttributeSpec('sampling_rate', 'Sampling rate in Hz', dtype='float64'),
710
        AttributeSpec('num_channels', 'Number of recording channels', dtype='int')
711
    ]
712
)
713

714
# Generate custom class from specification
715
generator = CustomClassGenerator()
716
NeuralRecording = generator.generate_class(
717
    namespace='custom',
718
    data_type='NeuralRecording', 
719
    spec=recording_spec,
720
    parent_cls=Container
721
)
722

723
# Use the dynamically generated class
724
recording = NeuralRecording(
725
    name='session_001',
726
    data=np.random.randint(-1000, 1000, (30000, 64)),
727
    timestamps=np.arange(30000) / 30000.0,
728
    sampling_rate=30000.0,
729
    num_channels=64
730
)
731

732
print(f"Created {recording.__class__.__name__} with {recording.num_channels} channels")
733
```
734

735
### Build Process Monitoring
736

737
```python
738
import warnings
739
from hdmf.build import BuildWarning, MissingRequiredBuildWarning, DtypeConversionWarning
740

741
def build_with_monitoring(container, build_manager):
742
    """Build container with detailed monitoring and warning handling."""
743
    
744
    with warnings.catch_warnings(record=True) as warning_list:
745
        warnings.simplefilter("always")
746
        
747
        builder = build_manager.build(container)
748
        
749
        # Process warnings
750
        for warning in warning_list:
751
            if issubclass(warning.category, MissingRequiredBuildWarning):
752
                print(f"Missing required field: {warning.message}")
753
            elif issubclass(warning.category, DtypeConversionWarning):
754
                print(f"Data type conversion: {warning.message}")
755
            elif issubclass(warning.category, BuildWarning):
756
                print(f"Build warning: {warning.message}")
757
        
758
        return builder
759

760
# Usage
761
container = Container(name='monitored_build')
762
type_map = get_type_map()
763
build_manager = BuildManager(type_map)
764

765
builder = build_with_monitoring(container, build_manager)
766
print(f"Build completed for: {builder.name}")
767
```