0
# Visualization and Analysis
1

2
Visualization framework supporting multiple backends and model analysis tools for computing FLOPs, activation memory, and parameter counts with comprehensive reporting capabilities. The system provides tools for monitoring training progress and analyzing model complexity.
3

4
## Capabilities
5

6
### Visualizer Class
7

8
Main visualization class for drawing and displaying various visual elements.
9

10
```python { .api }
11
class Visualizer:
12
    def __init__(self, name: str = 'visualizer', image: np.ndarray = None, vis_backends: list = None, save_dir: str = None, bbox_color: str = 'green', text_color: str = 'green', mask_color: str = 'green', line_width: int = 3, alpha: float = 0.8):
13
        """
14
        Initialize visualizer with display and backend configuration.
15
        
16
        Parameters:
17
        - name: Visualizer name
18
        - image: Background image for visualization
19
        - vis_backends: List of visualization backends
20
        - save_dir: Directory to save visualizations
21
        - bbox_color: Default bounding box color
22
        - text_color: Default text color
23
        - mask_color: Default mask color
24
        - line_width: Default line width
25
        - alpha: Default transparency
26
        """
27

28
    def add_datasample(self, name: str, image: np.ndarray, data_sample = None, draw_gt: bool = True, draw_pred: bool = True, show: bool = False, wait_time: float = 0, out_file: str = None, pred_score_thr: float = 0.3, step: int = 0):
29
        """
30
        Add data sample visualization.
31
        
32
        Parameters:
33
        - name: Sample name
34
        - image: Input image
35
        - data_sample: Data sample with annotations/predictions
36
        - draw_gt: Whether to draw ground truth
37
        - draw_pred: Whether to draw predictions
38
        - show: Whether to display visualization
39
        - wait_time: Wait time for display
40
        - out_file: Output file path
41
        - pred_score_thr: Prediction score threshold
42
        - step: Step number for logging
43
        """
44

45
    def draw_bboxes(self, bboxes: np.ndarray, edge_colors: str = 'green', line_styles: str = '-', line_widths: int = 2, face_colors: str = 'none', alpha: float = 0.8):
46
        """
47
        Draw bounding boxes.
48
        
49
        Parameters:
50
        - bboxes: Bounding box coordinates
51
        - edge_colors: Edge colors
52
        - line_styles: Line styles
53
        - line_widths: Line widths
54
        - face_colors: Face colors
55
        - alpha: Transparency
56
        """
57

58
    def draw_points(self, positions: np.ndarray, colors: str = 'red', marker: str = 'o', sizes: int = 6, alpha: float = 0.8):
59
        """
60
        Draw points.
61
        
62
        Parameters:
63
        - positions: Point positions
64
        - colors: Point colors
65
        - marker: Marker style
66
        - sizes: Point sizes
67
        - alpha: Transparency
68
        """
69

70
    def draw_lines(self, x_datas: np.ndarray, y_datas: np.ndarray, colors: str = 'blue', line_styles: str = '-', line_widths: int = 1):
71
        """
72
        Draw lines.
73
        
74
        Parameters:
75
        - x_datas: X coordinates
76
        - y_datas: Y coordinates
77
        - colors: Line colors
78
        - line_styles: Line styles
79
        - line_widths: Line widths
80
        """
81

82
    def draw_texts(self, texts: list, positions: np.ndarray, font_sizes: int = 8, colors: str = 'black', vertical_alignments: str = 'bottom', horizontal_alignments: str = 'left'):
83
        """
84
        Draw text annotations.
85
        
86
        Parameters:
87
        - texts: Text strings
88
        - positions: Text positions
89
        - font_sizes: Font sizes
90
        - colors: Text colors
91
        - vertical_alignments: Vertical alignments
92
        - horizontal_alignments: Horizontal alignments
93
        """
94

95
    def draw_polygons(self, polygons: list, edge_colors: str = 'blue', line_styles: str = '-', line_widths: int = 1, face_colors: str = 'none', alpha: float = 0.8):
96
        """
97
        Draw polygons.
98
        
99
        Parameters:
100
        - polygons: Polygon coordinates
101
        - edge_colors: Edge colors
102
        - line_styles: Line styles
103
        - line_widths: Line widths
104
        - face_colors: Face colors
105
        - alpha: Transparency
106
        """
107

108
    def get_image(self) -> np.ndarray:
109
        """
110
        Get current visualization image.
111
        
112
        Returns:
113
        Visualization image as numpy array
114
        """
115

116
    def show(self, drawn_img: np.ndarray = None, win_name: str = 'image', wait_time: int = 0):
117
        """
118
        Display visualization.
119
        
120
        Parameters:
121
        - drawn_img: Image to display
122
        - win_name: Window name
123
        - wait_time: Wait time in milliseconds
124
        """
125

126
    @property
127
    def dataset_meta(self) -> dict:
128
        """Get dataset meta information."""
129
```
130

131
### Visualization Backends
132

133
Backend classes for different visualization and logging platforms.
134

135
```python { .api }
136
class BaseVisBackend:
137
    def __init__(self, save_dir: str = None):
138
        """
139
        Base visualization backend.
140
        
141
        Parameters:
142
        - save_dir: Directory to save outputs
143
        """
144

145
    def add_config(self, config: dict, **kwargs):
146
        """Add configuration to backend."""
147

148
    def add_image(self, name: str, image: np.ndarray, step: int = 0, **kwargs):
149
        """Add image to backend."""
150

151
    def add_scalar(self, name: str, value: float, step: int = 0, **kwargs):
152
        """Add scalar value to backend."""
153

154
    def add_scalars(self, scalar_dict: dict, step: int = 0, **kwargs):
155
        """Add multiple scalars to backend."""
156

157
class LocalVisBackend(BaseVisBackend):
158
    def __init__(self, save_dir: str = 'vis_data'):
159
        """
160
        Local filesystem visualization backend.
161
        
162
        Parameters:
163
        - save_dir: Local directory to save files
164
        """
165

166
class TensorboardVisBackend(BaseVisBackend):
167
    def __init__(self, save_dir: str = 'runs', **kwargs):
168
        """
169
        TensorBoard visualization backend.
170
        
171
        Parameters:
172
        - save_dir: TensorBoard log directory
173
        - **kwargs: Additional TensorBoard arguments
174
        """
175

176
class WandbVisBackend(BaseVisBackend):
177
    def __init__(self, save_dir: str = None, init_kwargs: dict = None, define_metric_cfg: dict = None, commit: bool = True, log_code_name: str = None, watch_kwargs: dict = None):
178
        """
179
        Weights & Biases visualization backend.
180
        
181
        Parameters:
182
        - save_dir: Save directory
183
        - init_kwargs: W&B initialization arguments
184
        - define_metric_cfg: Metric definition configuration
185
        - commit: Whether to commit logs immediately
186
        - log_code_name: Code logging name
187
        - watch_kwargs: Model watching arguments
188
        """
189

190
class MLflowVisBackend(BaseVisBackend):
191
    def __init__(self, save_dir: str = None, exp_name: str = None, run_name: str = None, tags: dict = None, params: dict = None, tracking_uri: str = None, artifact_suffix: tuple = ('.json', '.log', '.py', 'yaml')):
192
        """
193
        MLflow visualization backend.
194
        
195
        Parameters:
196
        - save_dir: Save directory
197
        - exp_name: Experiment name
198
        - run_name: Run name
199
        - tags: Run tags
200
        - params: Run parameters
201
        - tracking_uri: MLflow tracking URI
202
        - artifact_suffix: Artifact file suffixes to log
203
        """
204

205
class ClearMLVisBackend(BaseVisBackend):
206
    def __init__(self, save_dir: str = None, init_kwargs: dict = None, artifact_suffix: tuple = ('.json', '.log', '.py', 'yaml')):
207
        """
208
        ClearML visualization backend.
209
        
210
        Parameters:
211
        - save_dir: Save directory
212
        - init_kwargs: ClearML initialization arguments
213
        - artifact_suffix: Artifact file suffixes
214
        """
215

216
class NeptuneVisBackend(BaseVisBackend):
217
    def __init__(self, save_dir: str = None, init_kwargs: dict = None, artifact_suffix: tuple = ('.json', '.log', '.py', 'yaml')):
218
        """
219
        Neptune visualization backend.
220
        
221
        Parameters:
222
        - save_dir: Save directory
223
        - init_kwargs: Neptune initialization arguments
224
        - artifact_suffix: Artifact file suffixes
225
        """
226

227
class DVCLiveVisBackend(BaseVisBackend):
228
    def __init__(self, save_dir: str = None, init_kwargs: dict = None, artifact_suffix: tuple = ('.json', '.log', '.py', 'yaml')):
229
        """
230
        DVC Live visualization backend.
231
        
232
        Parameters:
233
        - save_dir: Save directory
234
        - init_kwargs: DVC Live initialization arguments
235
        - artifact_suffix: Artifact file suffixes
236
        """
237

238
class AimVisBackend(BaseVisBackend):
239
    def __init__(self, save_dir: str = None, init_kwargs: dict = None, artifact_suffix: tuple = ('.json', '.log', '.py', 'yaml')):
240
        """
241
        Aim visualization backend.
242
        
243
        Parameters:
244
        - save_dir: Save directory
245
        - init_kwargs: Aim initialization arguments
246
        - artifact_suffix: Artifact file suffixes
247
        """
248
```
249

250
### Model Analysis Tools
251

252
Tools for analyzing model complexity and performance characteristics.
253

254
```python { .api }
255
def get_model_complexity_info(model, input_shape: tuple, print_per_layer_stat: bool = True, as_strings: bool = True, input_constructor: callable = None, ost: any = None, verbose: bool = True, ignore_modules: list = None, custom_modules_hooks: dict = None):
256
    """
257
    Get comprehensive model complexity information.
258
    
259
    Parameters:
260
    - model: PyTorch model
261
    - input_shape: Input tensor shape
262
    - print_per_layer_stat: Whether to print per-layer statistics
263
    - as_strings: Whether to return formatted strings
264
    - input_constructor: Function to construct input
265
    - ost: Output stream for printing
266
    - verbose: Whether to print detailed information
267
    - ignore_modules: Modules to ignore in analysis
268
    - custom_modules_hooks: Custom hooks for specific modules
269
    
270
    Returns:
271
    Dictionary with FLOPs, parameters, and memory information
272
    """
273

274
def parameter_count(model) -> int:
275
    """
276
    Count total model parameters.
277
    
278
    Parameters:
279
    - model: PyTorch model
280
    
281
    Returns:
282
    Total number of parameters
283
    """
284

285
def parameter_count_table(model, max_depth: int = 3) -> str:
286
    """
287
    Generate parameter count table.
288
    
289
    Parameters:
290
    - model: PyTorch model
291
    - max_depth: Maximum depth for module hierarchy
292
    
293
    Returns:
294
    Formatted table string
295
    """
296

297
def flop_count(model, inputs: tuple, supported_ops: dict = None) -> int:
298
    """
299
    Count FLOPs of model.
300
    
301
    Parameters:
302
    - model: PyTorch model
303
    - inputs: Input tensors
304
    - supported_ops: Dictionary of supported operations
305
    
306
    Returns:
307
    Total FLOPs count
308
    """
309

310
def activation_count(model, inputs: tuple) -> int:
311
    """
312
    Count activation memory.
313
    
314
    Parameters:
315
    - model: PyTorch model
316
    - inputs: Input tensors
317
    
318
    Returns:
319
    Activation memory in bytes
320
    """
321
```
322

323
### Analysis Classes
324

325
Specialized analyzer classes for detailed model analysis.
326

327
```python { .api }
328
class FlopAnalyzer:
329
    def __init__(self, model, inputs: tuple):
330
        """
331
        Analyzer for FLOPs computation.
332
        
333
        Parameters:
334
        - model: PyTorch model
335
        - inputs: Input tensors
336
        """
337

338
    def total(self) -> int:
339
        """
340
        Get total FLOPs.
341
        
342
        Returns:
343
        Total FLOPs count
344
        """
345

346
    def by_module(self) -> dict:
347
        """
348
        Get FLOPs by module.
349
        
350
        Returns:
351
        Dictionary of module FLOPs
352
        """
353

354
    def by_operator(self) -> dict:
355
        """
356
        Get FLOPs by operator type.
357
        
358
        Returns:
359
        Dictionary of operator FLOPs
360
        """
361

362
class ActivationAnalyzer:
363
    def __init__(self, model, inputs: tuple):
364
        """
365
        Analyzer for activation memory.
366
        
367
        Parameters:
368
        - model: PyTorch model
369
        - inputs: Input tensors
370
        """
371

372
    def total(self) -> int:
373
        """
374
        Get total activation memory.
375
        
376
        Returns:
377
        Total activation memory in bytes
378
        """
379

380
    def by_module(self) -> dict:
381
        """
382
        Get activation memory by module.
383
        
384
        Returns:
385
        Dictionary of module activation memory
386
        """
387
```
388

389
## Usage Examples
390

391
### Basic Visualization Setup
392

393
```python
394
from mmengine.visualization import Visualizer
395
import numpy as np
396

397
# Create visualizer with multiple backends
398
visualizer = Visualizer(
399
    name='my_visualizer',
400
    vis_backends=[
401
        dict(type='LocalVisBackend', save_dir='./vis_outputs'),
402
        dict(type='TensorboardVisBackend', save_dir='./tb_logs'),
403
        dict(type='WandbVisBackend', init_kwargs=dict(project='my_project'))
404
    ]
405
)
406

407
# Visualize data sample
408
image = np.random.randint(0, 255, (224, 224, 3), dtype=np.uint8)
409
visualizer.add_datasample(
410
    name='sample_1',
411
    image=image,
412
    show=True,
413
    out_file='sample_1.jpg'
414
)
415
```
416

417
### Drawing Annotations
418

419
```python
420
# Draw bounding boxes
421
bboxes = np.array([[10, 10, 100, 100], [50, 50, 150, 150]])
422
visualizer.draw_bboxes(
423
    bboxes=bboxes,
424
    edge_colors=['red', 'blue'],
425
    line_widths=2,
426
    alpha=0.7
427
)
428

429
# Draw points
430
points = np.array([[30, 30], [70, 70], [120, 120]])
431
visualizer.draw_points(
432
    positions=points,
433
    colors=['red', 'green', 'blue'],
434
    sizes=10
435
)
436

437
# Draw text annotations
438
texts = ['Object 1', 'Object 2']
439
positions = np.array([[10, 5], [50, 45]])
440
visualizer.draw_texts(
441
    texts=texts,
442
    positions=positions,
443
    font_sizes=12,
444
    colors='white'
445
)
446

447
# Get final image
448
result_image = visualizer.get_image()
449
```
450

451
### Model Complexity Analysis
452

453
```python
454
import torch
455
from mmengine.analysis import get_model_complexity_info, parameter_count_table
456

457
# Create model
458
model = torch.nn.Sequential(
459
    torch.nn.Conv2d(3, 64, 3, padding=1),
460
    torch.nn.ReLU(),
461
    torch.nn.AdaptiveAvgPool2d(1),
462
    torch.nn.Flatten(),
463
    torch.nn.Linear(64, 10)
464
)
465

466
# Get complexity information
467
complexity_info = get_model_complexity_info(
468
    model=model,
469
    input_shape=(3, 224, 224),
470
    print_per_layer_stat=True,
471
    as_strings=True
472
)
473

474
print(f"FLOPs: {complexity_info['flops_str']}")
475
print(f"Parameters: {complexity_info['params_str']}")
476
print(f"Activations: {complexity_info['activations_str']}")
477

478
# Generate parameter table
479
param_table = parameter_count_table(model, max_depth=2)
480
print(param_table)
481
```
482

483
### Training Visualization Integration
484

485
```python
486
from mmengine import Runner
487
from mmengine.visualization import Visualizer
488

489
# Configure visualizer for training
490
visualizer_cfg = dict(
491
    type='Visualizer',
492
    vis_backends=[
493
        dict(type='LocalVisBackend'),
494
        dict(type='TensorboardVisBackend'),
495
        dict(
496
            type='WandbVisBackend',
497
            init_kwargs=dict(
498
                project='my_training',
499
                name='experiment_1',
500
                tags=['baseline', 'resnet50']
501
            )
502
        )
503
    ]
504
)
505

506
# Use in runner
507
runner = Runner(
508
    model=model,
509
    visualizer=visualizer_cfg,
510
    # ... other configs
511
)
512

513
# Custom visualization hook
514
class VisualizationHook(Hook):
515
    def __init__(self, interval=100):
516
        self.interval = interval
517
    
518
    def after_train_iter(self, runner):
519
        if runner.iter % self.interval == 0:
520
            # Log training images
521
            data_batch = runner.data_batch
522
            runner.visualizer.add_datasample(
523
                name=f'train_iter_{runner.iter}',
524
                image=data_batch['inputs'][0].cpu().numpy(),
525
                step=runner.iter
526
            )
527

528
runner.register_hook(VisualizationHook(interval=500))
529
```
530

531
### Custom Visualization Backend
532

533
```python
534
from mmengine.visualization import BaseVisBackend
535

536
class CustomVisBackend(BaseVisBackend):
537
    def __init__(self, api_key, project_id, **kwargs):
538
        super().__init__(**kwargs)
539
        self.api_key = api_key
540
        self.project_id = project_id
541
        self.client = self._init_client()
542
    
543
    def _init_client(self):
544
        # Initialize custom logging client
545
        return CustomLoggingClient(self.api_key, self.project_id)
546
    
547
    def add_scalar(self, name, value, step=0, **kwargs):
548
        self.client.log_metric(name, value, step)
549
    
550
    def add_image(self, name, image, step=0, **kwargs):
551
        self.client.log_image(name, image, step)
552

553
# Use custom backend
554
visualizer = Visualizer(
555
    vis_backends=[
556
        dict(type='CustomVisBackend', api_key='xxx', project_id='yyy')
557
    ]
558
)
559
```
560

561
### Detailed Model Analysis
562

563
```python
564
from mmengine.analysis import FlopAnalyzer, ActivationAnalyzer
565

566
# Create analyzers
567
inputs = (torch.randn(1, 3, 224, 224),)
568
flop_analyzer = FlopAnalyzer(model, inputs)
569
activation_analyzer = ActivationAnalyzer(model, inputs)
570

571
# Get detailed analysis
572
total_flops = flop_analyzer.total()
573
flops_by_module = flop_analyzer.by_module()
574
flops_by_operator = flop_analyzer.by_operator()
575

576
total_activations = activation_analyzer.total()
577
activations_by_module = activation_analyzer.by_module()
578

579
print(f"Total FLOPs: {total_flops:,}")
580
print(f"Total Activations: {total_activations:,} bytes")
581

582
# Print top modules by FLOPs
583
sorted_modules = sorted(flops_by_module.items(), 
584
                       key=lambda x: x[1], reverse=True)
585
print("\nTop modules by FLOPs:")
586
for module_name, flops in sorted_modules[:5]:
587
    print(f"  {module_name}: {flops:,}")
588
```
589

590
### Visualization with Multiple Data Types
591

592
```python
593
# Visualize different data types
594
for i, (image, target, prediction) in enumerate(dataloader):
595
    if i >= 5:  # Only visualize first 5 samples
596
        break
597
    
598
    visualizer.add_datasample(
599
        name=f'sample_{i}',
600
        image=image[0].permute(1, 2, 0).cpu().numpy(),
601
        data_sample={
602
            'gt_label': target[0].item(),
603
            'pred_label': prediction[0].argmax().item(),
604
            'pred_score': prediction[0].softmax(0).max().item()
605
        },
606
        step=i
607
    )
608
```