0
# Storage Systems
1

2
The storage system provides various backends for persisting and retrieving training statistics collected by DeepLearning4J. Different storage implementations offer trade-offs between performance, persistence, and memory usage.
3

4
## Core Storage Interface
5

6
### StatsStorage
7

8
The main interface for statistics storage systems.
9

10
```java { .api }
11
public interface StatsStorage extends StatsStorageRouter {
12
    // Lifecycle management
13
    void close() throws IOException;
14
    boolean isClosed();
15
    
16
    // Session management
17
    List<String> listSessionIDs();
18
    boolean sessionExists(String sessionID);
19
    
20
    // Static information storage and retrieval
21
    void putStaticInfo(Persistable staticInfo);
22
    Persistable getStaticInfo(String sessionID, String typeID, String workerID);
23
    List<Persistable> getAllStaticInfos(String sessionID, String typeID);
24
    
25
    // Session structure queries
26
    List<String> listTypeIDsForSession(String sessionID);
27
    List<String> listWorkerIDsForSession(String sessionID);
28
    List<String> listWorkerIDsForSessionAndType(String sessionID, String typeID);
29
    
30
    // Update count queries
31
    int getNumUpdateRecordsFor(String sessionID);
32
    int getNumUpdateRecordsFor(String sessionID, String typeID, String workerID);
33
    
34
    // Update storage and retrieval
35
    void putUpdate(Persistable update);
36
    Persistable getLatestUpdate(String sessionID, String typeID, String workerID);
37
    Persistable getUpdate(String sessionID, String typeID, String workerID, long timestamp);
38
    List<Persistable> getLatestUpdateAllWorkers(String sessionID, String typeID);
39
    List<Persistable> getAllUpdatesAfter(String sessionID, String typeID, String workerID, long timestamp);
40
    List<Persistable> getAllUpdatesAfter(String sessionID, String typeID, long timestamp);
41
    
42
    // Storage metadata management
43
    void putStorageMetaData(StorageMetaData storageMetaData);
44
    StorageMetaData getStorageMetaData(String sessionID, String typeID);
45
    
46
    // Event listener management
47
    void registerStatsStorageListener(StatsStorageListener listener);
48
    void deregisterStatsStorageListener(StatsStorageListener listener);
49
    void removeAllListeners();
50
    List<StatsStorageListener> getListeners();
51
}
52
```
53

54
### StatsStorageRouter
55

56
Base interface for routing statistics data.
57

58
```java { .api }
59
public interface StatsStorageRouter {
60
    void putStaticInfo(Persistable staticInfo);
61
    void putUpdate(Persistable update);
62
    void putStorageMetaData(StorageMetaData storageMetaData);
63
}
64
```
65

66
## Storage Implementations
67

68
### InMemoryStatsStorage
69

70
High-performance in-memory storage suitable for real-time monitoring and short training sessions.
71

72
```java { .api }
73
public class InMemoryStatsStorage extends BaseCollectionStatsStorage {
74
    // Default constructor
75
    public InMemoryStatsStorage();
76
    
77
    // Core storage operations
78
    public void putStaticInfo(Persistable staticInfo);
79
    public void putUpdate(Persistable update);
80
    public void putStorageMetaData(StorageMetaData storageMetaData);
81
    
82
    // Retrieval operations
83
    public Persistable getStaticInfo(String sessionID, String typeID, String workerID);
84
    public List<Persistable> getAllStaticInfos(String sessionID, String typeID);
85
    public Persistable getLatestUpdate(String sessionID, String typeID, String workerID);
86
}
87
```
88

89
### FileStatsStorage
90

91
File-based persistent storage for long-term statistics retention.
92

93
```java { .api }
94
public class FileStatsStorage implements StatsStorage {
95
    public FileStatsStorage(File file) throws IOException;
96
    public FileStatsStorage(File file, boolean readOnly) throws IOException;
97
    
98
    // Inherited storage operations from StatsStorage interface
99
    public void close();
100
}
101
```
102

103
### MapDBStatsStorage
104

105
MapDB-based persistent storage with good performance characteristics.
106

107
```java { .api }
108
public class MapDBStatsStorage implements StatsStorage {
109
    public MapDBStatsStorage();
110
    public MapDBStatsStorage(File file);
111
    public MapDBStatsStorage(File file, boolean readOnly);
112
    
113
    // MapDB-specific operations
114
    public void close();
115
    public boolean isClosed();
116
}
117
```
118

119
### J7FileStatsStorage
120

121
Java 7 compatible file-based storage using SQLite backend.
122

123
```java { .api }
124
public class J7FileStatsStorage implements StatsStorage {
125
    public J7FileStatsStorage(File file) throws SQLException;
126
    public J7FileStatsStorage(File file, boolean readOnly) throws SQLException;
127
    
128
    public void close();
129
}
130
```
131

132
## Base Storage Classes
133

134
### BaseCollectionStatsStorage
135

136
Abstract base class providing common functionality for collection-based storage implementations.
137

138
```java { .api }
139
public abstract class BaseCollectionStatsStorage implements StatsStorage {
140
    protected Set<String> sessionIDs;
141
    protected Map<SessionTypeWorkerId, StorageMetaData> storageMetaData;
142
    protected Map<SessionTypeWorkerId, Persistable> staticInfo;
143
    protected Map<SessionTypeWorkerId, Map<Long, Persistable>> updates;
144
    protected List<StatsStorageListener> listeners;
145
    
146
    // Template methods for subclasses
147
    protected abstract Map<Long, Persistable> getUpdateMap(String sessionID, String typeID, 
148
                                                          String workerID, boolean createIfRequired);
149
}
150
```
151

152
## Storage Metadata
153

154
### StorageMetaData
155

156
Interface for storage metadata containing information about stored sessions.
157

158
```java { .api }
159
public interface StorageMetaData extends Persistable {
160
    long getInitTime();
161
    String getSessionID(); 
162
    String getTypeID();
163
    String getWorkerID();
164
    Class<? extends StatsInitializationReport> getInitializationReportClass();
165
    Class<? extends StatsReport> getStatsReportClass();
166
}
167
```
168

169
### Implementation Classes
170

171
```java { .api }
172
// SBE-based metadata implementation
173
public class SbeStorageMetaData implements StorageMetaData {
174
    public SbeStorageMetaData(long initTime, String sessionID, String typeID, String workerID,
175
                              Class<? extends StatsInitializationReport> initReportClass,
176
                              Class<? extends StatsReport> updateReportClass);
177
}
178

179
// Java-based metadata implementation  
180
public class JavaStorageMetaData implements StorageMetaData {
181
    public JavaStorageMetaData(long initTime, String sessionID, String typeID, String workerID,
182
                               Class<? extends StatsInitializationReport> initReportClass,
183
                               Class<? extends StatsReport> updateReportClass);
184
}
185
```
186

187
## Event System
188

189
### StatsStorageListener
190

191
Interface for listening to storage events.
192

193
```java { .api }
194
public interface StatsStorageListener {
195
    enum EventType {
196
        PostStaticInfo, PostUpdate, PostMetaData
197
    }
198
    
199
    void notify(StatsStorageEvent event);
200
}
201
```
202

203
### StatsStorageEvent
204

205
Event object containing information about storage operations.
206

207
```java { .api }
208
public class StatsStorageEvent {
209
    public StatsStorageEvent(StatsStorage source, StatsStorageListener.EventType eventType,
210
                            String sessionID, String typeID, String workerID, long timestamp);
211
    
212
    public StatsStorage getSource();
213
    public StatsStorageListener.EventType getEventType();
214
    public String getSessionID();
215
    public String getTypeID(); 
216
    public String getWorkerID();
217
    public long getTimestamp();
218
}
219
```
220

221
### Listener Implementations
222

223
```java { .api }
224
// Queue-based listener for asynchronous processing
225
public class QueueStatsStorageListener implements StatsStorageListener {
226
    public QueueStatsStorageListener(BlockingQueue<StatsStorageEvent> queue);
227
    public void notify(StatsStorageEvent event);
228
}
229

230
// Pair queue listener for coordinated processing
231
public class QueuePairStatsStorageListener implements StatsStorageListener {
232
    public QueuePairStatsStorageListener(BlockingQueue<Pair<StatsStorageEvent, StatsStorage>> queue);
233
    public void notify(StatsStorageEvent event);
234
}
235
```
236

237
## Usage Examples
238

239
### Basic In-Memory Storage
240

241
```java
242
import org.deeplearning4j.ui.storage.InMemoryStatsStorage;
243
import org.deeplearning4j.ui.stats.StatsListener;
244
import org.deeplearning4j.ui.api.UIServer;
245

246
// Create in-memory storage
247
InMemoryStatsStorage storage = new InMemoryStatsStorage();
248

249
// Attach to UI server
250
UIServer uiServer = UIServer.getInstance();
251
uiServer.attach(storage);
252

253
// Use with stats listener
254
StatsListener listener = new StatsListener(storage);
255
model.setListeners(listener);
256

257
// Data is stored in memory and lost when application stops
258
```
259

260
### Persistent File Storage
261

262
```java
263
import org.deeplearning4j.ui.storage.FileStatsStorage;
264
import java.io.File;
265

266
// Create file-based storage
267
File statsFile = new File("training-stats.dl4j");
268
FileStatsStorage storage = new FileStatsStorage(statsFile);
269

270
// Use with listener
271
StatsListener listener = new StatsListener(storage);
272
model.setListeners(listener);
273

274
// Data persists between application runs
275
// Remember to close storage when done
276
storage.close();
277
```
278

279
### MapDB Storage for Better Performance
280

281
```java
282
import org.deeplearning4j.ui.storage.mapdb.MapDBStatsStorage;
283

284
// Create MapDB storage
285
File dbFile = new File("training-stats.mapdb");
286
MapDBStatsStorage storage = new MapDBStatsStorage(dbFile);
287

288
// Attach to UI and use with listener
289
UIServer.getInstance().attach(storage);
290
StatsListener listener = new StatsListener(storage);
291
model.setListeners(listener);
292

293
// Close when done
294
storage.close();
295
```
296

297
### Read-Only Storage Access
298

299
```java
300
// Open existing storage in read-only mode
301
FileStatsStorage readOnlyStorage = new FileStatsStorage(
302
    new File("existing-stats.dl4j"), true);
303

304
// Attach to UI for visualization only
305
UIServer.getInstance().attach(readOnlyStorage);
306

307
// Analyze stored sessions
308
List<String> sessions = readOnlyStorage.listSessionIDs();
309
for (String sessionID : sessions) {
310
    List<String> typeIDs = readOnlyStorage.listTypeIDsForSession(sessionID);
311
    System.out.println("Session " + sessionID + " has types: " + typeIDs);
312
}
313
```
314

315
### Storage Event Handling
316

317
```java
318
import org.deeplearning4j.ui.storage.impl.QueueStatsStorageListener;
319
import java.util.concurrent.BlockingQueue;
320
import java.util.concurrent.LinkedBlockingQueue;
321

322
// Create event queue
323
BlockingQueue<StatsStorageEvent> eventQueue = new LinkedBlockingQueue<>();
324

325
// Create and register listener
326
QueueStatsStorageListener eventListener = new QueueStatsStorageListener(eventQueue);
327
storage.registerStatsStorageListener(eventListener);
328

329
// Process events asynchronously
330
new Thread(() -> {
331
    while (true) {
332
        try {
333
            StatsStorageEvent event = eventQueue.take();
334
            System.out.println("Storage event: " + event.getEventType() + 
335
                             " for session " + event.getSessionID());
336
        } catch (InterruptedException e) {
337
            break;
338
        }
339
    }
340
}).start();
341
```
342

343
### Storage Migration
344

345
```java
346
// Copy data from one storage to another
347
InMemoryStatsStorage source = new InMemoryStatsStorage();
348
FileStatsStorage destination = new FileStatsStorage(new File("backup.dl4j"));
349

350
// Copy all sessions
351
for (String sessionID : source.listSessionIDs()) {
352
    for (String typeID : source.listTypeIDsForSession(sessionID)) {
353
        // Copy metadata
354
        StorageMetaData metadata = source.getStorageMetaData(sessionID, typeID);
355
        if (metadata != null) {
356
            destination.putStorageMetaData(metadata);
357
        }
358
        
359
        // Copy static info
360
        List<Persistable> staticInfos = source.getAllStaticInfos(sessionID, typeID);
361
        for (Persistable staticInfo : staticInfos) {
362
            destination.putStaticInfo(staticInfo);
363
        }
364
        
365
        // Copy updates for each worker
366
        for (String workerID : source.listWorkerIDsForSessionAndType(sessionID, typeID)) {
367
            List<Persistable> updates = source.getAllUpdatesAfter(sessionID, typeID, workerID, 0);
368
            for (Persistable update : updates) {
369
                destination.putUpdate(update);
370
            }
371
        }
372
    }
373
}
374

375
destination.close();
376
```