tessl/maven-org-deeplearning4j--deeplearning4j-ui-2-11

Web-based user interface module for DeepLearning4J that provides visualization and monitoring capabilities for deep learning model training and evaluation

Overview

Eval results

Files

Storage Systems

Name: tessl/maven-org-deeplearning4j--deeplearning4j-ui-2-11
Author: tessl

The storage system provides various backends for persisting and retrieving training statistics collected by DeepLearning4J. Different storage implementations offer trade-offs between performance, persistence, and memory usage.

Core Storage Interface

StatsStorage

The main interface for statistics storage systems.

public interface StatsStorage extends StatsStorageRouter {
    // Lifecycle management
    void close() throws IOException;
    boolean isClosed();
    
    // Session management
    List<String> listSessionIDs();
    boolean sessionExists(String sessionID);
    
    // Static information storage and retrieval
    void putStaticInfo(Persistable staticInfo);
    Persistable getStaticInfo(String sessionID, String typeID, String workerID);
    List<Persistable> getAllStaticInfos(String sessionID, String typeID);
    
    // Session structure queries
    List<String> listTypeIDsForSession(String sessionID);
    List<String> listWorkerIDsForSession(String sessionID);
    List<String> listWorkerIDsForSessionAndType(String sessionID, String typeID);
    
    // Update count queries
    int getNumUpdateRecordsFor(String sessionID);
    int getNumUpdateRecordsFor(String sessionID, String typeID, String workerID);
    
    // Update storage and retrieval
    void putUpdate(Persistable update);
    Persistable getLatestUpdate(String sessionID, String typeID, String workerID);
    Persistable getUpdate(String sessionID, String typeID, String workerID, long timestamp);
    List<Persistable> getLatestUpdateAllWorkers(String sessionID, String typeID);
    List<Persistable> getAllUpdatesAfter(String sessionID, String typeID, String workerID, long timestamp);
    List<Persistable> getAllUpdatesAfter(String sessionID, String typeID, long timestamp);
    
    // Storage metadata management
    void putStorageMetaData(StorageMetaData storageMetaData);
    StorageMetaData getStorageMetaData(String sessionID, String typeID);
    
    // Event listener management
    void registerStatsStorageListener(StatsStorageListener listener);
    void deregisterStatsStorageListener(StatsStorageListener listener);
    void removeAllListeners();
    List<StatsStorageListener> getListeners();
}

StatsStorageRouter

Base interface for routing statistics data.

public interface StatsStorageRouter {
    void putStaticInfo(Persistable staticInfo);
    void putUpdate(Persistable update);
    void putStorageMetaData(StorageMetaData storageMetaData);
}

Storage Implementations

InMemoryStatsStorage

High-performance in-memory storage suitable for real-time monitoring and short training sessions.

public class InMemoryStatsStorage extends BaseCollectionStatsStorage {
    // Default constructor
    public InMemoryStatsStorage();
    
    // Core storage operations
    public void putStaticInfo(Persistable staticInfo);
    public void putUpdate(Persistable update);
    public void putStorageMetaData(StorageMetaData storageMetaData);
    
    // Retrieval operations
    public Persistable getStaticInfo(String sessionID, String typeID, String workerID);
    public List<Persistable> getAllStaticInfos(String sessionID, String typeID);
    public Persistable getLatestUpdate(String sessionID, String typeID, String workerID);
}

FileStatsStorage

File-based persistent storage for long-term statistics retention.

public class FileStatsStorage implements StatsStorage {
    public FileStatsStorage(File file) throws IOException;
    public FileStatsStorage(File file, boolean readOnly) throws IOException;
    
    // Inherited storage operations from StatsStorage interface
    public void close();
}

MapDBStatsStorage

MapDB-based persistent storage with good performance characteristics.

public class MapDBStatsStorage implements StatsStorage {
    public MapDBStatsStorage();
    public MapDBStatsStorage(File file);
    public MapDBStatsStorage(File file, boolean readOnly);
    
    // MapDB-specific operations
    public void close();
    public boolean isClosed();
}

J7FileStatsStorage

Java 7 compatible file-based storage using SQLite backend.

public class J7FileStatsStorage implements StatsStorage {
    public J7FileStatsStorage(File file) throws SQLException;
    public J7FileStatsStorage(File file, boolean readOnly) throws SQLException;
    
    public void close();
}

Base Storage Classes

BaseCollectionStatsStorage

Abstract base class providing common functionality for collection-based storage implementations.

public abstract class BaseCollectionStatsStorage implements StatsStorage {
    protected Set<String> sessionIDs;
    protected Map<SessionTypeWorkerId, StorageMetaData> storageMetaData;
    protected Map<SessionTypeWorkerId, Persistable> staticInfo;
    protected Map<SessionTypeWorkerId, Map<Long, Persistable>> updates;
    protected List<StatsStorageListener> listeners;
    
    // Template methods for subclasses
    protected abstract Map<Long, Persistable> getUpdateMap(String sessionID, String typeID, 
                                                          String workerID, boolean createIfRequired);
}

Storage Metadata

StorageMetaData

Interface for storage metadata containing information about stored sessions.

public interface StorageMetaData extends Persistable {
    long getInitTime();
    String getSessionID(); 
    String getTypeID();
    String getWorkerID();
    Class<? extends StatsInitializationReport> getInitializationReportClass();
    Class<? extends StatsReport> getStatsReportClass();
}

Implementation Classes

// SBE-based metadata implementation
public class SbeStorageMetaData implements StorageMetaData {
    public SbeStorageMetaData(long initTime, String sessionID, String typeID, String workerID,
                              Class<? extends StatsInitializationReport> initReportClass,
                              Class<? extends StatsReport> updateReportClass);
}

// Java-based metadata implementation  
public class JavaStorageMetaData implements StorageMetaData {
    public JavaStorageMetaData(long initTime, String sessionID, String typeID, String workerID,
                               Class<? extends StatsInitializationReport> initReportClass,
                               Class<? extends StatsReport> updateReportClass);
}

Event System

StatsStorageListener

Interface for listening to storage events.

public interface StatsStorageListener {
    enum EventType {
        PostStaticInfo, PostUpdate, PostMetaData
    }
    
    void notify(StatsStorageEvent event);
}

StatsStorageEvent

Event object containing information about storage operations.

public class StatsStorageEvent {
    public StatsStorageEvent(StatsStorage source, StatsStorageListener.EventType eventType,
                            String sessionID, String typeID, String workerID, long timestamp);
    
    public StatsStorage getSource();
    public StatsStorageListener.EventType getEventType();
    public String getSessionID();
    public String getTypeID(); 
    public String getWorkerID();
    public long getTimestamp();
}

Listener Implementations

// Queue-based listener for asynchronous processing
public class QueueStatsStorageListener implements StatsStorageListener {
    public QueueStatsStorageListener(BlockingQueue<StatsStorageEvent> queue);
    public void notify(StatsStorageEvent event);
}

// Pair queue listener for coordinated processing
public class QueuePairStatsStorageListener implements StatsStorageListener {
    public QueuePairStatsStorageListener(BlockingQueue<Pair<StatsStorageEvent, StatsStorage>> queue);
    public void notify(StatsStorageEvent event);
}

Usage Examples

Basic In-Memory Storage

import org.deeplearning4j.ui.storage.InMemoryStatsStorage;
import org.deeplearning4j.ui.stats.StatsListener;
import org.deeplearning4j.ui.api.UIServer;

// Create in-memory storage
InMemoryStatsStorage storage = new InMemoryStatsStorage();

// Attach to UI server
UIServer uiServer = UIServer.getInstance();
uiServer.attach(storage);

// Use with stats listener
StatsListener listener = new StatsListener(storage);
model.setListeners(listener);

// Data is stored in memory and lost when application stops

Persistent File Storage

import org.deeplearning4j.ui.storage.FileStatsStorage;
import java.io.File;

// Create file-based storage
File statsFile = new File("training-stats.dl4j");
FileStatsStorage storage = new FileStatsStorage(statsFile);

// Use with listener
StatsListener listener = new StatsListener(storage);
model.setListeners(listener);

// Data persists between application runs
// Remember to close storage when done
storage.close();

MapDB Storage for Better Performance

import org.deeplearning4j.ui.storage.mapdb.MapDBStatsStorage;

// Create MapDB storage
File dbFile = new File("training-stats.mapdb");
MapDBStatsStorage storage = new MapDBStatsStorage(dbFile);

// Attach to UI and use with listener
UIServer.getInstance().attach(storage);
StatsListener listener = new StatsListener(storage);
model.setListeners(listener);

// Close when done
storage.close();

Read-Only Storage Access

// Open existing storage in read-only mode
FileStatsStorage readOnlyStorage = new FileStatsStorage(
    new File("existing-stats.dl4j"), true);

// Attach to UI for visualization only
UIServer.getInstance().attach(readOnlyStorage);

// Analyze stored sessions
List<String> sessions = readOnlyStorage.listSessionIDs();
for (String sessionID : sessions) {
    List<String> typeIDs = readOnlyStorage.listTypeIDsForSession(sessionID);
    System.out.println("Session " + sessionID + " has types: " + typeIDs);
}

Storage Event Handling

import org.deeplearning4j.ui.storage.impl.QueueStatsStorageListener;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue;

// Create event queue
BlockingQueue<StatsStorageEvent> eventQueue = new LinkedBlockingQueue<>();

// Create and register listener
QueueStatsStorageListener eventListener = new QueueStatsStorageListener(eventQueue);
storage.registerStatsStorageListener(eventListener);

// Process events asynchronously
new Thread(() -> {
    while (true) {
        try {
            StatsStorageEvent event = eventQueue.take();
            System.out.println("Storage event: " + event.getEventType() + 
                             " for session " + event.getSessionID());
        } catch (InterruptedException e) {
            break;
        }
    }
}).start();

Storage Migration

// Copy data from one storage to another
InMemoryStatsStorage source = new InMemoryStatsStorage();
FileStatsStorage destination = new FileStatsStorage(new File("backup.dl4j"));

// Copy all sessions
for (String sessionID : source.listSessionIDs()) {
    for (String typeID : source.listTypeIDsForSession(sessionID)) {
        // Copy metadata
        StorageMetaData metadata = source.getStorageMetaData(sessionID, typeID);
        if (metadata != null) {
            destination.putStorageMetaData(metadata);
        }
        
        // Copy static info
        List<Persistable> staticInfos = source.getAllStaticInfos(sessionID, typeID);
        for (Persistable staticInfo : staticInfos) {
            destination.putStaticInfo(staticInfo);
        }
        
        // Copy updates for each worker
        for (String workerID : source.listWorkerIDsForSessionAndType(sessionID, typeID)) {
            List<Persistable> updates = source.getAllUpdatesAfter(sessionID, typeID, workerID, 0);
            for (Persistable update : updates) {
                destination.putUpdate(update);
            }
        }
    }
}

destination.close();

Install with Tessl CLI

npx tessl i tessl/maven-org-deeplearning4j--deeplearning4j-ui-2-11

docs