CtrlK
BlogDocsLog inGet started
Tessl Logo

tessl/maven-org-bytedeco--libfreenect2

JavaCPP Presets for libfreenect2 providing Java bindings for Kinect for Windows v2 device drivers

—

Pending
Overview
Eval results
Files

frame-processing.mddocs/

Frame Processing

Comprehensive frame handling system for RGB, depth, and infrared data streams from Kinect v2 devices. This module provides both synchronous and asynchronous processing options with support for multiple frame types and formats.

Capabilities

Frame Data Structure

Container for image frame data with metadata including timing, format, and sensor information.

/**
 * Frame format and metadata
 */
class Frame {
    // Frame type constants
    /** 1920x1080 BGRX or RGBX color frame */
    static final int Color = 1;
    /** 512x424 float IR frame, range [0.0, 65535.0] */
    static final int Ir = 2;
    /** 512x424 float depth frame, unit: millimeter */
    static final int Depth = 4;
    
    // Pixel format constants
    /** Invalid format */
    static final int Invalid = 0;
    /** Raw bitstream - bytes_per_pixel defines number of bytes */
    static final int Raw = 1;
    /** 4-byte float per pixel */
    static final int Float = 2;
    /** 4 bytes of B, G, R, and unused per pixel */
    static final int BGRX = 4;
    /** 4 bytes of R, G, B, and unused per pixel */
    static final int RGBX = 5;
    /** 1 byte of gray per pixel */
    static final int Gray = 6;
    
    /** Constructor for new frame */
    Frame(@Cast("size_t") long width, @Cast("size_t") long height, @Cast("size_t") long bytes_per_pixel);
    /** Constructor with existing data buffer */
    Frame(@Cast("size_t") long width, @Cast("size_t") long height, @Cast("size_t") long bytes_per_pixel, @Cast("unsigned char*") BytePointer data);
    
    /** Length of a line (in pixels) */
    @Cast("size_t") long width(); Frame width(long setter);
    /** Number of lines in the frame */
    @Cast("size_t") long height(); Frame height(long setter);
    /** Number of bytes in a pixel */
    @Cast("size_t") long bytes_per_pixel(); Frame bytes_per_pixel(long setter);
    /** Data of the frame (aligned) */
    @Cast("unsigned char*") BytePointer data(); Frame data(BytePointer setter);
    /** Unit: roughly or exactly 0.1 millisecond */
    @Cast("uint32_t") int timestamp(); Frame timestamp(int setter);
    /** Increasing frame sequence number */
    @Cast("uint32_t") int sequence(); Frame sequence(int setter);
    /** From 0.5 (very bright) to ~60.0 (fully covered) */
    float exposure(); Frame exposure(float setter);
    /** From 1.0 (bright) to 1.5 (covered) */
    float gain(); Frame gain(float setter);
    /** From 1.0 (bright) to 6.4 (covered) */
    float gamma(); Frame gamma(float setter);
    /** Zero if ok; non-zero for errors */
    @Cast("uint32_t") int status(); Frame status(int setter);
    /** Byte format - informative only, doesn't indicate errors */
    @Cast("libfreenect2::Frame::Format") int format(); Frame format(int setter);
}

Usage Examples:

// Check frame properties
System.out.println("Frame dimensions: " + frame.width() + "x" + frame.height());
System.out.println("Bytes per pixel: " + frame.bytes_per_pixel());
System.out.println("Timestamp: " + frame.timestamp());
System.out.println("Status: " + (frame.status() == 0 ? "OK" : "Error"));

// Access frame data
BytePointer frameData = frame.data();
byte[] buffer = new byte[1000];
frameData.get(buffer);

// Check frame type and format
if (frame.format() == Frame.BGRX) {
    System.out.println("Color frame in BGRX format");
} else if (frame.format() == Frame.Float) {
    System.out.println("Depth or IR frame in float format");
}

Frame Listener Interface

Base callback interface for receiving new frames from the device. Implement this for custom frame processing.

/**
 * Callback interface to receive new frames.
 * You can inherit from FrameListener and define your own listener.
 */
abstract class FrameListener {
    /**
     * libfreenect2 calls this function when a new frame is decoded.
     * @param type Type of the new frame (Frame.Color, Frame.Ir, Frame.Depth)
     * @param frame Data of the frame
     * @return true if you want to take ownership of the frame, 
     *         false to let caller reuse/delete it
     */
    @Cast("bool") abstract boolean onNewFrame(@Cast("libfreenect2::Frame::Type") int type, Frame frame);
}

Usage Examples:

// Custom frame listener implementation
class MyFrameListener extends FrameListener {
    @Override
    public boolean onNewFrame(int type, Frame frame) {
        switch (type) {
            case Frame.Color:
                System.out.println("Color frame: " + frame.width() + "x" + frame.height());
                break;
            case Frame.Depth:
                System.out.println("Depth frame at " + frame.timestamp());
                break;
            case Frame.Ir:
                System.out.println("IR frame, exposure: " + frame.exposure());
                break;
        }
        return false; // Let caller manage memory
    }
}

// Use custom listener
MyFrameListener listener = new MyFrameListener();
device.setColorFrameListener(listener);
device.setIrAndDepthFrameListener(listener);

Synchronous Multi-Frame Listener

Specialized frame listener that collects multiple frame types synchronously with blocking wait operations.

/**
 * Collect multiple types of frames synchronously
 */
class SyncMultiFrameListener extends FrameListener {
    /**
     * Constructor specifying frame types to collect
     * @param frame_types Use bitwise OR to combine multiple types, 
     *                   e.g. Frame.Ir | Frame.Depth
     */
    SyncMultiFrameListener(@Cast("unsigned int") int frame_types);
    
    /** Test if there are new frames available (non-blocking) */
    @Cast("bool") boolean hasNewFrame();
    
    /** 
     * Wait for new frames with timeout
     * @param frames Output container for received frames
     * @param milliseconds Timeout in milliseconds
     * @return true if frames received, false if timeout
     */
    @Cast("bool") boolean waitForNewFrame(@ByRef FrameMap frames, int milliseconds);
    
    /** 
     * Wait indefinitely for new frames
     * @param frames Output container for received frames  
     */
    void waitForNewFrame(@ByRef FrameMap frames);
    
    /** Shortcut to delete all frames in the map */
    void release(@ByRef FrameMap frames);
    
    /** Override from FrameListener - handles frame collection */
    @Cast("bool") boolean onNewFrame(@Cast("libfreenect2::Frame::Type") int type, Frame frame);
}

Usage Examples:

// Set up multi-frame listener for all frame types
int frameTypes = Frame.Color | Frame.Ir | Frame.Depth;
SyncMultiFrameListener listener = new SyncMultiFrameListener(frameTypes);

device.setColorFrameListener(listener);
device.setIrAndDepthFrameListener(listener);
device.start();

// Wait for frames with timeout
FrameMap frames = new FrameMap();
if (listener.waitForNewFrame(frames, 10000)) { // 10 second timeout
    Frame color = frames.get(Frame.Color);
    Frame depth = frames.get(Frame.Depth);
    Frame ir = frames.get(Frame.Ir);
    
    if (color != null) {
        System.out.println("Color: " + color.width() + "x" + color.height());
    }
    if (depth != null) {
        System.out.println("Depth: " + depth.width() + "x" + depth.height());
    }
    
    // Always release frames when done
    listener.release(frames);
} else {
    System.out.println("Frame timeout!");
}

// Non-blocking check
if (listener.hasNewFrame()) {
    listener.waitForNewFrame(frames);
    // Process frames...
    listener.release(frames);
}

Frame Map Container

Container class for managing multiple frame types, implemented as a wrapper around std::map.

/**
 * Container for multiple frame types (std::map wrapper)
 */
class FrameMap {
    /** Default constructor */
    FrameMap();
    
    /** Check if map contains no frames */
    boolean empty();
    
    /** Get number of frames in the map */
    long size();
    
    /** Get frame by type */
    Frame get(@Cast("libfreenect2::Frame::Type") int type);
    
    /** Store frame by type */
    FrameMap put(@Cast("libfreenect2::Frame::Type") int type, Frame value);
    
    /** Remove frame at iterator position */
    void erase(@ByVal Iterator pos);
    
    /** Get begin iterator */
    @ByVal Iterator begin();
    
    /** Get end iterator */
    @ByVal Iterator end();
    
    /**
     * Iterator for traversing frames in the map
     */
    static class Iterator {
        Iterator();
        
        /** Advance to next element */
        @Name("operator ++") @ByRef Iterator increment();
        
        /** Check equality with another iterator */
        @Name("operator ==") boolean equals(@ByRef Iterator it);
        
        /** Get frame type key */
        @Name("operator *().first") @MemberGetter @Cast("libfreenect2::Frame::Type") int first();
        
        /** Get frame value */
        @Name("operator *().second") @MemberGetter @Const Frame second();
    }
}

Usage Examples:

FrameMap frames = new FrameMap();

// After receiving frames from listener
if (!frames.empty()) {
    System.out.println("Received " + frames.size() + " frames");
    
    // Access specific frame types
    Frame colorFrame = frames.get(Frame.Color);
    Frame depthFrame = frames.get(Frame.Depth);
    
    // Check if specific frame type is available
    if (colorFrame != null) {
        System.out.println("Color frame available: " + 
                          colorFrame.width() + "x" + colorFrame.height());
    }
    
    // Iterate through all frames
    FrameMap.Iterator it = frames.begin();
    FrameMap.Iterator end = frames.end();
    while (!it.equals(end)) {
        int frameType = it.first();
        Frame frame = it.second();
        System.out.println("Frame type " + frameType + 
                          ": " + frame.width() + "x" + frame.height());
        it.increment();
    }
}

// Manual frame insertion
Frame customFrame = new Frame(640, 480, 4);
frames.put(Frame.Color, customFrame);

Frame Processing Patterns

Basic Frame Capture Loop

// Setup
SyncMultiFrameListener listener = new SyncMultiFrameListener(
    Frame.Color | Frame.Depth);
device.setColorFrameListener(listener);
device.setIrAndDepthFrameListener(listener);
device.start();

// Capture loop
FrameMap frames = new FrameMap();
for (int i = 0; i < 100; i++) {
    if (listener.waitForNewFrame(frames, 5000)) {
        Frame color = frames.get(Frame.Color);
        Frame depth = frames.get(Frame.Depth);
        
        // Process frames here
        System.out.println("Frame " + i + " captured");
        
        listener.release(frames);
    } else {
        System.err.println("Frame timeout");
        break;
    }
}

device.stop();
device._close();

Asynchronous Frame Processing

class AsyncFrameProcessor extends FrameListener {
    private volatile boolean running = true;
    
    @Override
    public boolean onNewFrame(int type, Frame frame) {
        if (!running) return false;
        
        // Process frame on callback thread
        switch (type) {
            case Frame.Color:
                processColorFrame(frame);
                break;
            case Frame.Depth:
                processDepthFrame(frame);
                break;
        }
        
        return false; // Don't take ownership
    }
    
    private void processColorFrame(Frame frame) {
        // Color processing logic
    }
    
    private void processDepthFrame(Frame frame) {
        // Depth processing logic
    }
    
    public void stop() {
        running = false;
    }
}

Install with Tessl CLI

npx tessl i tessl/maven-org-bytedeco--libfreenect2

docs

device-management.md

frame-processing.md

index.md

logging.md

pipeline-configuration.md

registration-geometry.md

tile.json