0
# Frame Processing
1

2
Comprehensive frame handling system for RGB, depth, and infrared data streams from Kinect v2 devices. This module provides both synchronous and asynchronous processing options with support for multiple frame types and formats.
3

4
## Capabilities
5

6
### Frame Data Structure
7

8
Container for image frame data with metadata including timing, format, and sensor information.
9

10
```java { .api }
11
/**
12
 * Frame format and metadata
13
 */
14
class Frame {
15
    // Frame type constants
16
    /** 1920x1080 BGRX or RGBX color frame */
17
    static final int Color = 1;
18
    /** 512x424 float IR frame, range [0.0, 65535.0] */
19
    static final int Ir = 2;
20
    /** 512x424 float depth frame, unit: millimeter */
21
    static final int Depth = 4;
22
    
23
    // Pixel format constants
24
    /** Invalid format */
25
    static final int Invalid = 0;
26
    /** Raw bitstream - bytes_per_pixel defines number of bytes */
27
    static final int Raw = 1;
28
    /** 4-byte float per pixel */
29
    static final int Float = 2;
30
    /** 4 bytes of B, G, R, and unused per pixel */
31
    static final int BGRX = 4;
32
    /** 4 bytes of R, G, B, and unused per pixel */
33
    static final int RGBX = 5;
34
    /** 1 byte of gray per pixel */
35
    static final int Gray = 6;
36
    
37
    /** Constructor for new frame */
38
    Frame(@Cast("size_t") long width, @Cast("size_t") long height, @Cast("size_t") long bytes_per_pixel);
39
    /** Constructor with existing data buffer */
40
    Frame(@Cast("size_t") long width, @Cast("size_t") long height, @Cast("size_t") long bytes_per_pixel, @Cast("unsigned char*") BytePointer data);
41
    
42
    /** Length of a line (in pixels) */
43
    @Cast("size_t") long width(); Frame width(long setter);
44
    /** Number of lines in the frame */
45
    @Cast("size_t") long height(); Frame height(long setter);
46
    /** Number of bytes in a pixel */
47
    @Cast("size_t") long bytes_per_pixel(); Frame bytes_per_pixel(long setter);
48
    /** Data of the frame (aligned) */
49
    @Cast("unsigned char*") BytePointer data(); Frame data(BytePointer setter);
50
    /** Unit: roughly or exactly 0.1 millisecond */
51
    @Cast("uint32_t") int timestamp(); Frame timestamp(int setter);
52
    /** Increasing frame sequence number */
53
    @Cast("uint32_t") int sequence(); Frame sequence(int setter);
54
    /** From 0.5 (very bright) to ~60.0 (fully covered) */
55
    float exposure(); Frame exposure(float setter);
56
    /** From 1.0 (bright) to 1.5 (covered) */
57
    float gain(); Frame gain(float setter);
58
    /** From 1.0 (bright) to 6.4 (covered) */
59
    float gamma(); Frame gamma(float setter);
60
    /** Zero if ok; non-zero for errors */
61
    @Cast("uint32_t") int status(); Frame status(int setter);
62
    /** Byte format - informative only, doesn't indicate errors */
63
    @Cast("libfreenect2::Frame::Format") int format(); Frame format(int setter);
64
}
65
```
66

67
**Usage Examples:**
68

69
```java
70
// Check frame properties
71
System.out.println("Frame dimensions: " + frame.width() + "x" + frame.height());
72
System.out.println("Bytes per pixel: " + frame.bytes_per_pixel());
73
System.out.println("Timestamp: " + frame.timestamp());
74
System.out.println("Status: " + (frame.status() == 0 ? "OK" : "Error"));
75

76
// Access frame data
77
BytePointer frameData = frame.data();
78
byte[] buffer = new byte[1000];
79
frameData.get(buffer);
80

81
// Check frame type and format
82
if (frame.format() == Frame.BGRX) {
83
    System.out.println("Color frame in BGRX format");
84
} else if (frame.format() == Frame.Float) {
85
    System.out.println("Depth or IR frame in float format");
86
}
87
```
88

89
### Frame Listener Interface
90

91
Base callback interface for receiving new frames from the device. Implement this for custom frame processing.
92

93
```java { .api }
94
/**
95
 * Callback interface to receive new frames.
96
 * You can inherit from FrameListener and define your own listener.
97
 */
98
abstract class FrameListener {
99
    /**
100
     * libfreenect2 calls this function when a new frame is decoded.
101
     * @param type Type of the new frame (Frame.Color, Frame.Ir, Frame.Depth)
102
     * @param frame Data of the frame
103
     * @return true if you want to take ownership of the frame, 
104
     *         false to let caller reuse/delete it
105
     */
106
    @Cast("bool") abstract boolean onNewFrame(@Cast("libfreenect2::Frame::Type") int type, Frame frame);
107
}
108
```
109

110
**Usage Examples:**
111

112
```java
113
// Custom frame listener implementation
114
class MyFrameListener extends FrameListener {
115
    @Override
116
    public boolean onNewFrame(int type, Frame frame) {
117
        switch (type) {
118
            case Frame.Color:
119
                System.out.println("Color frame: " + frame.width() + "x" + frame.height());
120
                break;
121
            case Frame.Depth:
122
                System.out.println("Depth frame at " + frame.timestamp());
123
                break;
124
            case Frame.Ir:
125
                System.out.println("IR frame, exposure: " + frame.exposure());
126
                break;
127
        }
128
        return false; // Let caller manage memory
129
    }
130
}
131

132
// Use custom listener
133
MyFrameListener listener = new MyFrameListener();
134
device.setColorFrameListener(listener);
135
device.setIrAndDepthFrameListener(listener);
136
```
137

138
### Synchronous Multi-Frame Listener
139

140
Specialized frame listener that collects multiple frame types synchronously with blocking wait operations.
141

142
```java { .api }
143
/**
144
 * Collect multiple types of frames synchronously
145
 */
146
class SyncMultiFrameListener extends FrameListener {
147
    /**
148
     * Constructor specifying frame types to collect
149
     * @param frame_types Use bitwise OR to combine multiple types, 
150
     *                   e.g. Frame.Ir | Frame.Depth
151
     */
152
    SyncMultiFrameListener(@Cast("unsigned int") int frame_types);
153
    
154
    /** Test if there are new frames available (non-blocking) */
155
    @Cast("bool") boolean hasNewFrame();
156
    
157
    /** 
158
     * Wait for new frames with timeout
159
     * @param frames Output container for received frames
160
     * @param milliseconds Timeout in milliseconds
161
     * @return true if frames received, false if timeout
162
     */
163
    @Cast("bool") boolean waitForNewFrame(@ByRef FrameMap frames, int milliseconds);
164
    
165
    /** 
166
     * Wait indefinitely for new frames
167
     * @param frames Output container for received frames  
168
     */
169
    void waitForNewFrame(@ByRef FrameMap frames);
170
    
171
    /** Shortcut to delete all frames in the map */
172
    void release(@ByRef FrameMap frames);
173
    
174
    /** Override from FrameListener - handles frame collection */
175
    @Cast("bool") boolean onNewFrame(@Cast("libfreenect2::Frame::Type") int type, Frame frame);
176
}
177
```
178

179
**Usage Examples:**
180

181
```java
182
// Set up multi-frame listener for all frame types
183
int frameTypes = Frame.Color | Frame.Ir | Frame.Depth;
184
SyncMultiFrameListener listener = new SyncMultiFrameListener(frameTypes);
185

186
device.setColorFrameListener(listener);
187
device.setIrAndDepthFrameListener(listener);
188
device.start();
189

190
// Wait for frames with timeout
191
FrameMap frames = new FrameMap();
192
if (listener.waitForNewFrame(frames, 10000)) { // 10 second timeout
193
    Frame color = frames.get(Frame.Color);
194
    Frame depth = frames.get(Frame.Depth);
195
    Frame ir = frames.get(Frame.Ir);
196
    
197
    if (color != null) {
198
        System.out.println("Color: " + color.width() + "x" + color.height());
199
    }
200
    if (depth != null) {
201
        System.out.println("Depth: " + depth.width() + "x" + depth.height());
202
    }
203
    
204
    // Always release frames when done
205
    listener.release(frames);
206
} else {
207
    System.out.println("Frame timeout!");
208
}
209

210
// Non-blocking check
211
if (listener.hasNewFrame()) {
212
    listener.waitForNewFrame(frames);
213
    // Process frames...
214
    listener.release(frames);
215
}
216
```
217

218
### Frame Map Container
219

220
Container class for managing multiple frame types, implemented as a wrapper around std::map.
221

222
```java { .api }
223
/**
224
 * Container for multiple frame types (std::map wrapper)
225
 */
226
class FrameMap {
227
    /** Default constructor */
228
    FrameMap();
229
    
230
    /** Check if map contains no frames */
231
    boolean empty();
232
    
233
    /** Get number of frames in the map */
234
    long size();
235
    
236
    /** Get frame by type */
237
    Frame get(@Cast("libfreenect2::Frame::Type") int type);
238
    
239
    /** Store frame by type */
240
    FrameMap put(@Cast("libfreenect2::Frame::Type") int type, Frame value);
241
    
242
    /** Remove frame at iterator position */
243
    void erase(@ByVal Iterator pos);
244
    
245
    /** Get begin iterator */
246
    @ByVal Iterator begin();
247
    
248
    /** Get end iterator */
249
    @ByVal Iterator end();
250
    
251
    /**
252
     * Iterator for traversing frames in the map
253
     */
254
    static class Iterator {
255
        Iterator();
256
        
257
        /** Advance to next element */
258
        @Name("operator ++") @ByRef Iterator increment();
259
        
260
        /** Check equality with another iterator */
261
        @Name("operator ==") boolean equals(@ByRef Iterator it);
262
        
263
        /** Get frame type key */
264
        @Name("operator *().first") @MemberGetter @Cast("libfreenect2::Frame::Type") int first();
265
        
266
        /** Get frame value */
267
        @Name("operator *().second") @MemberGetter @Const Frame second();
268
    }
269
}
270
```
271

272
**Usage Examples:**
273

274
```java
275
FrameMap frames = new FrameMap();
276

277
// After receiving frames from listener
278
if (!frames.empty()) {
279
    System.out.println("Received " + frames.size() + " frames");
280
    
281
    // Access specific frame types
282
    Frame colorFrame = frames.get(Frame.Color);
283
    Frame depthFrame = frames.get(Frame.Depth);
284
    
285
    // Check if specific frame type is available
286
    if (colorFrame != null) {
287
        System.out.println("Color frame available: " + 
288
                          colorFrame.width() + "x" + colorFrame.height());
289
    }
290
    
291
    // Iterate through all frames
292
    FrameMap.Iterator it = frames.begin();
293
    FrameMap.Iterator end = frames.end();
294
    while (!it.equals(end)) {
295
        int frameType = it.first();
296
        Frame frame = it.second();
297
        System.out.println("Frame type " + frameType + 
298
                          ": " + frame.width() + "x" + frame.height());
299
        it.increment();
300
    }
301
}
302

303
// Manual frame insertion
304
Frame customFrame = new Frame(640, 480, 4);
305
frames.put(Frame.Color, customFrame);
306
```
307

308
## Frame Processing Patterns
309

310
### Basic Frame Capture Loop
311

312
```java
313
// Setup
314
SyncMultiFrameListener listener = new SyncMultiFrameListener(
315
    Frame.Color | Frame.Depth);
316
device.setColorFrameListener(listener);
317
device.setIrAndDepthFrameListener(listener);
318
device.start();
319

320
// Capture loop
321
FrameMap frames = new FrameMap();
322
for (int i = 0; i < 100; i++) {
323
    if (listener.waitForNewFrame(frames, 5000)) {
324
        Frame color = frames.get(Frame.Color);
325
        Frame depth = frames.get(Frame.Depth);
326
        
327
        // Process frames here
328
        System.out.println("Frame " + i + " captured");
329
        
330
        listener.release(frames);
331
    } else {
332
        System.err.println("Frame timeout");
333
        break;
334
    }
335
}
336

337
device.stop();
338
device._close();
339
```
340

341
### Asynchronous Frame Processing
342

343
```java
344
class AsyncFrameProcessor extends FrameListener {
345
    private volatile boolean running = true;
346
    
347
    @Override
348
    public boolean onNewFrame(int type, Frame frame) {
349
        if (!running) return false;
350
        
351
        // Process frame on callback thread
352
        switch (type) {
353
            case Frame.Color:
354
                processColorFrame(frame);
355
                break;
356
            case Frame.Depth:
357
                processDepthFrame(frame);
358
                break;
359
        }
360
        
361
        return false; // Don't take ownership
362
    }
363
    
364
    private void processColorFrame(Frame frame) {
365
        // Color processing logic
366
    }
367
    
368
    private void processDepthFrame(Frame frame) {
369
        // Depth processing logic
370
    }
371
    
372
    public void stop() {
373
        running = false;
374
    }
375
}