0
# Registration and Geometry
1

2
Advanced depth-to-color registration and 3D point cloud generation with camera calibration parameters. This module enables precise alignment between depth and color data and conversion to 3D coordinate systems.
3

4
## Capabilities
5

6
### Registration System
7

8
Main registration class for combining depth and color frames with accurate geometric alignment.
9

10
```java { .api }
11
/**
12
 * Combine frames of depth and color camera.
13
 * 
14
 * This class uses a reverse engineered formula that uses factory
15
 * preset extrinsic parameters. We do not have a clear understanding of these
16
 * particular extrinsic parameters and do not know how to calibrate them by hand.
17
 * 
18
 * If you want to perform registration with standard camera extrinsic matrix,
19
 * you probably need something else.
20
 */
21
class Registration {
22
    /**
23
     * Constructor with camera parameters
24
     * @param depth_p Depth camera parameters (can use factory values or custom)
25
     * @param rgb_p Color camera parameters (recommend using factory values)
26
     */
27
    Registration(@ByVal Freenect2Device.IrCameraParams depth_p, 
28
                @ByVal Freenect2Device.ColorCameraParams rgb_p);
29
    
30
    /**
31
     * Undistort and register a single depth point to color camera
32
     * @param dx Distorted depth coordinate x (pixel)
33
     * @param dy Distorted depth coordinate y (pixel)  
34
     * @param dz Depth value (millimeter)
35
     * @param cx Output: Undistorted color coordinate x (normalized)
36
     * @param cy Output: Undistorted color coordinate y (normalized)
37
     */
38
    void apply(int dx, int dy, float dz, @ByRef FloatPointer cx, @ByRef FloatPointer cy);
39
    void apply(int dx, int dy, float dz, @ByRef FloatBuffer cx, @ByRef FloatBuffer cy);
40
    void apply(int dx, int dy, float dz, @ByRef float[] cx, @ByRef float[] cy);
41
    
42
    /**
43
     * Map color images onto depth images (full registration)
44
     * @param rgb Color image (1920x1080 BGRX)
45
     * @param depth Depth image (512x424 float)
46
     * @param undistorted Output: Undistorted depth image
47
     * @param registered Output: Color image for the depth image (512x424)
48
     */
49
    void apply(@Const Frame rgb, @Const Frame depth, Frame undistorted, Frame registered);
50
    
51
    /**
52
     * Full registration with additional options
53
     * @param rgb Color image (1920x1080 BGRX)
54
     * @param depth Depth image (512x424 float)
55
     * @param undistorted Output: Undistorted depth image
56
     * @param registered Output: Color image for depth image (512x424)
57
     * @param enable_filter Filter out pixels not visible to both cameras
58
     * @param bigdepth Output: Mapping of depth onto colors (1920x1082 float, optional)
59
     * @param color_depth_map Output: Index of mapped color pixel for each depth pixel (optional)
60
     */
61
    void apply(@Const Frame rgb, @Const Frame depth, Frame undistorted, Frame registered,
62
              @Cast("const bool") boolean enable_filter, Frame bigdepth, IntPointer color_depth_map);
63
    void apply(@Const Frame rgb, @Const Frame depth, Frame undistorted, Frame registered,
64
              @Cast("const bool") boolean enable_filter, Frame bigdepth, IntBuffer color_depth_map);
65
    void apply(@Const Frame rgb, @Const Frame depth, Frame undistorted, Frame registered,
66
              @Cast("const bool") boolean enable_filter, Frame bigdepth, int[] color_depth_map);
67
    
68
    /**
69
     * Undistort depth image only
70
     * @param depth Depth image (512x424 float)
71
     * @param undistorted Output: Undistorted depth image
72
     */
73
    void undistortDepth(@Const Frame depth, Frame undistorted);
74
    
75
    /**
76
     * Construct a 3-D point with color in a point cloud
77
     * @param undistorted Undistorted depth frame from apply()
78
     * @param registered Registered color frame from apply()
79
     * @param r Row (y) index in depth image
80
     * @param c Column (x) index in depth image
81
     * @param x Output: X coordinate of the 3-D point (meter)
82
     * @param y Output: Y coordinate of the 3-D point (meter)
83
     * @param z Output: Z coordinate of the 3-D point (meter)
84
     * @param rgb Output: Color of the 3-D point (BGRX format)
85
     */
86
    void getPointXYZRGB(@Const Frame undistorted, @Const Frame registered, int r, int c,
87
                       @ByRef FloatPointer x, @ByRef FloatPointer y, @ByRef FloatPointer z, @ByRef FloatPointer rgb);
88
    void getPointXYZRGB(@Const Frame undistorted, @Const Frame registered, int r, int c,
89
                       @ByRef FloatBuffer x, @ByRef FloatBuffer y, @ByRef FloatBuffer z, @ByRef FloatBuffer rgb);
90
    void getPointXYZRGB(@Const Frame undistorted, @Const Frame registered, int r, int c,
91
                       @ByRef float[] x, @ByRef float[] y, @ByRef float[] z, @ByRef float[] rgb);
92
    
93
    /**
94
     * Construct a 3-D point in a point cloud (without color)
95
     * @param undistorted Undistorted depth frame from apply()
96
     * @param r Row (y) index in depth image
97
     * @param c Column (x) index in depth image
98
     * @param x Output: X coordinate of the 3-D point (meter)
99
     * @param y Output: Y coordinate of the 3-D point (meter)
100
     * @param z Output: Z coordinate of the 3-D point (meter)
101
     */
102
    void getPointXYZ(@Const Frame undistorted, int r, int c,
103
                    @ByRef FloatPointer x, @ByRef FloatPointer y, @ByRef FloatPointer z);
104
    void getPointXYZ(@Const Frame undistorted, int r, int c,
105
                    @ByRef FloatBuffer x, @ByRef FloatBuffer y, @ByRef FloatBuffer z);
106
    void getPointXYZ(@Const Frame undistorted, int r, int c,
107
                    @ByRef float[] x, @ByRef float[] y, @ByRef float[] z);
108
}
109
```
110

111
**Usage Examples:**
112

113
```java
114
// Initialize registration with device parameters
115
Freenect2Device.IrCameraParams irParams = device.getIrCameraParams();
116
Freenect2Device.ColorCameraParams colorParams = device.getColorCameraParams();
117
Registration registration = new Registration(irParams, colorParams);
118

119
// Capture synchronized frames
120
FrameMap frames = new FrameMap();
121
if (listener.waitForNewFrame(frames, 10000)) {
122
    Frame color = frames.get(Frame.Color);
123
    Frame depth = frames.get(Frame.Depth);
124
    
125
    // Create output frames
126
    Frame undistorted = new Frame(512, 424, 4); // float depth
127
    Frame registered = new Frame(512, 424, 4);  // BGRX color
128
    
129
    // Perform registration
130
    registration.apply(color, depth, undistorted, registered);
131
    
132
    System.out.println("Registration complete");
133
    
134
    listener.release(frames);
135
}
136

137
// Single point registration
138
float[] cx = new float[1];
139
float[] cy = new float[1];
140
registration.apply(256, 212, 1000.0f, cx, cy); // Center pixel at 1m depth
141
System.out.println("Depth point maps to color coordinates: " + cx[0] + ", " + cy[0]);
142
```
143

144
### Point Cloud Generation
145

146
Generate 3D point clouds from registered depth and color data.
147

148
**Usage Examples:**
149

150
```java
151
// Generate point cloud from registered frames
152
Frame undistorted = new Frame(512, 424, 4);
153
Frame registered = new Frame(512, 424, 4);
154
registration.apply(color, depth, undistorted, registered);
155

156
// Create point cloud data structures
157
int width = (int)undistorted.width();
158
int height = (int)undistorted.height();
159
float[] pointCloud = new float[width * height * 3]; // x,y,z per point
160
int[] colorData = new int[width * height];          // BGRX per point
161

162
// Extract point cloud
163
float[] x = new float[1], y = new float[1], z = new float[1];
164
float[] rgb = new float[1];
165

166
int pointIndex = 0;
167
for (int r = 0; r < height; r++) {
168
    for (int c = 0; c < width; c++) {
169
        registration.getPointXYZRGB(undistorted, registered, r, c, x, y, z, rgb);
170
        
171
        // Store 3D coordinates (in meters)
172
        pointCloud[pointIndex * 3] = x[0];
173
        pointCloud[pointIndex * 3 + 1] = y[0];
174
        pointCloud[pointIndex * 3 + 2] = z[0];
175
        
176
        // Store color (unpack BGRX)
177
        int rgbInt = Float.floatToIntBits(rgb[0]);
178
        colorData[pointIndex] = rgbInt;
179
        
180
        pointIndex++;
181
    }
182
}
183

184
// Filter valid points (remove NaN and infinite values)
185
List<float[]> validPoints = new ArrayList<>();
186
List<Integer> validColors = new ArrayList<>();
187

188
for (int i = 0; i < pointIndex; i++) {
189
    float px = pointCloud[i * 3];
190
    float py = pointCloud[i * 3 + 1];
191
    float pz = pointCloud[i * 3 + 2];
192
    
193
    if (Float.isFinite(px) && Float.isFinite(py) && Float.isFinite(pz) && pz > 0) {
194
        validPoints.add(new float[]{px, py, pz});
195
        validColors.add(colorData[i]);
196
    }
197
}
198

199
System.out.println("Generated " + validPoints.size() + " valid 3D points");
200
```
201

202
### Advanced Registration Options
203

204
```java
205
// Full registration with filtering and additional outputs
206
Frame bigdepth = new Frame(1920, 1082, 4); // Note: 1082 not 1080
207
int[] colorDepthMap = new int[512 * 424];
208

209
registration.apply(
210
    color, depth,           // Input frames
211
    undistorted, registered, // Standard outputs
212
    true,                   // Enable filtering
213
    bigdepth,              // Depth mapped to color resolution
214
    colorDepthMap          // Per-depth-pixel color mapping
215
);
216

217
// Process big depth image (depth data at color resolution)
218
System.out.println("Big depth dimensions: " + bigdepth.width() + "x" + bigdepth.height());
219

220
// Use color-depth mapping for efficient lookups
221
BytePointer colorData = registered.data();
222
for (int depthPixel = 0; depthPixel < colorDepthMap.length; depthPixel++) {
223
    int colorPixel = colorDepthMap[depthPixel];
224
    if (colorPixel >= 0) {
225
        // Valid mapping from depth pixel to color pixel
226
        int colorOffset = colorPixel * 4; // BGRX format
227
        // Access color data at colorData + colorOffset
228
    }
229
}
230
```
231

232
### Registration Implementation
233

234
Internal registration implementation details.
235

236
```java { .api }
237
/**
238
 * Implementation details for registration
239
 */
240
class RegistrationImpl {
241
    // Internal implementation - not for direct use
242
}
243
```
244

245
## Coordinate Systems and Calibration
246

247
### Understanding Coordinate Systems
248

249
The Kinect v2 uses different coordinate systems for depth and color cameras:
250

251
- **Depth Camera**: 512x424 resolution, centered coordinate system
252
- **Color Camera**: 1920x1080 resolution, different intrinsic parameters
253
- **3D World**: Metric coordinates in meters, origin at depth camera
254

255
### Calibration Parameter Usage
256

257
```java
258
// Examine factory calibration parameters
259
Freenect2Device.IrCameraParams irParams = device.getIrCameraParams();
260
System.out.println("IR Camera Intrinsics:");
261
System.out.println("  fx: " + irParams.fx() + ", fy: " + irParams.fy());
262
System.out.println("  cx: " + irParams.cx() + ", cy: " + irParams.cy());
263
System.out.println("  Distortion - k1: " + irParams.k1() + ", k2: " + irParams.k2());
264

265
Freenect2Device.ColorCameraParams colorParams = device.getColorCameraParams();
266
System.out.println("Color Camera Intrinsics:");
267
System.out.println("  fx: " + colorParams.fx() + ", fy: " + colorParams.fy());
268
System.out.println("  cx: " + colorParams.cx() + ", cy: " + colorParams.cy());
269
System.out.println("  Extrinsic shift: " + colorParams.shift_d() + ", " + colorParams.shift_m());
270

271
// Use custom calibrated parameters (advanced users)
272
irParams.fx(365.456f);  // Custom focal length
273
irParams.fy(365.456f);
274
irParams.cx(257.608f);  // Custom principal point
275
irParams.cy(210.040f);
276
device.setIrCameraParams(irParams);
277

278
// Create registration with custom parameters
279
Registration customRegistration = new Registration(irParams, colorParams);
280
```
281

282
### Color Data Format
283

284
When extracting color from point clouds, the RGB data is packed in BGRX format:
285

286
```java
287
// Unpack BGRX color data
288
float[] rgbFloat = new float[1];
289
registration.getPointXYZRGB(undistorted, registered, r, c, x, y, z, rgbFloat);
290

291
int rgbPacked = Float.floatToIntBits(rgbFloat[0]);
292
int blue = rgbPacked & 0xFF;
293
int green = (rgbPacked >> 8) & 0xFF;
294
int red = (rgbPacked >> 16) & 0xFF;
295
// Alpha/unused byte is (rgbPacked >> 24) & 0xFF
296

297
System.out.println("Point color - R: " + red + ", G: " + green + ", B: " + blue);
298
```