0
# Face Processing Utilities
1

2
Comprehensive utilities for face alignment, transformation, and preprocessing including normalization, cropping, coordinate transformations, and geometric operations on facial landmarks and images.
3

4
## Capabilities
5

6
### Face Alignment and Cropping
7

8
Functions for normalizing face regions based on facial landmarks for consistent processing.
9

10
```python { .api }
11
def norm_crop(img, landmark, image_size=112, mode='arcface') -> np.ndarray:
12
    """
13
    Normalize and crop face region based on landmarks.
14
    
15
    Parameters:
16
    - img: np.ndarray, input image
17
    - landmark: np.ndarray, facial landmarks, shape (5, 2) or (68, 2)
18
    - image_size: int, output image size (square)
19
    - mode: str, alignment mode ('arcface', 'ffhq')
20
    
21
    Returns:
22
    np.ndarray: normalized and cropped face image
23
    """
24

25
def norm_crop2(img, landmark, image_size=112, mode='arcface') -> Tuple[np.ndarray, np.ndarray]:
26
    """
27
    Normalize and crop face with transformation matrix.
28
    
29
    Parameters: same as norm_crop
30
    
31
    Returns:
32
    tuple: (cropped_face, transformation_matrix)
33
    - cropped_face: np.ndarray, normalized face image
34
    - transformation_matrix: np.ndarray, 2x3 affine transformation matrix
35
    """
36

37
def estimate_norm(lmk, image_size=112, mode='arcface') -> np.ndarray:
38
    """
39
    Estimate normalization transformation matrix from landmarks.
40
    
41
    Parameters:
42
    - lmk: np.ndarray, facial landmarks
43
    - image_size: int, target image size
44
    - mode: str, alignment mode
45
    
46
    Returns:
47
    np.ndarray: 2x3 affine transformation matrix
48
    """
49

50
def square_crop(im, S) -> Tuple[np.ndarray, float]:
51
    """
52
    Crop image to square with scaling.
53
    
54
    Parameters:
55
    - im: np.ndarray, input image
56
    - S: int, target square size
57
    
58
    Returns:
59
    tuple: (cropped_image, scale_factor)
60
    """
61
```
62

63
### Geometric Transformations
64

65
Functions for applying geometric transformations to images and point coordinates.
66

67
```python { .api }
68
def transform(data, center, output_size, scale, rotation) -> Tuple[np.ndarray, np.ndarray]:
69
    """
70
    Apply geometric transformation to image data.
71
    
72
    Parameters:
73
    - data: np.ndarray, input image
74
    - center: tuple, center point (x, y)
75
    - output_size: tuple, output size (width, height)
76
    - scale: float, scaling factor
77
    - rotation: float, rotation angle in degrees
78
    
79
    Returns:
80
    tuple: (transformed_image, transformation_matrix)
81
    """
82

83
def trans_points2d(pts, M) -> np.ndarray:
84
    """
85
    Transform 2D points using affine transformation matrix.
86
    
87
    Parameters:
88
    - pts: np.ndarray, 2D points, shape (N, 2)
89
    - M: np.ndarray, 2x3 or 3x3 transformation matrix
90
    
91
    Returns:
92
    np.ndarray: transformed points, shape (N, 2)
93
    """
94

95
def trans_points3d(pts, M) -> np.ndarray:
96
    """
97
    Transform 3D points using transformation matrix.
98
    
99
    Parameters:
100
    - pts: np.ndarray, 3D points, shape (N, 3)
101
    - M: np.ndarray, 3x3 or 4x4 transformation matrix
102
    
103
    Returns:
104
    np.ndarray: transformed points, shape (N, 3)
105
    """
106

107
def trans_points(pts, M) -> np.ndarray:
108
    """
109
    Transform points (auto-detects 2D or 3D).
110
    
111
    Parameters:
112
    - pts: np.ndarray, points to transform
113
    - M: np.ndarray, transformation matrix
114
    
115
    Returns:
116
    np.ndarray: transformed points
117
    """
118
```
119

120
### 3D Geometry Operations
121

122
Advanced functions for 3D facial geometry processing and pose estimation.
123

124
```python { .api }
125
def estimate_affine_matrix_3d23d(X, Y) -> np.ndarray:
126
    """
127
    Estimate 3D-to-3D affine transformation matrix.
128
    
129
    Parameters:
130
    - X: np.ndarray, source 3D points, shape (N, 3)
131
    - Y: np.ndarray, target 3D points, shape (N, 3)
132
    
133
    Returns:
134
    np.ndarray: 4x4 affine transformation matrix
135
    """
136

137
def P2sRt(P) -> Tuple[float, np.ndarray, np.ndarray]:
138
    """
139
    Decompose projection matrix into scale, rotation, and translation.
140
    
141
    Parameters:
142
    - P: np.ndarray, 3x4 projection matrix
143
    
144
    Returns:
145
    tuple: (scale, rotation_matrix, translation_vector)
146
    - scale: float, scaling factor
147
    - rotation_matrix: np.ndarray, 3x3 rotation matrix
148
    - translation_vector: np.ndarray, 3D translation vector
149
    """
150

151
def matrix2angle(R) -> Tuple[float, float, float]:
152
    """
153
    Convert rotation matrix to Euler angles.
154
    
155
    Parameters:
156
    - R: np.ndarray, 3x3 rotation matrix
157
    
158
    Returns:
159
    tuple: (pitch, yaw, roll) angles in radians
160
    """
161
```
162

163
## Usage Examples
164

165
### Face Alignment for Recognition
166

167
```python
168
import cv2
169
import numpy as np
170
from insightface.utils.face_align import norm_crop, estimate_norm
171

172
# Load image and detect landmarks
173
img = cv2.imread('face.jpg')
174
# Assume landmarks detected: shape (5, 2) for 5-point landmarks
175
landmarks = np.array([[38.2946, 51.6963],
176
                      [73.5318, 51.5014], 
177
                      [56.0252, 71.7366],
178
                      [41.5493, 92.3655],
179
                      [70.7299, 92.2041]])
180

181
# Normalize face for recognition model input
182
aligned_face = norm_crop(img, landmarks, image_size=112, mode='arcface')
183

184
# Save aligned face
185
cv2.imwrite('aligned_face.jpg', aligned_face)
186
print(f"Aligned face shape: {aligned_face.shape}")  # (112, 112, 3)
187
```
188

189
### Face Alignment with Transformation Matrix
190

191
```python
192
# Get both aligned face and transformation matrix
193
aligned_face, transform_matrix = norm_crop2(img, landmarks, image_size=112)
194

195
print(f"Transform matrix shape: {transform_matrix.shape}")  # (2, 3)
196
print(f"Transform matrix:\n{transform_matrix}")
197

198
# Apply same transformation to other points
199
additional_landmarks = np.array([[45, 85], [65, 85]])  # chin points
200
transformed_landmarks = trans_points2d(additional_landmarks, transform_matrix)
201
print(f"Transformed chin points: {transformed_landmarks}")
202
```
203

204
### Batch Face Alignment
205

206
```python
207
from insightface.app import FaceAnalysis
208

209
app = FaceAnalysis()
210
app.prepare(ctx_id=0)
211

212
def align_faces_from_detection(img, faces):
213
    """Align all detected faces in an image."""
214
    aligned_faces = []
215
    
216
    for face in faces:
217
        if face.kps is not None:
218
            # Use 5-point landmarks for alignment
219
            aligned = norm_crop(img, face.kps, image_size=112, mode='arcface')
220
            aligned_faces.append(aligned)
221
    
222
    return aligned_faces
223

224
# Process image
225
img = cv2.imread('group_photo.jpg')
226
faces = app.get(img)
227
aligned_faces = align_faces_from_detection(img, faces)
228

229
print(f"Aligned {len(aligned_faces)} faces")
230
for i, aligned in enumerate(aligned_faces):
231
    cv2.imwrite(f'aligned_face_{i}.jpg', aligned)
232
```
233

234
### 3D Face Processing
235

236
```python
237
import numpy as np
238
from insightface.utils.transform import matrix2angle, P2sRt
239

240
# Example: Process 3D landmarks for pose estimation
241
def estimate_head_pose(landmarks_3d):
242
    """Estimate head pose from 3D landmarks."""
243
    # Select key points for pose estimation
244
    nose_tip = landmarks_3d[30]      # Nose tip
245
    left_eye = landmarks_3d[36]      # Left eye corner
246
    right_eye = landmarks_3d[45]     # Right eye corner
247
    left_mouth = landmarks_3d[48]    # Left mouth corner
248
    right_mouth = landmarks_3d[54]   # Right mouth corner
249
    
250
    # Create reference points (canonical face)
251
    ref_points = np.array([
252
        [0, 0, 0],        # Nose tip (origin)
253
        [-30, -20, -5],   # Left eye
254
        [30, -20, -5],    # Right eye  
255
        [-20, 20, -5],    # Left mouth
256
        [20, 20, -5]      # Right mouth
257
    ])
258
    
259
    face_points = np.array([nose_tip, left_eye, right_eye, left_mouth, right_mouth])
260
    
261
    # Estimate transformation
262
    transform_matrix = estimate_affine_matrix_3d23d(ref_points, face_points)
263
    
264
    # Extract rotation matrix and convert to angles
265
    rotation_matrix = transform_matrix[:3, :3]
266
    pitch, yaw, roll = matrix2angle(rotation_matrix)
267
    
268
    return np.degrees([pitch, yaw, roll])
269

270
# Example usage with 3D landmarks
271
if hasattr(face, 'landmark_3d_68') and face.landmark_3d_68 is not None:
272
    pose_angles = estimate_head_pose(face.landmark_3d_68)
273
    print(f"Head pose - Pitch: {pose_angles[0]:.1f}°, Yaw: {pose_angles[1]:.1f}°, Roll: {pose_angles[2]:.1f}°")
274
```
275

276
### Custom Face Preprocessing Pipeline
277

278
```python
279
def preprocess_face_for_model(img, face, target_size=112, mode='arcface'):
280
    """
281
    Complete preprocessing pipeline for face recognition.
282
    
283
    Parameters:
284
    - img: input image
285
    - face: Face object with landmarks
286
    - target_size: output image size
287
    - mode: alignment mode
288
    
289
    Returns:
290
    - preprocessed face ready for model input
291
    """
292
    if face.kps is None:
293
        # Fallback: use bounding box center
294
        x1, y1, x2, y2 = face.bbox
295
        center_x, center_y = (x1 + x2) / 2, (y1 + y2) / 2
296
        
297
        # Create approximate landmarks for alignment
298
        face_width = x2 - x1
299
        eye_y = y1 + face_width * 0.35
300
        
301
        landmarks = np.array([
302
            [x1 + face_width * 0.3, eye_y],      # Left eye approx
303
            [x1 + face_width * 0.7, eye_y],      # Right eye approx
304
            [center_x, y1 + face_width * 0.6],   # Nose approx
305
            [x1 + face_width * 0.35, y1 + face_width * 0.8],  # Left mouth
306
            [x1 + face_width * 0.65, y1 + face_width * 0.8]   # Right mouth
307
        ])
308
    else:
309
        landmarks = face.kps
310
    
311
    # Align and normalize face
312
    aligned_face = norm_crop(img, landmarks, image_size=target_size, mode=mode)
313
    
314
    # Additional preprocessing (normalize pixel values)
315
    aligned_face = aligned_face.astype(np.float32)
316
    aligned_face = (aligned_face - 127.5) / 128.0  # Normalize to [-1, 1]
317
    
318
    return aligned_face
319

320
# Usage
321
preprocessed = preprocess_face_for_model(img, face)
322
print(f"Preprocessed face shape: {preprocessed.shape}")
323
print(f"Value range: [{preprocessed.min():.3f}, {preprocessed.max():.3f}]")
324
```
325

326
### Transform Validation and Inverse Operations
327

328
```python
329
def validate_alignment_quality(original_landmarks, aligned_landmarks, transform_matrix):
330
    """Validate face alignment quality by checking landmark consistency."""
331
    
332
    # Apply transformation to original landmarks
333
    predicted_landmarks = trans_points2d(original_landmarks, transform_matrix)
334
    
335
    # Calculate alignment error
336
    error = np.mean(np.linalg.norm(predicted_landmarks - aligned_landmarks, axis=1))
337
    
338
    return error
339

340
# Example usage
341
original_kps = face.kps
342
aligned_face, transform_matrix = norm_crop2(img, original_kps, image_size=112)
343

344
# In aligned image, estimate where landmarks should be
345
expected_kps = np.array([[38, 51], [73, 51], [56, 71], [41, 92], [70, 92]])
346

347
error = validate_alignment_quality(original_kps, expected_kps, transform_matrix)
348
print(f"Alignment error: {error:.2f} pixels")
349

350
if error > 5.0:
351
    print("Warning: Poor alignment quality detected")
352
```