0
# 3D Mask Rendering
1

2
Advanced 3D face mask rendering capabilities using morphable face models for applications like face swapping, virtual try-on, augmented reality, and face mask simulation. Supports both realistic mask overlays and data augmentation for training.
3

4
## Capabilities
5

6
### MaskRenderer Class
7

8
3D face mask rendering engine that uses morphable models to fit virtual masks to faces with proper perspective and lighting.
9

10
```python { .api }
11
class MaskRenderer:
12
    def __init__(self, name='buffalo_l', root='~/.insightface', insfa=None):
13
        """
14
        Initialize 3D mask renderer.
15
        
16
        Parameters:
17
        - name: str, model pack name containing 3D morphable model
18
        - root: str, model storage directory
19
        - insfa: FaceAnalysis instance, if provided will use for face detection
20
        """
21
    
22
    def prepare(self, ctx_id=0, det_thresh=0.5, det_size=(128, 128)):
23
        """
24
        Prepare renderer for inference.
25
        
26
        Parameters:
27
        - ctx_id: int, device context ID
28
        - det_thresh: float, detection confidence threshold
29
        - det_size: tuple, detection input size for face analysis
30
        """
31
    
32
    def build_params(self, face_image):
33
        """
34
        Build 3D face parameters from input image.
35
        
36
        Parameters:
37
        - face_image: np.ndarray, input face image
38
        
39
        Returns:
40
        dict: 3D face parameters including shape, expression, pose, and texture
41
        """
42
    
43
    def render_mask(self, face_image, mask_image, params, input_is_rgb=False, auto_blend=True, positions=[0.1, 0.33, 0.9, 0.7]):
44
        """
45
        Render 3D mask on face image.
46
        
47
        Parameters:
48
        - face_image: np.ndarray, target face image
49
        - mask_image: np.ndarray, mask texture image
50
        - params: dict, 3D face parameters from build_params()
51
        - input_is_rgb: bool, whether input images are RGB (default: BGR)
52
        - auto_blend: bool, automatically blend mask edges
53
        - positions: list, mask positioning parameters [x_offset, y_offset, width_scale, height_scale]
54
        
55
        Returns:
56
        np.ndarray: image with rendered 3D mask
57
        """
58
    
59
    def draw_lmk(self, face_image):
60
        """
61
        Draw 3D landmarks on face image for debugging.
62
        
63
        Parameters:
64
        - face_image: np.ndarray, input face image
65
        
66
        Returns:
67
        np.ndarray: image with drawn 3D landmarks
68
        """
69
```
70

71
### Static Parameter Operations
72

73
Utilities for encoding and decoding 3D face parameters for storage and transmission.
74

75
```python { .api }
76
@staticmethod
77
def encode_params(params):
78
    """
79
    Encode face parameters for storage.
80
    
81
    Parameters:
82
    - params: dict, face parameters dictionary
83
    
84
    Returns:
85
    bytes: encoded parameter data
86
    """
87

88
@staticmethod 
89
def decode_params(params):
90
    """
91
    Decode stored face parameters.
92
    
93
    Parameters:
94
    - params: bytes, encoded parameter data
95
    
96
    Returns:
97
    dict: decoded face parameters
98
    """
99
```
100

101
### MaskAugmentation Class
102

103
Albumentations-compatible data augmentation transform for training robust face analysis models.
104

105
```python { .api }
106
class MaskAugmentation(ImageOnlyTransform):
107
    def __init__(self, mask_names=['mask_white', 'mask_blue', 'mask_black', 'mask_green'],
108
                 mask_probs=[0.4, 0.4, 0.1, 0.1], h_low=0.33, h_high=0.35, always_apply=False, p=1.0):
109
        """
110
        Initialize mask augmentation transform.
111
        
112
        Parameters:
113
        - mask_names: list, available mask types
114
        - mask_probs: list, probability for each mask type
115
        - h_low: float, minimum mask height ratio
116
        - h_high: float, maximum mask height ratio  
117
        - always_apply: bool, whether to always apply transform
118
        - p: float, probability of applying transform
119
        """
120
    
121
    def apply(self, image, hlabel, mask_name, h_pos, **params):
122
        """
123
        Apply mask augmentation to image.
124
        
125
        Parameters:
126
        - image: np.ndarray, input image
127
        - hlabel: float, mask height label
128
        - mask_name: str, mask type to apply
129
        - h_pos: float, vertical position of mask
130
        - **params: additional parameters
131
        
132
        Returns:
133
        np.ndarray: augmented image with mask
134
        """
135
```
136

137
## Usage Examples
138

139
### Basic Mask Rendering
140

141
```python
142
import cv2
143
from insightface.app.mask_renderer import MaskRenderer
144

145
# Initialize mask renderer
146
renderer = MaskRenderer(name='buffalo_l')
147
renderer.prepare(ctx_id=0)
148

149
# Load face and mask images
150
face_img = cv2.imread('person.jpg')
151
mask_img = cv2.imread('surgical_mask.png')
152

153
# Build 3D face parameters
154
params = renderer.build_params(face_img)
155
print(f"Built parameters with keys: {list(params.keys())}")
156

157
# Render mask on face
158
result = renderer.render_mask(face_img, mask_img, params, auto_blend=True)
159

160
# Save result
161
cv2.imwrite('masked_face.jpg', result)
162
```
163

164
### Custom Mask Positioning
165

166
```python
167
# Fine-tune mask position and size
168
positions = [0.05, 0.4, 0.8, 0.6]  # [x_offset, y_offset, width_scale, height_scale]
169

170
result = renderer.render_mask(
171
    face_img, 
172
    mask_img, 
173
    params,
174
    positions=positions,
175
    auto_blend=True
176
)
177

178
cv2.imwrite('custom_positioned_mask.jpg', result)
179
```
180

181
### Batch Mask Rendering
182

183
```python
184
from insightface.app import FaceAnalysis
185

186
# Setup face analysis and mask renderer
187
app = FaceAnalysis()
188
app.prepare(ctx_id=0)
189

190
renderer = MaskRenderer()
191
renderer.prepare(ctx_id=0)
192

193
def render_masks_on_all_faces(image, mask_image):
194
    """Apply masks to all detected faces in an image."""
195
    faces = app.get(image)
196
    result_img = image.copy()
197
    
198
    for i, face in enumerate(faces):
199
        # Extract face region
200
        x1, y1, x2, y2 = face.bbox.astype(int)
201
        face_region = image[y1:y2, x1:x2]
202
        
203
        if face_region.size > 0:
204
            # Build parameters for this face
205
            params = renderer.build_params(face_region)
206
            
207
            # Render mask on face region
208
            masked_region = renderer.render_mask(face_region, mask_image, params)
209
            
210
            # Paste back to original image
211
            result_img[y1:y2, x1:x2] = masked_region
212
    
213
    return result_img
214

215
# Process group photo
216
group_img = cv2.imread('group_photo.jpg')
217
mask_texture = cv2.imread('blue_mask.png')
218

219
masked_group = render_masks_on_all_faces(group_img, mask_texture)
220
cv2.imwrite('masked_group.jpg', masked_group)
221
```
222

223
### Parameter Persistence
224

225
```python
226
# Build and save parameters for later use
227
face_img = cv2.imread('face.jpg')
228
params = renderer.build_params(face_img)
229

230
# Encode parameters for storage
231
encoded_params = MaskRenderer.encode_params(params)
232
print(f"Encoded parameters size: {len(encoded_params)} bytes")
233

234
# Save to file
235
with open('face_params.bin', 'wb') as f:
236
    f.write(encoded_params)
237

238
# Later: load and decode parameters
239
with open('face_params.bin', 'rb') as f:
240
    loaded_encoded = f.read()
241

242
decoded_params = MaskRenderer.decode_params(loaded_encoded)
243

244
# Use loaded parameters for rendering
245
mask_img = cv2.imread('new_mask.png')
246
result = renderer.render_mask(face_img, mask_img, decoded_params)
247
```
248

249
### Data Augmentation for Training
250

251
```python
252
import albumentations as A
253
from insightface.app.mask_renderer import MaskAugmentation
254

255
# Setup augmentation pipeline
256
transform = A.Compose([
257
    A.HorizontalFlip(p=0.5),
258
    A.RandomBrightnessContrast(p=0.3),
259
    MaskAugmentation(
260
        mask_names=['mask_white', 'mask_blue', 'mask_surgical'],
261
        mask_probs=[0.3, 0.3, 0.4],
262
        p=0.7  # 70% chance to apply mask
263
    ),
264
    A.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])
265
])
266

267
# Apply to training images
268
def augment_training_data(image_path):
269
    image = cv2.imread(image_path)
270
    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
271
    
272
    augmented = transform(image=image)
273
    return augmented['image']
274

275
# Process training dataset
276
training_images = ['face1.jpg', 'face2.jpg', 'face3.jpg']
277
augmented_data = []
278

279
for img_path in training_images:
280
    # Generate multiple augmented versions
281
    for i in range(5):
282
        aug_image = augment_training_data(img_path)
283
        augmented_data.append(aug_image)
284

285
print(f"Generated {len(augmented_data)} augmented training samples")
286
```
287

288
### 3D Landmark Visualization
289

290
```python
291
# Visualize 3D face fitting quality
292
def visualize_3d_fitting(face_image):
293
    """Visualize 3D face model fitting quality."""
294
    # Build 3D parameters
295
    params = renderer.build_params(face_image)
296
    
297
    # Draw 3D landmarks
298
    landmark_img = renderer.draw_lmk(face_image)
299
    
300
    # Create side-by-side comparison
301
    combined = np.hstack([face_image, landmark_img])
302
    
303
    return combined, params
304

305
face_img = cv2.imread('test_face.jpg')
306
comparison, params = visualize_3d_fitting(face_img)
307

308
cv2.imwrite('3d_fitting_comparison.jpg', comparison)
309
print(f"3D model parameters: {params.keys()}")
310
```
311

312
### Multiple Mask Types
313

314
```python
315
# Apply different mask types with varying parameters
316
mask_types = {
317
    'surgical': {
318
        'image': 'surgical_mask.png',
319
        'positions': [0.1, 0.35, 0.8, 0.5]
320
    },
321
    'n95': {
322
        'image': 'n95_mask.png', 
323
        'positions': [0.05, 0.3, 0.9, 0.6]
324
    },
325
    'cloth': {
326
        'image': 'cloth_mask.png',
327
        'positions': [0.08, 0.38, 0.85, 0.45]
328
    }
329
}
330

331
def render_different_masks(face_image, params):
332
    """Render different mask types on the same face."""
333
    results = {}
334
    
335
    for mask_name, config in mask_types.items():
336
        mask_img = cv2.imread(config['image'])
337
        
338
        result = renderer.render_mask(
339
            face_image,
340
            mask_img, 
341
            params,
342
            positions=config['positions'],
343
            auto_blend=True
344
        )
345
        
346
        results[mask_name] = result
347
    
348
    return results
349

350
# Apply different masks
351
face_img = cv2.imread('person.jpg')
352
params = renderer.build_params(face_img)
353
mask_results = render_different_masks(face_img, params)
354

355
# Save results
356
for mask_type, result_img in mask_results.items():
357
    cv2.imwrite(f'face_with_{mask_type}_mask.jpg', result_img)
358
```
359

360
### Real-time Mask Rendering
361

362
```python
363
import time
364

365
def real_time_mask_rendering():
366
    """Real-time mask rendering from webcam."""
367
    cap = cv2.VideoCapture(0)
368
    
369
    # Load mask image
370
    mask_img = cv2.imread('default_mask.png')
371
    
372
    while True:
373
        ret, frame = cap.read()
374
        if not ret:
375
            break
376
        
377
        try:
378
            # Build parameters for current frame
379
            params = renderer.build_params(frame)
380
            
381
            # Render mask
382
            masked_frame = renderer.render_mask(frame, mask_img, params)
383
            
384
            cv2.imshow('Real-time Mask', masked_frame)
385
            
386
        except Exception as e:
387
            # Fallback to original frame if rendering fails
388
            cv2.imshow('Real-time Mask', frame)
389
        
390
        if cv2.waitKey(1) & 0xFF == ord('q'):
391
            break
392
    
393
    cap.release()
394
    cv2.destroyAllWindows()
395

396
# Run real-time rendering (uncomment to use)
397
# real_time_mask_rendering()
398
```