0
# BodyPix
1

2
BodyPix is a TensorFlow.js library that provides real-time person and body part segmentation in web browsers using machine learning models. It enables pixel-level classification of human figures and their 24 distinct body parts within images or video streams, supporting both single and multi-person scenarios with configurable neural network architectures.
3

4
## Package Information
5

6
- **Package Name**: @tensorflow-models/body-pix
7
- **Package Type**: npm
8
- **Language**: TypeScript
9
- **Installation**: `npm install @tensorflow-models/body-pix`
10
- **Peer Dependencies**: `@tensorflow/tfjs-core`, `@tensorflow/tfjs-converter`, `@tensorflow/tfjs-backend-webgl`
11

12
## Core Imports
13

14
```typescript
15
import * as bodyPix from '@tensorflow-models/body-pix';
16
```
17

18
Named imports:
19

20
```typescript
21
import { load, BodyPix, toMask, drawBokehEffect } from '@tensorflow-models/body-pix';
22
```
23

24
For CommonJS:
25

26
```javascript
27
const bodyPix = require('@tensorflow-models/body-pix');
28
```
29

30
## Basic Usage
31

32
```typescript
33
import * as bodyPix from '@tensorflow-models/body-pix';
34

35
// Load the model
36
const net = await bodyPix.load();
37

38
// Get image/video element
39
const imageElement = document.getElementById('person');
40

41
// Segment person
42
const segmentation = await net.segmentPerson(imageElement);
43

44
// Create and apply background blur effect
45
const canvas = document.getElementById('canvas');
46
bodyPix.drawBokehEffect(canvas, imageElement, segmentation);
47
```
48

49
## Architecture
50

51
BodyPix is built around several key components:
52

53
- **Model Loading**: Configurable loading of MobileNet or ResNet architectures with different accuracy/performance trade-offs
54
- **Segmentation Engine**: Core neural network inference providing person and body part segmentation
55
- **Multi-Person Support**: Instance-level segmentation supporting multiple people in a single image
56
- **Rendering Pipeline**: Utilities for mask generation, visualization, and special effects like background blur
57
- **Pose Integration**: Each segmentation result includes associated pose keypoints for enhanced functionality
58

59
## Capabilities
60

61
### Model Loading and Configuration
62

63
Load and configure BodyPix models with different architectures and performance characteristics.
64

65
```typescript { .api }
66
function load(config?: ModelConfig): Promise<BodyPix>;
67

68
interface ModelConfig {
69
  architecture: 'MobileNetV1' | 'ResNet50';
70
  outputStride: 8 | 16 | 32;
71
  multiplier?: 0.50 | 0.75 | 1.0;
72
  modelUrl?: string;
73
  quantBytes?: 1 | 2 | 4;
74
}
75
```
76

77
[Model Loading](./model-loading.md)
78

79
### Person Segmentation
80

81
Segment people in images and videos with semantic and instance-level segmentation options.
82

83
```typescript { .api }
84
class BodyPix {
85
  segmentPerson(
86
    input: BodyPixInput,
87
    config?: PersonInferenceConfig
88
  ): Promise<SemanticPersonSegmentation>;
89
  
90
  segmentMultiPerson(
91
    input: BodyPixInput,
92
    config?: MultiPersonInstanceInferenceConfig
93
  ): Promise<PersonSegmentation[]>;
94
}
95

96
type BodyPixInput = ImageData | HTMLImageElement | HTMLCanvasElement | HTMLVideoElement | OffscreenCanvas | tf.Tensor3D;
97
```
98

99
[Person Segmentation](./person-segmentation.md)
100

101
### Body Part Segmentation
102

103
Segment 24 distinct body parts with detailed classification for advanced applications.
104

105
```typescript { .api }
106
class BodyPix {
107
  segmentPersonParts(
108
    input: BodyPixInput,
109
    config?: PersonInferenceConfig
110
  ): Promise<SemanticPartSegmentation>;
111
  
112
  segmentMultiPersonParts(
113
    input: BodyPixInput,
114
    config?: MultiPersonInstanceInferenceConfig
115
  ): Promise<PartSegmentation[]>;
116
}
117
```
118

119
[Body Part Segmentation](./body-part-segmentation.md)
120

121
### Rendering and Effects
122

123
Create visual effects, masks, and overlays from segmentation results.
124

125
```typescript { .api }
126
function toMask(
127
  segmentation: SemanticPersonSegmentation | PersonSegmentation[],
128
  foreground?: Color,
129
  background?: Color
130
): ImageData;
131

132
function drawBokehEffect(
133
  canvas: HTMLCanvasElement | OffscreenCanvas,
134
  image: HTMLImageElement | HTMLVideoElement | HTMLCanvasElement,
135
  personSegmentation: SemanticPersonSegmentation | PersonSegmentation[],
136
  backgroundBlurAmount?: number,
137
  edgeBlurAmount?: number,
138
  flipHorizontal?: boolean
139
): void;
140

141
function blurBodyPart(
142
  canvas: HTMLCanvasElement | OffscreenCanvas,
143
  image: HTMLImageElement | HTMLVideoElement | HTMLCanvasElement,
144
  partSegmentation: SemanticPartSegmentation | PartSegmentation[],
145
  bodyPartIdsToBlur?: number[],
146
  backgroundBlurAmount?: number,
147
  edgeBlurAmount?: number
148
): void;
149
```
150

151
[Rendering and Effects](./rendering-effects.md)
152

153
### Utility Functions
154

155
Helper functions for pose manipulation, tensor operations, and coordinate transformations.
156

157
```typescript { .api }
158
function flipPoseHorizontal(pose: Pose, imageWidth: number): Pose;
159

160
function resizeAndPadTo(
161
  imageTensor: tf.Tensor3D,
162
  [targetH, targetW]: [number, number],
163
  flipHorizontal?: boolean
164
): {
165
  resizedAndPadded: tf.Tensor3D,
166
  paddedBy: [[number, number], [number, number]]
167
};
168
```
169

170
[Utility Functions](./utilities.md)
171

172
## Core Types
173

174
```typescript { .api }
175
interface PersonSegmentation {
176
  data: Uint8Array;    // Binary mask (0=background, 1=person)
177
  width: number;       // Mask width
178
  height: number;      // Mask height
179
  pose: Pose;         // Associated pose keypoints
180
}
181

182
interface SemanticPersonSegmentation {
183
  data: Uint8Array;    // Combined mask for all people
184
  width: number;       // Mask width
185
  height: number;      // Mask height
186
  allPoses: Pose[];   // All detected poses
187
}
188

189
interface PartSegmentation {
190
  data: Int32Array;    // Part IDs (0-23) or -1 for background
191
  width: number;       // Mask width
192
  height: number;      // Mask height
193
  pose: Pose;         // Associated pose keypoints
194
}
195

196
interface SemanticPartSegmentation {
197
  data: Int32Array;    // Combined part IDs for all people
198
  width: number;       // Mask width
199
  height: number;      // Mask height
200
  allPoses: Pose[];   // All detected poses
201
}
202

203
interface Pose {
204
  keypoints: Keypoint[];  // 17 body keypoints
205
  score: number;         // Overall pose confidence (0-1)
206
}
207

208
interface Keypoint {
209
  score: number;         // Keypoint confidence (0-1)
210
  position: Vector2D;    // Pixel coordinates
211
  part: string;         // Body part name
212
}
213

214
interface Vector2D {
215
  x: number;            // X coordinate
216
  y: number;            // Y coordinate
217
}
218

219
interface Color {
220
  r: number;            // Red (0-255)
221
  g: number;            // Green (0-255)
222
  b: number;            // Blue (0-255)
223
  a: number;            // Alpha (0-255)
224
}
225
```
226

227
## Constants
228

229
```typescript { .api }
230
const PART_CHANNELS: string[] = [
231
  'left_face',           // 0
232
  'right_face',          // 1
233
  'left_upper_arm_front', // 2
234
  'left_upper_arm_back',  // 3
235
  'right_upper_arm_front', // 4
236
  'right_upper_arm_back',  // 5
237
  'left_lower_arm_front',  // 6
238
  'left_lower_arm_back',   // 7
239
  'right_lower_arm_front', // 8
240
  'right_lower_arm_back',  // 9
241
  'left_hand',            // 10
242
  'right_hand',           // 11
243
  'torso_front',          // 12
244
  'torso_back',           // 13
245
  'left_upper_leg_front', // 14
246
  'left_upper_leg_back',  // 15
247
  'right_upper_leg_front', // 16
248
  'right_upper_leg_back',  // 17
249
  'left_lower_leg_front', // 18
250
  'left_lower_leg_back',  // 19
251
  'right_lower_leg_front', // 20
252
  'right_lower_leg_back',  // 21
253
  'left_feet',            // 22
254
  'right_feet'            // 23
255
];
256

257
const version: string;
258
// Library version
259
```