0
# Utility Functions
1

2
Helper functions for pose manipulation, tensor operations, and coordinate transformations. These utilities support advanced use cases and custom processing workflows.
3

4
## Capabilities
5

6
### Pose Manipulation
7

8
Transform and manipulate pose keypoints for different coordinate systems and orientations.
9

10
```typescript { .api }
11
/**
12
 * Flips pose keypoints horizontally for mirrored images/videos
13
 * @param pose - Input pose to flip
14
 * @param imageWidth - Image width for coordinate transformation
15
 * @returns New pose with horizontally flipped keypoint coordinates
16
 */
17
function flipPoseHorizontal(pose: Pose, imageWidth: number): Pose;
18

19
interface Pose {
20
  /** Array of 17 body keypoints */
21
  keypoints: Keypoint[];
22
  /** Overall pose confidence score (0-1) */
23
  score: number;
24
}
25

26
interface Keypoint {
27
  /** Keypoint confidence score (0-1) */
28
  score: number;
29
  /** Pixel coordinates of the keypoint */
30
  position: Vector2D;
31
  /** Body part name */
32
  part: string;
33
}
34

35
interface Vector2D {
36
  x: number;
37
  y: number;
38
}
39
```
40

41
### Tensor Operations
42

43
Low-level tensor manipulation functions for custom processing pipelines.
44

45
```typescript { .api }
46
/**
47
 * Resizes tensor to target size with padding to maintain aspect ratio
48
 * @param imageTensor - Input image tensor (H x W x 3)
49
 * @param [targetH, targetW] - Target dimensions
50
 * @param flipHorizontal - Flip horizontally during resize
51
 * @returns Object with resized tensor and padding information
52
 */
53
function resizeAndPadTo(
54
  imageTensor: tf.Tensor3D,
55
  [targetH, targetW]: [number, number],
56
  flipHorizontal?: boolean
57
): {
58
  resizedAndPadded: tf.Tensor3D;
59
  paddedBy: [[number, number], [number, number]];
60
};
61

62
/**
63
 * Scales and crops tensor back to original input dimensions
64
 * @param tensor - Input tensor to transform
65
 * @param inputTensorShape - Original input dimensions [height, width]
66
 * @param resizedShape - Current tensor dimensions [height, width]
67
 * @param padding - Padding amounts from resizeAndPadTo
68
 * @param applySigmoidActivation - Apply sigmoid activation function
69
 * @returns Scaled and cropped tensor
70
 */
71
function scaleAndCropToInputTensorShape(
72
  tensor: tf.Tensor3D,
73
  inputTensorShape: [number, number],
74
  resizedShape: [number, number],
75
  padding: [[number, number], [number, number]],
76
  applySigmoidActivation?: boolean
77
): tf.Tensor3D;
78
```
79

80
## Usage Examples
81

82
### Pose Coordinate Transformation
83

84
```typescript
85
import * as bodyPix from '@tensorflow-models/body-pix';
86

87
const net = await bodyPix.load();
88
const video = document.getElementById('webcam'); // 640x480 webcam
89
const canvas = document.getElementById('output');
90

91
// Get segmentation with pose
92
const segmentation = await net.segmentPerson(video, { flipHorizontal: true });
93

94
// Process each detected pose
95
segmentation.allPoses.forEach((pose, index) => {
96
  console.log(`Pose ${index} score: ${pose.score}`);
97
  
98
  // Flip pose coordinates back to original orientation if needed
99
  const originalPose = bodyPix.flipPoseHorizontal(pose, video.videoWidth);
100
  
101
  // Draw keypoints on canvas
102
  drawPoseKeypoints(originalPose, canvas);
103
});
104

105
function drawPoseKeypoints(pose: Pose, canvas: HTMLCanvasElement) {
106
  const ctx = canvas.getContext('2d');
107
  ctx.fillStyle = 'red';
108
  
109
  pose.keypoints.forEach(keypoint => {
110
    if (keypoint.score > 0.5) {
111
      ctx.beginPath();
112
      ctx.arc(keypoint.position.x, keypoint.position.y, 3, 0, 2 * Math.PI);
113
      ctx.fill();
114
    }
115
  });
116
}
117
```
118

119
### Multiple Pose Transformations
120

121
```typescript
122
// Process multiple people with coordinate transformations
123
const peopleSegmentations = await net.segmentMultiPerson(imageElement);
124

125
const transformedPoses = peopleSegmentations.map(segmentation => {
126
  const pose = segmentation.pose;
127
  
128
  // Apply transformations based on your coordinate system
129
  const flippedPose = bodyPix.flipPoseHorizontal(pose, imageElement.width);
130
  
131
  // Scale pose to different coordinate system if needed
132
  const scaledPose = scalePoseKeypoints(flippedPose, 0.5, 0.5);
133
  
134
  return scaledPose;
135
});
136

137
function scalePoseKeypoints(pose: Pose, scaleX: number, scaleY: number): Pose {
138
  return {
139
    ...pose,
140
    keypoints: pose.keypoints.map(keypoint => ({
141
      ...keypoint,
142
      position: {
143
        x: keypoint.position.x * scaleX,
144
        y: keypoint.position.y * scaleY
145
      }
146
    }))
147
  };
148
}
149
```
150

151
### Custom Tensor Processing
152

153
```typescript
154
import * as tf from '@tensorflow/tfjs-core';
155

156
// Example: Custom preprocessing pipeline
157
async function customImageProcessing(imageElement: HTMLImageElement) {
158
  // Convert image to tensor
159
  const imageTensor = tf.browser.fromPixels(imageElement);
160
  
161
  // Resize and pad to model input size (513x513 for example)
162
  const { resizedAndPadded, paddedBy } = bodyPix.resizeAndPadTo(
163
    imageTensor, 
164
    [513, 513], 
165
    false
166
  );
167
  
168
  // Run your custom processing here
169
  const processedTensor = await customModelInference(resizedAndPadded);
170
  
171
  // Scale back to original dimensions
172
  const originalSize = bodyPix.scaleAndCropToInputTensorShape(
173
    processedTensor,
174
    [imageElement.height, imageElement.width],
175
    [513, 513],
176
    paddedBy,
177
    true
178
  );
179
  
180
  // Clean up tensors
181
  imageTensor.dispose();
182
  resizedAndPadded.dispose();
183
  processedTensor.dispose();
184
  
185
  return originalSize;
186
}
187

188
async function customModelInference(tensor: tf.Tensor3D): Promise<tf.Tensor3D> {
189
  // Your custom model inference logic here
190
  // This is just an example - apply your own transformations
191
  return tensor.mul(tf.scalar(2.0)); // Example: double the values
192
}
193
```
194

195
### Pose Analysis and Filtering
196

197
```typescript
198
// Filter poses by quality and analyze keypoint visibility
199
function analyzePoses(poses: Pose[]): Pose[] {
200
  return poses
201
    .filter(pose => pose.score > 0.5) // Only high-confidence poses
202
    .map(pose => {
203
      // Flip if needed for coordinate system consistency
204
      const flippedPose = bodyPix.flipPoseHorizontal(pose, 640);
205
      
206
      // Filter keypoints by confidence
207
      const confidentKeypoints = flippedPose.keypoints.filter(
208
        keypoint => keypoint.score > 0.3
209
      );
210
      
211
      return {
212
        ...flippedPose,
213
        keypoints: confidentKeypoints
214
      };
215
    });
216
}
217

218
// Usage with segmentation results
219
const segmentation = await net.segmentPerson(videoElement);
220
const qualityPoses = analyzePoses(segmentation.allPoses);
221

222
console.log(`Found ${qualityPoses.length} high-quality poses`);
223
qualityPoses.forEach(pose => {
224
  const visibleKeypoints = pose.keypoints.length;
225
  console.log(`Pose score: ${pose.score}, visible keypoints: ${visibleKeypoints}`);
226
});
227
```
228

229
### Coordinate System Conversion
230

231
```typescript
232
// Convert between different coordinate systems
233
function convertPoseCoordinates(
234
  pose: Pose, 
235
  fromWidth: number, 
236
  fromHeight: number,
237
  toWidth: number, 
238
  toHeight: number
239
): Pose {
240
  const scaleX = toWidth / fromWidth;
241
  const scaleY = toHeight / fromHeight;
242
  
243
  return {
244
    ...pose,
245
    keypoints: pose.keypoints.map(keypoint => ({
246
      ...keypoint,
247
      position: {
248
        x: keypoint.position.x * scaleX,
249
        y: keypoint.position.y * scaleY
250
      }
251
    }))
252
  };
253
}
254

255
// Example: Convert from video coordinates to canvas coordinates
256
const videoElement = document.getElementById('video'); // 1920x1080
257
const canvasElement = document.getElementById('canvas'); // 640x360
258

259
const segmentation = await net.segmentPerson(videoElement);
260
const canvasPoses = segmentation.allPoses.map(pose => 
261
  convertPoseCoordinates(
262
    pose,
263
    videoElement.videoWidth,
264
    videoElement.videoHeight, 
265
    canvasElement.width,
266
    canvasElement.height
267
  )
268
);
269
```
270

271
## Advanced Use Cases
272

273
### Custom Image Preprocessing
274
- Resize images while maintaining aspect ratio
275
- Apply custom padding strategies  
276
- Implement custom coordinate transformations
277

278
### Pose Data Analysis
279
- Transform poses between coordinate systems
280
- Filter poses by confidence thresholds
281
- Analyze keypoint visibility and quality
282

283
### Multi-Resolution Processing
284
- Process at different resolutions for speed/accuracy trade-offs
285
- Scale results back to original dimensions
286
- Handle aspect ratio preservation
287

288
### Integration with Custom Models
289
- Prepare tensors for custom TensorFlow.js models
290
- Apply consistent preprocessing pipelines
291
- Handle tensor memory management
292

293
## Performance Considerations
294

295
- **Tensor Memory**: Always dispose of tensors when done to prevent memory leaks
296
- **Coordinate Caching**: Cache transformed coordinates when processing video frames
297
- **Batch Processing**: Process multiple poses together when possible
298
- **Async Operations**: Use async/await properly with tensor operations to avoid blocking
299
- **Memory Monitoring**: Monitor GPU memory usage with `tf.memory()` when using many tensor operations