tessl/npm-tensorflow-models--body-pix
Real-time person and body part segmentation using TensorFlow.js for web browsers with machine learning models.
—
Pending
utilities.mddocs/
Utility Functions
Helper functions for pose manipulation, tensor operations, and coordinate transformations. These utilities support advanced use cases and custom processing workflows.
Capabilities
Pose Manipulation
Transform and manipulate pose keypoints for different coordinate systems and orientations.
/**
* Flips pose keypoints horizontally for mirrored images/videos
* @param pose - Input pose to flip
* @param imageWidth - Image width for coordinate transformation
* @returns New pose with horizontally flipped keypoint coordinates
*/
function flipPoseHorizontal(pose: Pose, imageWidth: number): Pose;
interface Pose {
/** Array of 17 body keypoints */
keypoints: Keypoint[];
/** Overall pose confidence score (0-1) */
score: number;
}
interface Keypoint {
/** Keypoint confidence score (0-1) */
score: number;
/** Pixel coordinates of the keypoint */
position: Vector2D;
/** Body part name */
part: string;
}
interface Vector2D {
x: number;
y: number;
}Tensor Operations
Low-level tensor manipulation functions for custom processing pipelines.
/**
* Resizes tensor to target size with padding to maintain aspect ratio
* @param imageTensor - Input image tensor (H x W x 3)
* @param [targetH, targetW] - Target dimensions
* @param flipHorizontal - Flip horizontally during resize
* @returns Object with resized tensor and padding information
*/
function resizeAndPadTo(
imageTensor: tf.Tensor3D,
[targetH, targetW]: [number, number],
flipHorizontal?: boolean
): {
resizedAndPadded: tf.Tensor3D;
paddedBy: [[number, number], [number, number]];
};
/**
* Scales and crops tensor back to original input dimensions
* @param tensor - Input tensor to transform
* @param inputTensorShape - Original input dimensions [height, width]
* @param resizedShape - Current tensor dimensions [height, width]
* @param padding - Padding amounts from resizeAndPadTo
* @param applySigmoidActivation - Apply sigmoid activation function
* @returns Scaled and cropped tensor
*/
function scaleAndCropToInputTensorShape(
tensor: tf.Tensor3D,
inputTensorShape: [number, number],
resizedShape: [number, number],
padding: [[number, number], [number, number]],
applySigmoidActivation?: boolean
): tf.Tensor3D;Usage Examples
Pose Coordinate Transformation
import * as bodyPix from '@tensorflow-models/body-pix';
const net = await bodyPix.load();
const video = document.getElementById('webcam'); // 640x480 webcam
const canvas = document.getElementById('output');
// Get segmentation with pose
const segmentation = await net.segmentPerson(video, { flipHorizontal: true });
// Process each detected pose
segmentation.allPoses.forEach((pose, index) => {
console.log(`Pose ${index} score: ${pose.score}`);
// Flip pose coordinates back to original orientation if needed
const originalPose = bodyPix.flipPoseHorizontal(pose, video.videoWidth);
// Draw keypoints on canvas
drawPoseKeypoints(originalPose, canvas);
});
function drawPoseKeypoints(pose: Pose, canvas: HTMLCanvasElement) {
const ctx = canvas.getContext('2d');
ctx.fillStyle = 'red';
pose.keypoints.forEach(keypoint => {
if (keypoint.score > 0.5) {
ctx.beginPath();
ctx.arc(keypoint.position.x, keypoint.position.y, 3, 0, 2 * Math.PI);
ctx.fill();
}
});
}Multiple Pose Transformations
// Process multiple people with coordinate transformations
const peopleSegmentations = await net.segmentMultiPerson(imageElement);
const transformedPoses = peopleSegmentations.map(segmentation => {
const pose = segmentation.pose;
// Apply transformations based on your coordinate system
const flippedPose = bodyPix.flipPoseHorizontal(pose, imageElement.width);
// Scale pose to different coordinate system if needed
const scaledPose = scalePoseKeypoints(flippedPose, 0.5, 0.5);
return scaledPose;
});
function scalePoseKeypoints(pose: Pose, scaleX: number, scaleY: number): Pose {
return {
...pose,
keypoints: pose.keypoints.map(keypoint => ({
...keypoint,
position: {
x: keypoint.position.x * scaleX,
y: keypoint.position.y * scaleY
}
}))
};
}Custom Tensor Processing
import * as tf from '@tensorflow/tfjs-core';
// Example: Custom preprocessing pipeline
async function customImageProcessing(imageElement: HTMLImageElement) {
// Convert image to tensor
const imageTensor = tf.browser.fromPixels(imageElement);
// Resize and pad to model input size (513x513 for example)
const { resizedAndPadded, paddedBy } = bodyPix.resizeAndPadTo(
imageTensor,
[513, 513],
false
);
// Run your custom processing here
const processedTensor = await customModelInference(resizedAndPadded);
// Scale back to original dimensions
const originalSize = bodyPix.scaleAndCropToInputTensorShape(
processedTensor,
[imageElement.height, imageElement.width],
[513, 513],
paddedBy,
true
);
// Clean up tensors
imageTensor.dispose();
resizedAndPadded.dispose();
processedTensor.dispose();
return originalSize;
}
async function customModelInference(tensor: tf.Tensor3D): Promise<tf.Tensor3D> {
// Your custom model inference logic here
// This is just an example - apply your own transformations
return tensor.mul(tf.scalar(2.0)); // Example: double the values
}Pose Analysis and Filtering
// Filter poses by quality and analyze keypoint visibility
function analyzePoses(poses: Pose[]): Pose[] {
return poses
.filter(pose => pose.score > 0.5) // Only high-confidence poses
.map(pose => {
// Flip if needed for coordinate system consistency
const flippedPose = bodyPix.flipPoseHorizontal(pose, 640);
// Filter keypoints by confidence
const confidentKeypoints = flippedPose.keypoints.filter(
keypoint => keypoint.score > 0.3
);
return {
...flippedPose,
keypoints: confidentKeypoints
};
});
}
// Usage with segmentation results
const segmentation = await net.segmentPerson(videoElement);
const qualityPoses = analyzePoses(segmentation.allPoses);
console.log(`Found ${qualityPoses.length} high-quality poses`);
qualityPoses.forEach(pose => {
const visibleKeypoints = pose.keypoints.length;
console.log(`Pose score: ${pose.score}, visible keypoints: ${visibleKeypoints}`);
});Coordinate System Conversion
// Convert between different coordinate systems
function convertPoseCoordinates(
pose: Pose,
fromWidth: number,
fromHeight: number,
toWidth: number,
toHeight: number
): Pose {
const scaleX = toWidth / fromWidth;
const scaleY = toHeight / fromHeight;
return {
...pose,
keypoints: pose.keypoints.map(keypoint => ({
...keypoint,
position: {
x: keypoint.position.x * scaleX,
y: keypoint.position.y * scaleY
}
}))
};
}
// Example: Convert from video coordinates to canvas coordinates
const videoElement = document.getElementById('video'); // 1920x1080
const canvasElement = document.getElementById('canvas'); // 640x360
const segmentation = await net.segmentPerson(videoElement);
const canvasPoses = segmentation.allPoses.map(pose =>
convertPoseCoordinates(
pose,
videoElement.videoWidth,
videoElement.videoHeight,
canvasElement.width,
canvasElement.height
)
);Advanced Use Cases
Custom Image Preprocessing
- Resize images while maintaining aspect ratio
- Apply custom padding strategies
- Implement custom coordinate transformations
Pose Data Analysis
- Transform poses between coordinate systems
- Filter poses by confidence thresholds
- Analyze keypoint visibility and quality
Multi-Resolution Processing
- Process at different resolutions for speed/accuracy trade-offs
- Scale results back to original dimensions
- Handle aspect ratio preservation
Integration with Custom Models
- Prepare tensors for custom TensorFlow.js models
- Apply consistent preprocessing pipelines
- Handle tensor memory management
Performance Considerations
- Tensor Memory: Always dispose of tensors when done to prevent memory leaks
- Coordinate Caching: Cache transformed coordinates when processing video frames
- Batch Processing: Process multiple poses together when possible
- Async Operations: Use async/await properly with tensor operations to avoid blocking
- Memory Monitoring: Monitor GPU memory usage with
tf.memory()when using many tensor operations
Install with Tessl CLI
npx tessl i tessl/npm-tensorflow-models--body-pix