tessl/npm-tensorflow-models--posenet
Pretrained PoseNet model in TensorFlow.js for real-time human pose estimation from images and video streams
—
Pending
advanced-apis.mddocs/
Advanced APIs and Low-Level Functions
Low-level decoding functions and neural network classes for custom pose estimation implementations and advanced use cases.
Capabilities
Pose Decoding Functions
Low-level functions for decoding poses from neural network outputs. These are typically used internally by PoseNet but can be useful for custom implementations.
Single Pose Decoding
Decode a single pose from raw neural network tensor outputs.
/**
* Decode single pose from neural network heatmap and offset tensors
* @param heatmapScores - 3D tensor with shape [height, width, numParts] containing confidence scores
* @param offsets - 3D tensor with shape [height, width, numParts * 2] containing offset vectors
* @param outputStride - Output stride used by the model (8, 16, or 32)
* @returns Promise resolving to decoded pose with keypoints and score
*/
function decodeSinglePose(
heatmapScores: tf.Tensor3D,
offsets: tf.Tensor3D,
outputStride: PoseNetOutputStride
): Promise<Pose>;Usage Examples:
import { decodeSinglePose } from '@tensorflow-models/posenet';
import * as tf from '@tensorflow/tfjs-core';
// Custom model inference and pose decoding
async function customSinglePoseInference(model: tf.GraphModel, input: tf.Tensor3D) {
// Run custom model inference
const outputs = model.predict(input) as tf.Tensor4D[];
const [heatmapScores, offsets] = outputs.map(output => tf.squeeze(output, [0]));
// Decode pose using PoseNet decoder
const pose = await decodeSinglePose(
heatmapScores as tf.Tensor3D,
offsets as tf.Tensor3D,
16 // outputStride
);
// Clean up tensors
heatmapScores.dispose();
offsets.dispose();
return pose;
}
// Process pre-computed model outputs
async function processSavedOutputs(heatmapData: Float32Array, offsetData: Float32Array) {
const height = 33, width = 33, numKeypoints = 17;
// Create tensors from saved data
const heatmapScores = tf.tensor3d(heatmapData, [height, width, numKeypoints]);
const offsets = tf.tensor3d(offsetData, [height, width, numKeypoints * 2]);
const pose = await decodeSinglePose(heatmapScores, offsets, 16);
heatmapScores.dispose();
offsets.dispose();
return pose;
}Multiple Pose Decoding
Decode multiple poses from raw neural network tensor buffers with non-maximum suppression.
/**
* Decode multiple poses from neural network tensor buffers using fast greedy decoding
* @param scoresBuffer - TensorBuffer3D containing heatmap confidence scores
* @param offsetsBuffer - TensorBuffer3D containing offset vectors
* @param displacementsFwdBuffer - TensorBuffer3D containing forward displacement vectors
* @param displacementsBwdBuffer - TensorBuffer3D containing backward displacement vectors
* @param outputStride - Output stride used by the model
* @param maxPoseDetections - Maximum number of poses to detect
* @param scoreThreshold - Minimum confidence score for pose detection (default: 0.5)
* @param nmsRadius - Non-maximum suppression radius in pixels (default: 20)
* @returns Array of decoded poses with keypoints and scores
*/
function decodeMultiplePoses(
scoresBuffer: TensorBuffer3D,
offsetsBuffer: TensorBuffer3D,
displacementsFwdBuffer: TensorBuffer3D,
displacementsBwdBuffer: TensorBuffer3D,
outputStride: number,
maxPoseDetections: number,
scoreThreshold?: number,
nmsRadius?: number
): Pose[];Usage Examples:
import { decodeMultiplePoses } from '@tensorflow-models/posenet';
import * as tf from '@tensorflow/tfjs-core';
// Custom multi-pose inference with fine-tuned parameters
async function customMultiPoseInference(
model: tf.GraphModel,
input: tf.Tensor3D,
customScoreThreshold: number = 0.7
) {
// Run model inference
const outputs = model.predict(input) as tf.Tensor4D[];
const [heatmapScores, offsets, displacementFwd, displacementBwd] =
outputs.map(output => tf.squeeze(output, [0]));
// Convert to tensor buffers for decoding
const [scoresBuffer, offsetsBuffer, displacementsFwdBuffer, displacementsBwdBuffer] =
await Promise.all([
heatmapScores.buffer(),
offsets.buffer(),
displacementFwd.buffer(),
displacementBwd.buffer()
]);
// Decode with custom parameters
const poses = decodeMultiplePoses(
scoresBuffer as TensorBuffer3D,
offsetsBuffer as TensorBuffer3D,
displacementsFwdBuffer as TensorBuffer3D,
displacementsBwdBuffer as TensorBuffer3D,
16, // outputStride
10, // maxPoseDetections
customScoreThreshold, // higher threshold for quality
15 // smaller NMS radius for closer detection
);
// Clean up tensors
heatmapScores.dispose();
offsets.dispose();
displacementFwd.dispose();
displacementBwd.dispose();
return poses;
}
// Batch processing of pre-computed buffers
function processBufferBatch(tensorBuffers: {
scores: TensorBuffer3D;
offsets: TensorBuffer3D;
displacementsFwd: TensorBuffer3D;
displacementsBwd: TensorBuffer3D;
}[]): Pose[][] {
return tensorBuffers.map(buffers =>
decodeMultiplePoses(
buffers.scores,
buffers.offsets,
buffers.displacementsFwd,
buffers.displacementsBwd,
16, // outputStride
5, // maxPoseDetections
0.6, // scoreThreshold
25 // nmsRadius
)
);
}Neural Network Models
Low-level neural network model classes for custom implementations.
MobileNet Model
MobileNetV1-based neural network model for pose estimation.
/**
* MobileNetV1 neural network model implementation
* Extends BaseModel with MobileNet-specific preprocessing and output handling
*/
class MobileNet extends BaseModel {
/** Preprocess input tensor for MobileNet model (normalize to [-1, 1]) */
preprocessInput(input: tf.Tensor3D): tf.Tensor3D;
/** Map model outputs to named tensor results */
nameOutputResults(results: tf.Tensor3D[]): {
heatmap: tf.Tensor3D;
offsets: tf.Tensor3D;
displacementFwd: tf.Tensor3D;
displacementBwd: tf.Tensor3D;
};
}Usage Examples:
import { MobileNet } from '@tensorflow-models/posenet';
import * as tfconv from '@tensorflow/tfjs-converter';
// Create custom MobileNet instance
async function createCustomMobileNet(modelUrl: string) {
const graphModel = await tfconv.loadGraphModel(modelUrl);
const mobileNet = new MobileNet(graphModel, 16); // outputStride = 16
return mobileNet;
}
// Use MobileNet for custom inference pipeline
async function customInference(mobileNet: MobileNet, inputTensor: tf.Tensor3D) {
// Get model predictions
const predictions = mobileNet.predict(inputTensor);
// Extract individual tensors
const { heatmapScores, offsets, displacementFwd, displacementBwd } = predictions;
// Process results as needed
const maxConfidence = await heatmapScores.max().data();
console.log('Maximum confidence:', maxConfidence[0]);
// Clean up
heatmapScores.dispose();
offsets.dispose();
displacementFwd.dispose();
displacementBwd.dispose();
}Base Model Interface
Abstract base class for neural network models used in pose estimation.
/**
* Abstract base class for pose estimation neural network models
* Provides common interface for MobileNet and ResNet implementations
*/
abstract class BaseModel {
/** Underlying TensorFlow.js GraphModel */
protected readonly model: tfconv.GraphModel;
/** Output stride of the model */
public readonly outputStride: PoseNetOutputStride;
/** Abstract method for model-specific input preprocessing */
abstract preprocessInput(input: tf.Tensor3D): tf.Tensor3D;
/** Abstract method for mapping model outputs to named results */
abstract nameOutputResults(results: tf.Tensor3D[]): {
heatmap: tf.Tensor3D;
offsets: tf.Tensor3D;
displacementFwd: tf.Tensor3D;
displacementBwd: tf.Tensor3D;
};
/** Run inference and return pose estimation tensors */
predict(input: tf.Tensor3D): {
heatmapScores: tf.Tensor3D;
offsets: tf.Tensor3D;
displacementFwd: tf.Tensor3D;
displacementBwd: tf.Tensor3D;
};
/** Release GPU/CPU memory allocated by the model */
dispose(): void;
}Supporting Types
type TensorBuffer3D = tf.TensorBuffer<tf.Rank.R3>;
type PoseNetOutputStride = 32 | 16 | 8;
interface Pose {
keypoints: Keypoint[];
score: number;
}
interface Keypoint {
score: number;
position: Vector2D;
part: string;
}
interface Vector2D {
x: number;
y: number;
}Use Cases
These advanced APIs are useful for:
- Custom Model Integration: Using PoseNet decoders with custom trained models
- Performance Optimization: Batch processing of pre-computed tensor outputs
- Research and Development: Experimenting with different decoding parameters
- Pipeline Integration: Incorporating pose estimation into larger ML pipelines
- Mobile Optimization: Using MobileNet directly for memory-constrained environments
Performance Considerations
- Memory Management: Always dispose of tensors after use to prevent memory leaks
- Batch Processing: Process multiple inputs together for better GPU utilization
- Parameter Tuning: Adjust scoreThreshold and nmsRadius based on your use case
- Model Selection: Choose between MobileNet (speed) and ResNet (accuracy) based on requirements
Install with Tessl CLI
npx tessl i tessl/npm-tensorflow-models--posenet