0
# Advanced APIs and Low-Level Functions
1
2
Low-level decoding functions and neural network classes for custom pose estimation implementations and advanced use cases.
3
4
## Capabilities
5
6
### Pose Decoding Functions
7
8
Low-level functions for decoding poses from neural network outputs. These are typically used internally by PoseNet but can be useful for custom implementations.
9
10
### Single Pose Decoding
11
12
Decode a single pose from raw neural network tensor outputs.
13
14
```typescript { .api }
15
/**
16
* Decode single pose from neural network heatmap and offset tensors
17
* @param heatmapScores - 3D tensor with shape [height, width, numParts] containing confidence scores
18
* @param offsets - 3D tensor with shape [height, width, numParts * 2] containing offset vectors
19
* @param outputStride - Output stride used by the model (8, 16, or 32)
20
* @returns Promise resolving to decoded pose with keypoints and score
21
*/
22
function decodeSinglePose(
23
heatmapScores: tf.Tensor3D,
24
offsets: tf.Tensor3D,
25
outputStride: PoseNetOutputStride
26
): Promise<Pose>;
27
```
28
29
**Usage Examples:**
30
31
```typescript
32
import { decodeSinglePose } from '@tensorflow-models/posenet';
33
import * as tf from '@tensorflow/tfjs-core';
34
35
// Custom model inference and pose decoding
36
async function customSinglePoseInference(model: tf.GraphModel, input: tf.Tensor3D) {
37
// Run custom model inference
38
const outputs = model.predict(input) as tf.Tensor4D[];
39
const [heatmapScores, offsets] = outputs.map(output => tf.squeeze(output, [0]));
40
41
// Decode pose using PoseNet decoder
42
const pose = await decodeSinglePose(
43
heatmapScores as tf.Tensor3D,
44
offsets as tf.Tensor3D,
45
16 // outputStride
46
);
47
48
// Clean up tensors
49
heatmapScores.dispose();
50
offsets.dispose();
51
52
return pose;
53
}
54
55
// Process pre-computed model outputs
56
async function processSavedOutputs(heatmapData: Float32Array, offsetData: Float32Array) {
57
const height = 33, width = 33, numKeypoints = 17;
58
59
// Create tensors from saved data
60
const heatmapScores = tf.tensor3d(heatmapData, [height, width, numKeypoints]);
61
const offsets = tf.tensor3d(offsetData, [height, width, numKeypoints * 2]);
62
63
const pose = await decodeSinglePose(heatmapScores, offsets, 16);
64
65
heatmapScores.dispose();
66
offsets.dispose();
67
68
return pose;
69
}
70
```
71
72
### Multiple Pose Decoding
73
74
Decode multiple poses from raw neural network tensor buffers with non-maximum suppression.
75
76
```typescript { .api }
77
/**
78
* Decode multiple poses from neural network tensor buffers using fast greedy decoding
79
* @param scoresBuffer - TensorBuffer3D containing heatmap confidence scores
80
* @param offsetsBuffer - TensorBuffer3D containing offset vectors
81
* @param displacementsFwdBuffer - TensorBuffer3D containing forward displacement vectors
82
* @param displacementsBwdBuffer - TensorBuffer3D containing backward displacement vectors
83
* @param outputStride - Output stride used by the model
84
* @param maxPoseDetections - Maximum number of poses to detect
85
* @param scoreThreshold - Minimum confidence score for pose detection (default: 0.5)
86
* @param nmsRadius - Non-maximum suppression radius in pixels (default: 20)
87
* @returns Array of decoded poses with keypoints and scores
88
*/
89
function decodeMultiplePoses(
90
scoresBuffer: TensorBuffer3D,
91
offsetsBuffer: TensorBuffer3D,
92
displacementsFwdBuffer: TensorBuffer3D,
93
displacementsBwdBuffer: TensorBuffer3D,
94
outputStride: number,
95
maxPoseDetections: number,
96
scoreThreshold?: number,
97
nmsRadius?: number
98
): Pose[];
99
```
100
101
**Usage Examples:**
102
103
```typescript
104
import { decodeMultiplePoses } from '@tensorflow-models/posenet';
105
import * as tf from '@tensorflow/tfjs-core';
106
107
// Custom multi-pose inference with fine-tuned parameters
108
async function customMultiPoseInference(
109
model: tf.GraphModel,
110
input: tf.Tensor3D,
111
customScoreThreshold: number = 0.7
112
) {
113
// Run model inference
114
const outputs = model.predict(input) as tf.Tensor4D[];
115
const [heatmapScores, offsets, displacementFwd, displacementBwd] =
116
outputs.map(output => tf.squeeze(output, [0]));
117
118
// Convert to tensor buffers for decoding
119
const [scoresBuffer, offsetsBuffer, displacementsFwdBuffer, displacementsBwdBuffer] =
120
await Promise.all([
121
heatmapScores.buffer(),
122
offsets.buffer(),
123
displacementFwd.buffer(),
124
displacementBwd.buffer()
125
]);
126
127
// Decode with custom parameters
128
const poses = decodeMultiplePoses(
129
scoresBuffer as TensorBuffer3D,
130
offsetsBuffer as TensorBuffer3D,
131
displacementsFwdBuffer as TensorBuffer3D,
132
displacementsBwdBuffer as TensorBuffer3D,
133
16, // outputStride
134
10, // maxPoseDetections
135
customScoreThreshold, // higher threshold for quality
136
15 // smaller NMS radius for closer detection
137
);
138
139
// Clean up tensors
140
heatmapScores.dispose();
141
offsets.dispose();
142
displacementFwd.dispose();
143
displacementBwd.dispose();
144
145
return poses;
146
}
147
148
// Batch processing of pre-computed buffers
149
function processBufferBatch(tensorBuffers: {
150
scores: TensorBuffer3D;
151
offsets: TensorBuffer3D;
152
displacementsFwd: TensorBuffer3D;
153
displacementsBwd: TensorBuffer3D;
154
}[]): Pose[][] {
155
return tensorBuffers.map(buffers =>
156
decodeMultiplePoses(
157
buffers.scores,
158
buffers.offsets,
159
buffers.displacementsFwd,
160
buffers.displacementsBwd,
161
16, // outputStride
162
5, // maxPoseDetections
163
0.6, // scoreThreshold
164
25 // nmsRadius
165
)
166
);
167
}
168
```
169
170
### Neural Network Models
171
172
Low-level neural network model classes for custom implementations.
173
174
### MobileNet Model
175
176
MobileNetV1-based neural network model for pose estimation.
177
178
```typescript { .api }
179
/**
180
* MobileNetV1 neural network model implementation
181
* Extends BaseModel with MobileNet-specific preprocessing and output handling
182
*/
183
class MobileNet extends BaseModel {
184
/** Preprocess input tensor for MobileNet model (normalize to [-1, 1]) */
185
preprocessInput(input: tf.Tensor3D): tf.Tensor3D;
186
187
/** Map model outputs to named tensor results */
188
nameOutputResults(results: tf.Tensor3D[]): {
189
heatmap: tf.Tensor3D;
190
offsets: tf.Tensor3D;
191
displacementFwd: tf.Tensor3D;
192
displacementBwd: tf.Tensor3D;
193
};
194
}
195
```
196
197
**Usage Examples:**
198
199
```typescript
200
import { MobileNet } from '@tensorflow-models/posenet';
201
import * as tfconv from '@tensorflow/tfjs-converter';
202
203
// Create custom MobileNet instance
204
async function createCustomMobileNet(modelUrl: string) {
205
const graphModel = await tfconv.loadGraphModel(modelUrl);
206
const mobileNet = new MobileNet(graphModel, 16); // outputStride = 16
207
208
return mobileNet;
209
}
210
211
// Use MobileNet for custom inference pipeline
212
async function customInference(mobileNet: MobileNet, inputTensor: tf.Tensor3D) {
213
// Get model predictions
214
const predictions = mobileNet.predict(inputTensor);
215
216
// Extract individual tensors
217
const { heatmapScores, offsets, displacementFwd, displacementBwd } = predictions;
218
219
// Process results as needed
220
const maxConfidence = await heatmapScores.max().data();
221
console.log('Maximum confidence:', maxConfidence[0]);
222
223
// Clean up
224
heatmapScores.dispose();
225
offsets.dispose();
226
displacementFwd.dispose();
227
displacementBwd.dispose();
228
}
229
```
230
231
### Base Model Interface
232
233
Abstract base class for neural network models used in pose estimation.
234
235
```typescript { .api }
236
/**
237
* Abstract base class for pose estimation neural network models
238
* Provides common interface for MobileNet and ResNet implementations
239
*/
240
abstract class BaseModel {
241
/** Underlying TensorFlow.js GraphModel */
242
protected readonly model: tfconv.GraphModel;
243
244
/** Output stride of the model */
245
public readonly outputStride: PoseNetOutputStride;
246
247
/** Abstract method for model-specific input preprocessing */
248
abstract preprocessInput(input: tf.Tensor3D): tf.Tensor3D;
249
250
/** Abstract method for mapping model outputs to named results */
251
abstract nameOutputResults(results: tf.Tensor3D[]): {
252
heatmap: tf.Tensor3D;
253
offsets: tf.Tensor3D;
254
displacementFwd: tf.Tensor3D;
255
displacementBwd: tf.Tensor3D;
256
};
257
258
/** Run inference and return pose estimation tensors */
259
predict(input: tf.Tensor3D): {
260
heatmapScores: tf.Tensor3D;
261
offsets: tf.Tensor3D;
262
displacementFwd: tf.Tensor3D;
263
displacementBwd: tf.Tensor3D;
264
};
265
266
/** Release GPU/CPU memory allocated by the model */
267
dispose(): void;
268
}
269
```
270
271
## Supporting Types
272
273
```typescript { .api }
274
type TensorBuffer3D = tf.TensorBuffer<tf.Rank.R3>;
275
type PoseNetOutputStride = 32 | 16 | 8;
276
277
interface Pose {
278
keypoints: Keypoint[];
279
score: number;
280
}
281
282
interface Keypoint {
283
score: number;
284
position: Vector2D;
285
part: string;
286
}
287
288
interface Vector2D {
289
x: number;
290
y: number;
291
}
292
```
293
294
## Use Cases
295
296
These advanced APIs are useful for:
297
298
- **Custom Model Integration**: Using PoseNet decoders with custom trained models
299
- **Performance Optimization**: Batch processing of pre-computed tensor outputs
300
- **Research and Development**: Experimenting with different decoding parameters
301
- **Pipeline Integration**: Incorporating pose estimation into larger ML pipelines
302
- **Mobile Optimization**: Using MobileNet directly for memory-constrained environments
303
304
## Performance Considerations
305
306
- **Memory Management**: Always dispose of tensors after use to prevent memory leaks
307
- **Batch Processing**: Process multiple inputs together for better GPU utilization
308
- **Parameter Tuning**: Adjust scoreThreshold and nmsRadius based on your use case
309
- **Model Selection**: Choose between MobileNet (speed) and ResNet (accuracy) based on requirements