0
# Image Processing
1

2
TensorFlow.js Node provides native image processing capabilities that allow you to decode various image formats directly into tensors and encode tensors back to image formats. These operations are optimized for performance using native implementations.
3

4
## Capabilities
5

6
### Image Decoding
7

8
#### Decode Any Image Format
9

10
Automatically detects and decodes BMP, GIF, JPEG, and PNG images.
11

12
```typescript { .api }
13
/**
14
 * Decode an image from a buffer into a tensor
15
 * @param contents - Image data as Uint8Array
16
 * @param channels - Number of color channels (0=auto, 1=grayscale, 3=RGB, 4=RGBA)
17
 * @param dtype - Output tensor data type (currently only 'int32' supported)
18
 * @param expandAnimations - If true, return 4D tensor for animated images
19
 * @returns 3D tensor [height, width, channels] or 4D [frames, height, width, channels] for animations
20
 */
21
function decodeImage(
22
  contents: Uint8Array,
23
  channels?: number,
24
  dtype?: 'int32',
25
  expandAnimations?: boolean
26
): Tensor3D | Tensor4D;
27
```
28

29
**Usage Example:**
30

31
```typescript
32
import * as tf from '@tensorflow/tfjs-node';
33
import * as fs from 'fs';
34

35
// Read image file
36
const imageBuffer = fs.readFileSync('path/to/image.jpg');
37
const imageArray = new Uint8Array(imageBuffer);
38

39
// Decode to tensor
40
const imageTensor = tf.node.decodeImage(imageArray, 3); // RGB channels
41
console.log('Image shape:', imageTensor.shape); // [height, width, 3]
42

43
// Process the tensor
44
const normalizedImage = imageTensor.div(255.0); // Normalize to [0,1]
45
```
46

47
#### Decode JPEG Images
48

49
Decode JPEG images with advanced options for quality and performance tuning.
50

51
```typescript { .api }
52
/**
53
 * Decode JPEG image with advanced options
54
 * @param contents - JPEG image data as Uint8Array
55
 * @param channels - Number of color channels (0=auto, 1=grayscale, 3=RGB)
56
 * @param ratio - Downscaling ratio for performance (1=full resolution)
57
 * @param fancyUpscaling - Use slower but higher quality chroma upscaling
58
 * @param tryRecoverTruncated - Attempt to recover truncated images
59
 * @param acceptableFraction - Minimum fraction of lines required for truncated recovery
60
 * @param dctMethod - DCT decompression method ("INTEGER_FAST" or "INTEGER_ACCURATE")
61
 * @returns 3D tensor [height, width, channels]
62
 */
63
function decodeJpeg(
64
  contents: Uint8Array,
65
  channels?: number,
66
  ratio?: number,
67
  fancyUpscaling?: boolean,
68
  tryRecoverTruncated?: boolean,
69
  acceptableFraction?: number,
70
  dctMethod?: string
71
): Tensor3D;
72
```
73

74
**Usage Example:**
75

76
```typescript
77
// High-quality JPEG decoding
78
const jpegTensor = tf.node.decodeJpeg(
79
  imageArray,
80
  3,  // RGB channels
81
  1,  // Full resolution
82
  true, // High-quality upscaling
83
  false, // Don't recover truncated
84
  1.0,   // Require 100% of lines
85
  "INTEGER_ACCURATE" // High-quality DCT
86
);
87

88
// Fast JPEG decoding for thumbnails
89
const thumbnailTensor = tf.node.decodeJpeg(
90
  imageArray,
91
  3,  // RGB channels
92
  4,  // 1/4 resolution (4x downscale)
93
  false, // Fast upscaling
94
  false, // Don't recover truncated
95
  1.0,   // Require 100% of lines
96
  "INTEGER_FAST" // Fast DCT
97
);
98
```
99

100
#### Decode PNG Images
101

102
Decode PNG images with support for various bit depths and transparency.
103

104
```typescript { .api }
105
/**
106
 * Decode PNG image
107
 * @param contents - PNG image data as Uint8Array
108
 * @param channels - Number of color channels (0=auto, 1=grayscale, 3=RGB, 4=RGBA)
109
 * @param dtype - Output tensor data type
110
 * @returns 3D tensor [height, width, channels]
111
 */
112
function decodePng(
113
  contents: Uint8Array,
114
  channels?: number,
115
  dtype?: 'int32' | 'uint8' | 'float32'
116
): Tensor3D;
117
```
118

119
#### Decode BMP Images
120

121
Decode Windows Bitmap (BMP) images.
122

123
```typescript { .api }
124
/**
125
 * Decode BMP image
126
 * @param contents - BMP image data as Uint8Array
127
 * @param channels - Number of color channels (0=auto, 1=grayscale, 3=RGB)
128
 * @returns 3D tensor [height, width, channels]
129
 */
130
function decodeBmp(
131
  contents: Uint8Array,
132
  channels?: number
133
): Tensor3D;
134
```
135

136
#### Decode GIF Images
137

138
Decode GIF images, including animated GIFs.
139

140
```typescript { .api }
141
/**
142
 * Decode GIF image (supports animation)
143
 * @param contents - GIF image data as Uint8Array
144
 * @returns 4D tensor [frames, height, width, channels] for animated GIFs
145
 */
146
function decodeGif(contents: Uint8Array): Tensor4D;
147
```
148

149
**Usage Example with Animated GIF:**
150

151
```typescript
152
const gifBuffer = fs.readFileSync('animated.gif');
153
const gifArray = new Uint8Array(gifBuffer);
154

155
// Decode animated GIF
156
const gifTensor = tf.node.decodeGif(gifArray);
157
console.log('GIF shape:', gifTensor.shape); // [frames, height, width, channels]
158

159
// Process each frame
160
const frameCount = gifTensor.shape[0];
161
for (let i = 0; i < frameCount; i++) {
162
  const frame = gifTensor.slice([i, 0, 0, 0], [1, -1, -1, -1]).squeeze([0]);
163
  // Process individual frame...
164
}
165
```
166

167
### Image Encoding
168

169
#### Encode JPEG Images
170

171
Encode tensors back to JPEG format with quality and format options.
172

173
```typescript { .api }
174
/**
175
 * Encode tensor as JPEG image
176
 * @param image - Input tensor [height, width, channels]
177
 * @param format - Output format (''=auto, 'grayscale', 'rgb')
178
 * @param quality - JPEG quality 0-100 (default: 95)
179
 * @param progressive - Use progressive JPEG encoding
180
 * @param optimizeSize - Optimize file size over quality
181
 * @param chromaDownsampling - Enable chroma subsampling for smaller files
182
 * @param densityUnit - DPI unit ('in' or 'cm')
183
 * @param xDensity - Horizontal DPI
184
 * @param yDensity - Vertical DPI
185
 * @param xmpMetadata - XMP metadata string
186
 * @returns Promise resolving to JPEG data as Uint8Array
187
 */
188
function encodeJpeg(
189
  image: Tensor3D,
190
  format?: '' | 'grayscale' | 'rgb',
191
  quality?: number,
192
  progressive?: boolean,
193
  optimizeSize?: boolean,
194
  chromaDownsampling?: boolean,
195
  densityUnit?: 'in' | 'cm',
196
  xDensity?: number,
197
  yDensity?: number,
198
  xmpMetadata?: string
199
): Promise<Uint8Array>;
200
```
201

202
**Usage Example:**
203

204
```typescript
205
// Create or process an image tensor
206
const imageTensor = tf.randomUniform([224, 224, 3], 0, 255).cast('int32');
207

208
// High-quality JPEG encoding
209
const highQualityJpeg = await tf.node.encodeJpeg(
210
  imageTensor,
211
  'rgb',     // RGB format
212
  95,        // High quality
213
  false,     // Not progressive
214
  false,     // Don't optimize size over quality
215
  false,     // No chroma downsampling
216
  'in',      // Inches
217
  300,       // 300 DPI horizontal
218
  300        // 300 DPI vertical
219
);
220

221
// Save to file
222
fs.writeFileSync('output-high-quality.jpg', Buffer.from(highQualityJpeg));
223

224
// Optimized JPEG for web
225
const webJpeg = await tf.node.encodeJpeg(
226
  imageTensor,
227
  'rgb',     // RGB format
228
  85,        // Good quality for web
229
  true,      // Progressive loading
230
  true,      // Optimize for size
231
  true       // Enable chroma downsampling
232
);
233

234
fs.writeFileSync('output-web.jpg', Buffer.from(webJpeg));
235
```
236

237
#### Encode PNG Images
238

239
Encode tensors as PNG images with compression options.
240

241
```typescript { .api }
242
/**
243
 * Encode tensor as PNG image
244
 * @param image - Input tensor [height, width, channels]
245
 * @param compression - Compression level 0-9 (0=none, 9=max, default: 1)
246
 * @returns Promise resolving to PNG data as Uint8Array
247
 */
248
function encodePng(
249
  image: Tensor3D,
250
  compression?: number
251
): Promise<Uint8Array>;
252
```
253

254
**Usage Example:**
255

256
```typescript
257
// Create or process an image tensor
258
const imageTensor = tf.randomUniform([224, 224, 4], 0, 255).cast('int32'); // RGBA
259

260
// Fast PNG encoding (low compression)
261
const fastPng = await tf.node.encodePng(imageTensor, 1);
262
fs.writeFileSync('output-fast.png', Buffer.from(fastPng));
263

264
// Maximum compression PNG
265
const compressedPng = await tf.node.encodePng(imageTensor, 9);
266
fs.writeFileSync('output-compressed.png', Buffer.from(compressedPng));
267
```
268

269
### Image Format Detection
270

271
#### Get Image Type
272

273
Determine the format of image data from file headers.
274

275
```typescript { .api }
276
/**
277
 * Detect image format from file header bytes
278
 * @param content - Image data as Uint8Array (first few bytes sufficient)
279
 * @returns Image format string ('jpeg', 'png', 'gif', 'bmp', or 'unknown')
280
 */
281
function getImageType(content: Uint8Array): string;
282
```
283

284
**Usage Example:**
285

286
```typescript
287
const imageBuffer = fs.readFileSync('unknown-image.data');
288
const imageArray = new Uint8Array(imageBuffer);
289

290
const imageType = tf.node.getImageType(imageArray);
291
console.log('Detected format:', imageType);
292

293
// Use appropriate decoder based on format
294
let decodedTensor;
295
switch (imageType) {
296
  case 'jpeg':
297
    decodedTensor = tf.node.decodeJpeg(imageArray);
298
    break;
299
  case 'png':
300
    decodedTensor = tf.node.decodePng(imageArray);
301
    break;
302
  case 'gif':
303
    decodedTensor = tf.node.decodeGif(imageArray);
304
    break;
305
  case 'bmp':
306
    decodedTensor = tf.node.decodeBmp(imageArray);
307
    break;
308
  default:
309
    throw new Error('Unsupported image format: ' + imageType);
310
}
311
```
312

313
## Types
314

315
```typescript { .api }
316
enum ImageType {
317
  JPEG = 'jpeg',
318
  PNG = 'png',
319
  GIF = 'gif',
320
  BMP = 'BMP'
321
}
322

323
type ImageFormat = '' | 'grayscale' | 'rgb';
324
type DensityUnit = 'in' | 'cm';
325
```
326

327
## Common Patterns
328

329
### Image Preprocessing Pipeline
330

331
```typescript
332
import * as tf from '@tensorflow/tfjs-node';
333
import * as fs from 'fs';
334

335
async function preprocessImage(imagePath: string): Promise<Tensor4D> {
336
  // Load image
337
  const imageBuffer = fs.readFileSync(imagePath);
338
  const imageArray = new Uint8Array(imageBuffer);
339
  
340
  // Decode image
341
  const imageTensor = tf.node.decodeImage(imageArray, 3); // RGB
342
  
343
  // Resize to model input size
344
  const resized = tf.image.resizeBilinear(imageTensor, [224, 224]);
345
  
346
  // Normalize to [0,1]
347
  const normalized = resized.div(255.0);
348
  
349
  // Add batch dimension
350
  const batched = normalized.expandDims(0);
351
  
352
  // Clean up intermediate tensors
353
  imageTensor.dispose();
354
  resized.dispose();
355
  normalized.dispose();
356
  
357
  return batched;
358
}
359
```
360

361
### Image Augmentation
362

363
```typescript
364
async function augmentAndSave(inputPath: string, outputPath: string) {
365
  const imageBuffer = fs.readFileSync(inputPath);
366
  const imageArray = new Uint8Array(imageBuffer);
367
  
368
  // Decode
369
  const original = tf.node.decodeImage(imageArray, 3);
370
  
371
  // Apply augmentations
372
  const flipped = tf.image.flipLeftRight(original);
373
  const rotated = tf.image.rot90(original);
374
  const brightened = tf.image.adjustBrightness(original, 0.2);
375
  
376
  // Encode and save variations
377
  const flippedJpeg = await tf.node.encodeJpeg(flipped.cast('int32'));
378
  const rotatedJpeg = await tf.node.encodeJpeg(rotated.cast('int32'));
379
  const brightenedJpeg = await tf.node.encodeJpeg(brightened.cast('int32'));
380
  
381
  fs.writeFileSync(`${outputPath}/flipped.jpg`, Buffer.from(flippedJpeg));
382
  fs.writeFileSync(`${outputPath}/rotated.jpg`, Buffer.from(rotatedJpeg));
383
  fs.writeFileSync(`${outputPath}/brightened.jpg`, Buffer.from(brightenedJpeg));
384
  
385
  // Clean up
386
  original.dispose();
387
  flipped.dispose();
388
  rotated.dispose();
389
  brightened.dispose();
390
}
391
```