0
# Buffer Writing (Node.js)
1

2
Synchronous buffer-based CAR writing for performance-critical scenarios where the final CAR size is known in advance. Provides direct buffer manipulation with precise memory control and no async overhead. **This functionality is only available in Node.js.**
3

4
## Capabilities
5

6
### CarBufferWriter Class
7

8
Synchronous CAR writer that writes directly to a pre-allocated buffer.
9

10
```typescript { .api }
11
/**
12
 * Synchronous CAR writer for pre-allocated buffers
13
 * Provides precise memory control and no async overhead
14
 * Suitable when final CAR size is known in advance
15
 * Node.js only - not available in browser environments
16
 */
17
interface CarBufferWriter {
18
  /** Buffer being written to */
19
  readonly bytes: Uint8Array;
20
  /** Current root CIDs */
21
  readonly roots: CID[];
22
  
23
  /** Add a root CID to the header, optionally resizing header space */
24
  addRoot(root: CID, options?: { resize?: boolean }): CarBufferWriter;
25
  
26
  /** Write a block to the buffer, throws if insufficient space */
27
  write(block: Block): CarBufferWriter;
28
  
29
  /** Finalize CAR and return final byte array, optionally resizing */
30
  close(options?: { resize?: boolean }): Uint8Array;
31
}
32

33
/**
34
 * Options for creating a buffer writer
35
 */
36
interface CarBufferWriterOptions {
37
  /** Initial root CIDs (default: []) */
38
  roots?: CID[];
39
  /** Byte offset in buffer to start writing (default: 0) */
40
  byteOffset?: number;
41
  /** Number of bytes to use from buffer (default: buffer.byteLength) */
42
  byteLength?: number;
43
  /** Reserved space for header (default: calculated from roots) */
44
  headerSize?: number;
45
}
46
```
47

48
### Creating Buffer Writers
49

50
Create buffer writers with precise memory allocation.
51

52
```typescript { .api }
53
/**
54
 * Create buffer writer from ArrayBuffer with optional configuration
55
 * @param buffer - ArrayBuffer to write CAR data into
56
 * @param options - Configuration options
57
 * @returns CarBufferWriter instance
58
 */
59
function createWriter(buffer: ArrayBuffer, options?: CarBufferWriterOptions): CarBufferWriter;
60
```
61

62
**Usage Examples:**
63

64
```typescript
65
import * as CarBufferWriter from "@ipld/car/buffer-writer";
66

67
// Create buffer with estimated size
68
const estimatedSize = 1024 * 1024; // 1MB
69
const buffer = new ArrayBuffer(estimatedSize);
70

71
// Create writer with known roots
72
const writer = CarBufferWriter.createWriter(buffer, {
73
  roots: [rootCid1, rootCid2],
74
  headerSize: CarBufferWriter.headerLength({ roots: [rootCid1, rootCid2] })
75
});
76

77
// Write blocks
78
writer
79
  .write({ cid: blockCid1, bytes: blockData1 })
80
  .write({ cid: blockCid2, bytes: blockData2 })
81
  .write({ cid: blockCid3, bytes: blockData3 });
82

83
// Finalize and get result
84
const carBytes = writer.close();
85
console.log(`Created CAR: ${carBytes.length} bytes`);
86
```
87

88
### Size Calculation Functions
89

90
Calculate buffer sizes and header requirements.
91

92
```typescript { .api }
93
/**
94
 * Calculate bytes needed for storing a block in CAR format
95
 * @param block - Block to calculate size for
96
 * @returns Number of bytes needed
97
 */
98
function blockLength(block: Block): number;
99

100
/**
101
 * Calculate header length for given roots
102
 * @param options - Object containing roots array
103
 * @returns Header length in bytes
104
 */
105
function headerLength(options: { roots: CID[] }): number;
106

107
/**
108
 * Calculate header length from root CID byte lengths
109
 * @param rootLengths - Array of root CID byte lengths
110
 * @returns Header length in bytes
111
 */
112
function calculateHeaderLength(rootLengths: number[]): number;
113

114
/**
115
 * Estimate header length for a given number of roots
116
 * @param rootCount - Number of root CIDs
117
 * @param rootByteLength - Expected bytes per root CID (default: 36)
118
 * @returns Estimated header length in bytes
119
 */
120
function estimateHeaderLength(rootCount: number, rootByteLength?: number): number;
121
```
122

123
**Usage Examples:**
124

125
```typescript
126
import * as CarBufferWriter from "@ipld/car/buffer-writer";
127

128
// Calculate exact buffer size needed
129
const blocks = [
130
  { cid: cid1, bytes: data1 },
131
  { cid: cid2, bytes: data2 },
132
  { cid: cid3, bytes: data3 }
133
];
134

135
const roots = [cid1];
136

137
// Calculate total size needed
138
const headerSize = CarBufferWriter.headerLength({ roots });
139
const blockSizes = blocks.reduce(
140
  (total, block) => total + CarBufferWriter.blockLength(block), 
141
  0
142
);
143
const totalSize = headerSize + blockSizes;
144

145
// Create perfectly sized buffer
146
const buffer = new ArrayBuffer(totalSize);
147
const writer = CarBufferWriter.createWriter(buffer, { 
148
  roots,
149
  headerSize 
150
});
151

152
// Write all blocks
153
for (const block of blocks) {
154
  writer.write(block);
155
}
156

157
const carBytes = writer.close();
158
console.log(`Perfect fit: ${carBytes.length} === ${totalSize}`);
159
```
160

161
### Dynamic Header Resizing
162

163
Handle scenarios where root count changes during writing.
164

165
```typescript
166
import * as CarBufferWriter from "@ipld/car/buffer-writer";
167

168
// Start with estimated header size
169
const buffer = new ArrayBuffer(1024 * 1024);
170
const estimatedHeaderSize = CarBufferWriter.estimateHeaderLength(5); // Expect ~5 roots
171

172
const writer = CarBufferWriter.createWriter(buffer, {
173
  headerSize: estimatedHeaderSize
174
});
175

176
// Add roots dynamically with resize option
177
try {
178
  writer.addRoot(root1);
179
  writer.addRoot(root2);
180
  writer.addRoot(root3);
181
  
182
  // This might exceed estimated header size
183
  writer.addRoot(root4, { resize: true }); // Automatically resize header
184
  writer.addRoot(root5, { resize: true });
185
  
186
} catch (error) {
187
  if (error instanceof RangeError && error.message.includes('resize')) {
188
    console.log('Use { resize: true } to expand header');
189
  }
190
}
191

192
// Write blocks and close with resize
193
writer
194
  .write(block1)
195
  .write(block2);
196

197
const carBytes = writer.close({ resize: true }); // Resize to actual header size
198
```
199

200
### Advanced Buffer Management
201

202
Manage buffer allocation and reuse patterns.
203

204
```typescript
205
import * as CarBufferWriter from "@ipld/car/buffer-writer";
206

207
// Pattern 1: Buffer pools for repeated CAR creation
208
class CarBufferPool {
209
  constructor(bufferSize = 1024 * 1024) {
210
    this.bufferSize = bufferSize;
211
    this.availableBuffers = [];
212
  }
213
  
214
  getBuffer() {
215
    return this.availableBuffers.pop() || new ArrayBuffer(this.bufferSize);
216
  }
217
  
218
  returnBuffer(buffer) {
219
    if (buffer.byteLength === this.bufferSize) {
220
      this.availableBuffers.push(buffer);
221
    }
222
  }
223
  
224
  async createCar(roots, blocks) {
225
    const buffer = this.getBuffer();
226
    
227
    try {
228
      const writer = CarBufferWriter.createWriter(buffer, { roots });
229
      
230
      for (const block of blocks) {
231
        writer.write(block);
232
      }
233
      
234
      return writer.close({ resize: true });
235
    } finally {
236
      this.returnBuffer(buffer);
237
    }
238
  }
239
}
240

241
// Pattern 2: Precise allocation for known data sets
242
function createOptimalCar(roots, blocks) {
243
  // Calculate exact size needed
244
  const headerSize = CarBufferWriter.headerLength({ roots });
245
  const dataSize = blocks.reduce(
246
    (total, block) => total + CarBufferWriter.blockLength(block),
247
    0
248
  );
249
  
250
  // Create exactly sized buffer
251
  const buffer = new ArrayBuffer(headerSize + dataSize);
252
  const writer = CarBufferWriter.createWriter(buffer, { 
253
    roots, 
254
    headerSize 
255
  });
256
  
257
  // Write all data
258
  blocks.forEach(block => writer.write(block));
259
  
260
  // No resize needed - should be perfect fit
261
  return writer.close();
262
}
263
```
264

265
### Error Handling
266

267
Handle buffer overflow and sizing errors.
268

269
```typescript
270
import * as CarBufferWriter from "@ipld/car/buffer-writer";
271

272
// Buffer capacity errors
273
const smallBuffer = new ArrayBuffer(1024); // Too small
274
const writer = CarBufferWriter.createWriter(smallBuffer);
275

276
try {
277
  writer.write(largeBlock); // Might exceed buffer capacity
278
} catch (error) {
279
  if (error instanceof RangeError && error.message.includes('capacity')) {
280
    console.log('Buffer too small for block');
281
    
282
    // Calculate needed size and recreate
283
    const neededSize = CarBufferWriter.blockLength(largeBlock);
284
    console.log(`Need at least ${neededSize} bytes`);
285
  }
286
}
287

288
// Header sizing errors
289
try {
290
  writer.addRoot(newRoot); // Might exceed header space
291
} catch (error) {
292
  if (error.message.includes('resize')) {
293
    console.log('Header space exceeded - use resize option');
294
    writer.addRoot(newRoot, { resize: true });
295
  }
296
}
297

298
// Close sizing errors
299
try {
300
  const carBytes = writer.close();
301
} catch (error) {
302
  if (error instanceof RangeError && error.message.includes('overestimated')) {
303
    console.log('Header was overestimated - use resize option');
304
    const carBytes = writer.close({ resize: true });
305
  }
306
}
307
```
308

309
### Performance Optimization
310

311
Optimize for different performance scenarios.
312

313
```typescript
314
import * as CarBufferWriter from "@ipld/car/buffer-writer";
315

316
// High-throughput CAR creation
317
class HighThroughputCarWriter {
318
  constructor() {
319
    // Pre-calculate common header sizes
320
    this.headerSizeCache = new Map();
321
    
322
    // Reuse buffers for similar-sized outputs
323
    this.bufferPool = new Map(); // size -> [buffers]
324
  }
325
  
326
  precalculateHeaderSize(rootCount) {
327
    if (!this.headerSizeCache.has(rootCount)) {
328
      const size = CarBufferWriter.estimateHeaderLength(rootCount);
329
      this.headerSizeCache.set(rootCount, size);
330
    }
331
    return this.headerSizeCache.get(rootCount);
332
  }
333
  
334
  getPooledBuffer(size) {
335
    const roundedSize = Math.ceil(size / 1024) * 1024; // Round to KB
336
    const pool = this.bufferPool.get(roundedSize) || [];
337
    
338
    if (pool.length > 0) {
339
      return pool.pop();
340
    }
341
    
342
    return new ArrayBuffer(roundedSize);
343
  }
344
  
345
  returnBuffer(buffer) {
346
    const size = buffer.byteLength;
347
    const pool = this.bufferPool.get(size) || [];
348
    
349
    if (pool.length < 10) { // Limit pool size
350
      pool.push(buffer);
351
      this.bufferPool.set(size, pool);
352
    }
353
  }
354
  
355
  createCar(roots, blocks) {
356
    // Fast path calculations
357
    const headerSize = this.precalculateHeaderSize(roots.length);
358
    const dataSize = blocks.reduce(
359
      (total, block) => total + CarBufferWriter.blockLength(block),
360
      0
361
    );
362
    
363
    const buffer = this.getPooledBuffer(headerSize + dataSize);
364
    
365
    try {
366
      const writer = CarBufferWriter.createWriter(buffer, { 
367
        roots, 
368
        headerSize 
369
      });
370
      
371
      // Batch write blocks
372
      blocks.forEach(block => writer.write(block));
373
      
374
      return writer.close({ resize: true });
375
    } finally {
376
      this.returnBuffer(buffer);
377
    }
378
  }
379
}
380
```
381

382
## Performance Considerations
383

384
### Memory Efficiency
385
- **Pre-allocation**: Most efficient when buffer size is known in advance
386
- **No Async Overhead**: Synchronous operations avoid Promise/callback overhead
387
- **Buffer Reuse**: Reuse buffers across multiple CAR creations
388

389
### Speed Optimization
390
- **Batch Operations**: Group multiple `write()` calls when possible
391
- **Size Calculation**: Pre-calculate sizes to avoid resizing
392
- **Header Estimation**: Use accurate header size estimates
393

394
### Use Cases
395
- **High-Performance Scenarios**: When async overhead is problematic
396
- **Embedded Systems**: Precise memory control requirements
397
- **Batch Processing**: Creating many small CAR files efficiently
398
- **Memory-Constrained Environments**: When streaming isn't suitable